commit bc2ed1304fad1dc922501dc146c1eaba89da156c Author: CownowAn Date: Fri Mar 15 14:38:51 2024 +0000 first commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..084e810 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +__pycache__ +checkpoints/ +*.pt +data/ +exp/ +vis/ +results/ +.empty/ +.prev/ \ No newline at end of file diff --git a/MobileNetV3/all_path.py b/MobileNetV3/all_path.py new file mode 100644 index 0000000..af016f9 --- /dev/null +++ b/MobileNetV3/all_path.py @@ -0,0 +1,9 @@ +RAW_DATA_PATH="./data/ofa/raw_data" +PROCESSED_DATA_PATH = "./data/ofa/data_transfer_nag" +SCORE_MODEL_DATA_PATH="./data/ofa/data_score_model/ofa_database_500000.pt" +SCORE_MODEL_DATA_IDX_PATH="./data/ofa/data_score_model/ridx-500000.pt" + +NOISE_META_PREDICTOR_CKPT_PATH = "./checkpoints/ofa/noise_aware_meta_surrogate/model_best.pth.tar" +SCORE_MODEL_CKPT_PATH="./checkpoints/ofa/score_model/model_best.pth.tar" +UNNOISE_META_PREDICTOR_CKPT_PATH="./checkpoints/ofa/unnoised_meta_surrogate_from_metad2a" +CONFIG_PATH='./configs/transfer_nag_ofa.pt' \ No newline at end of file diff --git a/MobileNetV3/analysis/arch_functions.py b/MobileNetV3/analysis/arch_functions.py new file mode 100644 index 0000000..d56c522 --- /dev/null +++ b/MobileNetV3/analysis/arch_functions.py @@ -0,0 +1,475 @@ +import numpy as np +import torch +import wandb +import igraph +from torch.nn.functional import one_hot + + +KS_LIST = [3, 5, 7] +EXPAND_LIST = [3, 4, 6] +DEPTH_LIST = [2, 3, 4] +NUM_STAGE = 5 +MAX_LAYER_PER_STAGE = 4 +MAX_N_BLOCK= NUM_STAGE * MAX_LAYER_PER_STAGE # 20 +OPS = { + '3-3': 0, '3-4': 1, '3-6': 2, + '5-3': 3, '5-4': 4, '5-6': 5, + '7-3': 6, '7-4': 7, '7-6': 8, + } + +OPS2STR = { + 0: '3-3', 1: '3-4', 2: '3-6', + 3: '5-3', 4: '5-4', 5: '5-6', + 6: '7-3', 7: '7-4', 8: '7-6', + } +NUM_OPS = len(OPS) +LONGEST_PATH_LENGTH = 20 + + +class BasicArchMetricsOFA(object): + def __init__(self, train_ds=None, train_arch_str_list=None, except_inout=False, data_root=None): + if data_root is not None: + self.ofa = torch.load(data_root) + self.train_arch_list = self.ofa['x'] + else: + self.ofa = None + self.train_arch_list = None + # self.ofa = torch.load(data_root) + self.ops_decoder = OPS + self.except_inout = except_inout + + def get_string_from_onehot_x(self, x): + # node_types = torch.nonzero(torch.tensor(x).long(), as_tuple=True)[1] + x = torch.tensor(x) + ds = torch.sum(x.view(NUM_STAGE, -1), dim=1) + string = '' + for i, _ in enumerate(x): + if sum(_) == 0: + string += '0-0-0_' + else: + string += f'{int(ds[int(i/MAX_LAYER_PER_STAGE)])}-' + OPS2STR[torch.nonzero(torch.tensor(_)).item()] + '_' + return string[:-1] + + + def compute_validity(self, generated, adj=None, mask=None): + """ generated: list of couples (positions, node_types)""" + valid = [] + error_types = [] + valid_str = [] + for x in generated: + is_valid, error_type = is_valid_OFA_x(x) + if is_valid: + valid.append(torch.tensor(x).long()) + valid_str.append(self.get_string_from_onehot_x(x)) + else: + error_types.append(error_type) + + return valid, len(valid) / len(generated), valid_str, None, error_types + + def compute_uniqueness(self, valid_arch): + unique = [] + for x in valid_arch: + if not any([torch.equal(x, tr_m) for tr_m in unique]): + unique.append(x) + return unique, len(unique) / len(valid_arch) + + def compute_novelty(self, unique): + num_novel = 0 + novel = [] + if self.train_arch_list is None: + print("Dataset arch_str is None, novelty computation skipped") + return 1, 1 + for arch in unique: + if not any([torch.equal(arch, tr_m) for tr_m in self.train_arch_list]): + # if arch not in self.train_arch_list[1:]: + novel.append(arch) + num_novel += 1 + return novel, num_novel / len(unique) + + def evaluate(self, generated, adj, mask, check_dataname='cifar10'): + """ generated: list of pairs """ + valid_arch, validity, _, _, error_types = self.compute_validity(generated, adj, mask) + + print(f"Validity over {len(generated)} archs: {validity * 100 :.2f}%") + error_1 = torch.sum(torch.tensor(error_types) == 1) / len(generated) + error_2 = torch.sum(torch.tensor(error_types) == 2) / len(generated) + error_3 = torch.sum(torch.tensor(error_types) == 3) / len(generated) + print(f"Unvalid-Multi_Node_Type over {len(generated)} archs: {error_1 * 100 :.2f}%") + print(f"INVALID_1OR2 over {len(generated)} archs: {error_2 * 100 :.2f}%") + print(f"INVALID_3AND4 over {len(generated)} archs: {error_3 * 100 :.2f}%") + # print(f"Number of connected components of {len(generated)} molecules: min:{nc_min:.2f} mean:{nc_mu:.2f} max:{nc_max:.2f}") + + if validity > 0: + unique, uniqueness = self.compute_uniqueness(valid_arch) + print(f"Uniqueness over {len(valid_arch)} valid archs: {uniqueness * 100 :.2f}%") + + if self.train_arch_list is not None: + _, novelty = self.compute_novelty(unique) + print(f"Novelty over {len(unique)} unique valid archs: {novelty * 100 :.2f}%") + else: + novelty = -1.0 + + else: + novelty = -1.0 + uniqueness = 0.0 + unique = [] + + test_acc_list, flops_list, params_list, latency_list = [0], [0], [0], [0] + all_arch_str = None + return ([validity, uniqueness, novelty, error_1, error_2, error_3], + unique, + dict(test_acc_list=test_acc_list, flops_list=flops_list, params_list=params_list, latency_list=latency_list), + all_arch_str) + + +class BasicArchMetricsMetaOFA(object): + def __init__(self, train_ds=None, train_arch_str_list=None, except_inout=False, data_root=None): + if data_root is not None: + self.ofa = torch.load(data_root) + self.train_arch_list = self.ofa['x'] + else: + self.ofa = None + self.train_arch_list = None + self.ops_decoder = OPS + + def get_string_from_onehot_x(self, x): + x = torch.tensor(x) + ds = torch.sum(x.view(NUM_STAGE, -1), dim=1) + string = '' + for i, _ in enumerate(x): + if sum(_) == 0: + string += '0-0-0_' + else: + string += f'{int(ds[int(i/MAX_LAYER_PER_STAGE)])}-' + OPS2STR[torch.nonzero(torch.tensor(_)).item()] + '_' + return string[:-1] + + def compute_validity(self, generated, adj=None, mask=None): + """ generated: list of couples (positions, node_types)""" + valid = [] + valid_arch_str = [] + all_arch_str = [] + error_types = [] + for x in generated: + is_valid, error_type = is_valid_OFA_x(x) + if is_valid: + valid.append(torch.tensor(x).long()) + arch_str = self.get_string_from_onehot_x(x) + valid_arch_str.append(arch_str) + else: + arch_str = None + error_types.append(error_type) + all_arch_str.append(arch_str) + validity = 0 if len(generated) == 0 else (len(valid)/len(generated)) + return valid, validity, valid_arch_str, all_arch_str, error_types + + def compute_uniqueness(self, valid_arch): + unique = [] + for x in valid_arch: + if not any([torch.equal(x, tr_m) for tr_m in unique]): + unique.append(x) + return unique, len(unique) / len(valid_arch) + + def compute_novelty(self, unique): + num_novel = 0 + novel = [] + if self.train_arch_list is None: + print("Dataset arch_str is None, novelty computation skipped") + return 1, 1 + for arch in unique: + if not any([torch.equal(arch, tr_m) for tr_m in self.train_arch_list]): + novel.append(arch) + num_novel += 1 + return novel, num_novel / len(unique) + + def evaluate(self, generated, adj, mask, check_dataname='imagenet1k'): + """ generated: list of pairs """ + valid_arch, validity, _, _, error_types = self.compute_validity(generated, adj, mask) + + print(f"Validity over {len(generated)} archs: {validity * 100 :.2f}%") + error_1 = torch.sum(torch.tensor(error_types) == 1) / len(generated) + error_2 = torch.sum(torch.tensor(error_types) == 2) / len(generated) + error_3 = torch.sum(torch.tensor(error_types) == 3) / len(generated) + print(f"Unvalid-Multi_Node_Type over {len(generated)} archs: {error_1 * 100 :.2f}%") + print(f"INVALID_1OR2 over {len(generated)} archs: {error_2 * 100 :.2f}%") + print(f"INVALID_3AND4 over {len(generated)} archs: {error_3 * 100 :.2f}%") + + if validity > 0: + unique, uniqueness = self.compute_uniqueness(valid_arch) + print(f"Uniqueness over {len(valid_arch)} valid archs: {uniqueness * 100 :.2f}%") + + if self.train_arch_list is not None: + _, novelty = self.compute_novelty(unique) + print(f"Novelty over {len(unique)} unique valid archs: {novelty * 100 :.2f}%") + else: + novelty = -1.0 + + else: + novelty = -1.0 + uniqueness = 0.0 + unique = [] + + test_acc_list, flops_list, params_list, latency_list = [0], [0], [0], [0] + all_arch_str = None + return ([validity, uniqueness, novelty, error_1, error_2, error_3], + unique, + dict(test_acc_list=test_acc_list, flops_list=flops_list, params_list=params_list, latency_list=latency_list), + all_arch_str) + + +def get_arch_acc_info(nasbench201, arch, dataname='cifar10'): + arch_index = nasbench201['str'].index(arch) + test_acc = nasbench201['test-acc'][dataname][arch_index] + flops = nasbench201['flops'][dataname][arch_index] + params = nasbench201['params'][dataname][arch_index] + latency = nasbench201['latency'][dataname][arch_index] + return test_acc, flops, params, latency + + +def get_arch_acc_info_meta(nasbench201, arch, dataname='cifar10'): + arch_index = nasbench201['str'].index(arch) + flops = nasbench201['flops'][dataname][arch_index] + params = nasbench201['params'][dataname][arch_index] + latency = nasbench201['latency'][dataname][arch_index] + if 'cifar' in dataname: + test_acc = nasbench201['test-acc'][dataname][arch_index] + else: + # TODO + test_acc = None + return arch_index, test_acc, flops, params, latency + + +def is_valid_DAG(g, START_TYPE=0, END_TYPE=1): + res = g.is_dag() + n_start, n_end = 0, 0 + for v in g.vs: + if v['type'] == START_TYPE: + n_start += 1 + elif v['type'] == END_TYPE: + n_end += 1 + if v.indegree() == 0 and v['type'] != START_TYPE: + return False + if v.outdegree() == 0 and v['type'] != END_TYPE: + return False + return res and n_start == 1 and n_end == 1 + +def check_single_node_type(x): + for x_elem in x: + if int(np.sum(x_elem)) != 1: + return False + return True + + +def check_start_end_nodes(x, START_TYPE, END_TYPE): + if x[0][START_TYPE] != 1: + return False + if x[-1][END_TYPE] != 1: + return False + return True + +def check_interm_node_types(x, START_TYPE, END_TYPE): + for x_elem in x[1:-1]: + if x_elem[START_TYPE] == 1: + return False + if x_elem[END_TYPE] == 1: + return False + return True + + +def construct_igraph(node_type, edge_type, ops_decoder, except_inout=True): + assert node_type.shape[0] == edge_type.shape[0] + + START_TYPE = ops_decoder.index('input') + END_TYPE = ops_decoder.index('output') + + g = igraph.Graph(directed=True) + for i, node in enumerate(node_type): + new_type = node.item() + g.add_vertex(type=new_type) + if new_type == END_TYPE: + end_vertices = set([v.index for v in g.vs.select(_outdegree_eq=0) if v.index != g.vcount()-1]) + for v in end_vertices: + g.add_edge(v, i) + elif i > 0: + for ek in range(i): + ek_score = edge_type[ek][i].item() + if ek_score >= 0.5: + g.add_edge(ek, i) + + return g + + +def compute_arch_metrics(arch_list, adj, mask, train_arch_str_list, + train_ds, timestep=None, name=None, except_inout=False, data_root=None): + """ arch_list: (dict) """ + metrics = BasicArchMetricsOFA(data_root=data_root) + arch_metrics = metrics.evaluate(arch_list, adj, mask, check_dataname='cifar10') + all_arch_str = arch_metrics[-1] + + if wandb.run: + arch_prop = arch_metrics[2] + test_acc_list = arch_prop['test_acc_list'] + flops_list = arch_prop['flops_list'] + params_list = arch_prop['params_list'] + latency_list = arch_prop['latency_list'] + if arch_metrics[0][1] > 0.: # uniquness > 0. + dic = { + 'Validity': arch_metrics[0][0], 'Uniqueness': arch_metrics[0][1], 'Novelty': arch_metrics[0][2], + 'test_acc_max': np.max(test_acc_list), 'test_acc_min': np.min(test_acc_list), 'test_acc_mean': np.mean(test_acc_list), 'test_acc_std': np.std(test_acc_list), + 'flops_max': np.max(flops_list), 'flops_min': np.min(flops_list), 'flops_mean': np.mean(flops_list), 'flops_std': np.std(flops_list), + 'params_max': np.max(params_list), 'params_min': np.min(params_list), 'params_mean': np.mean(params_list), 'params_std': np.std(params_list), + 'latency_max': np.max(latency_list), 'latency_min': np.min(latency_list), 'latency_mean': np.mean(latency_list), 'latency_std': np.std(latency_list), + } + else: + dic = { + 'Validity': arch_metrics[0][0], 'Uniqueness': arch_metrics[0][1], 'Novelty': arch_metrics[0][2], + 'test_acc_max': -1, 'test_acc_min': -1, 'test_acc_mean': -1, 'test_acc_std': 0, + 'flops_max': -1, 'flops_min': -1, 'flops_mean': -1, 'flops_std': 0, + 'params_max': -1, 'params_min': -1, 'params_mean': -1, 'params_std': 0, + 'latency_max': -1, 'latency_min': -1, 'latency_mean': -1, 'latency_std': 0, + } + if timestep is not None: + dic.update({'step': timestep}) + + wandb.log(dic) + + return arch_metrics, all_arch_str + +def compute_arch_metrics_meta( + arch_list, adj, mask, train_arch_str_list, train_ds, + timestep=None, check_dataname='cifar10', name=None): + """ arch_list: (dict) """ + + metrics = BasicArchMetricsMetaOFA(train_ds, train_arch_str_list) + arch_metrics = metrics.evaluate(arch_list, adj, mask, check_dataname=check_dataname) + if wandb.run: + arch_prop = arch_metrics[2] + if name != 'ofa': + arch_idx_list = arch_prop['arch_idx_list'] + test_acc_list = arch_prop['test_acc_list'] + flops_list = arch_prop['flops_list'] + params_list = arch_prop['params_list'] + latency_list = arch_prop['latency_list'] + if arch_metrics[0][1] > 0.: # uniquness > 0. + dic = { + 'Validity': arch_metrics[0][0], 'Uniqueness': arch_metrics[0][1], 'Novelty': arch_metrics[0][2], + 'test_acc_max': np.max(test_acc_list), 'test_acc_min': np.min(test_acc_list), 'test_acc_mean': np.mean(test_acc_list), 'test_acc_std': np.std(test_acc_list), + 'flops_max': np.max(flops_list), 'flops_min': np.min(flops_list), 'flops_mean': np.mean(flops_list), 'flops_std': np.std(flops_list), + 'params_max': np.max(params_list), 'params_min': np.min(params_list), 'params_mean': np.mean(params_list), 'params_std': np.std(params_list), + 'latency_max': np.max(latency_list), 'latency_min': np.min(latency_list), 'latency_mean': np.mean(latency_list), 'latency_std': np.std(latency_list), + } + else: + dic = { + 'Validity': arch_metrics[0][0], 'Uniqueness': arch_metrics[0][1], 'Novelty': arch_metrics[0][2], + 'test_acc_max': -1, 'test_acc_min': -1, 'test_acc_mean': -1, 'test_acc_std': 0, + 'flops_max': -1, 'flops_min': -1, 'flops_mean': -1, 'flops_std': 0, + 'params_max': -1, 'params_min': -1, 'params_mean': -1, 'params_std': 0, + 'latency_max': -1, 'latency_min': -1, 'latency_mean': -1, 'latency_std': 0, + } + if timestep is not None: + dic.update({'step': timestep}) + + return arch_metrics + + +def check_multiple_nodes(x): + assert len(x.shape) == 2 + for x_elem in x: + x_elem = np.array(x_elem) + if int(np.sum(x_elem)) > 1: + return False + return True + +def check_inout_node(x, START_TYPE=0, END_TYPE=1): + assert len(x.shape) == 2 + return x[0][START_TYPE] == 1 and x[-1][END_TYPE] == 1 + +def check_none_in_1_and_2_layers(x, NONE_TYPE=None): + assert len(x.shape) == 2 + first_and_second_layers = [0, 1, 4, 5, 8, 9, 12, 13, 16, 17] + for layer in first_and_second_layers: + if int(np.sum(x[layer])) == 0: + return False + return True + +def check_none_in_3_and_4_layers(x, NONE_TYPE=None): + assert len(x.shape) == 2 + third_layers = [2, 6, 10, 14, 18] + + for layer in third_layers: + if int(np.sum(x[layer])) == 0: + if int(np.sum(x[layer+1])) != 0: + return False + return True + + +def check_interm_inout_node(x, START_TYPE, END_TYPE): + for x_elem in x[1:-1]: + if x_elem[START_TYPE] == 1: + return False + if x_elem[END_TYPE] == 1: + return False + + +def is_valid_OFA_x(x): + ERORR = { + 'MULIPLE_NODES': 1, + 'INVALID_1OR2_LAYERS': 2, + 'INVALID_3AND4_LAYERS': 3, + 'NO_ERROR': -1 + } + if not check_multiple_nodes(x): + return False, ERORR['MULIPLE_NODES'] + + if not check_none_in_1_and_2_layers(x): + return False, ERORR['INVALID_1OR2_LAYERS'] + + if not check_none_in_3_and_4_layers(x): + return False, ERORR['INVALID_3AND4_LAYERS'] + + return True, ERORR['NO_ERROR'] + + +def get_x_adj_from_opsdict_ofa(ops): + node_types = torch.zeros(NUM_STAGE * MAX_LAYER_PER_STAGE).long() # w/o in / out + num_vertices = len(OPS.values()) + num_nodes = NUM_STAGE * MAX_LAYER_PER_STAGE + d_matrix = [] + + for i in range(NUM_STAGE): + ds = ops['d'][i] + for j in range(ds): + d_matrix.append(ds) + + for j in range(MAX_LAYER_PER_STAGE - ds): + d_matrix.append('none') + + for i, (ks, e, d) in enumerate(zip( + ops['ks'], ops['e'], d_matrix)): + if d == 'none': + pass + else: + node_types[i] = OPS[f'{ks}-{e}'] + + x = one_hot(node_types, num_vertices).float() + + def get_adj(): + adj = torch.zeros(num_nodes, num_nodes) + for i in range(num_nodes-1): + adj[i, i+1] = 1 + adj = np.array(adj) + return adj + + adj = get_adj() + return x, adj + + +def get_string_from_onehot_x(x): + x = torch.tensor(x) + ds = torch.sum(x.view(NUM_STAGE, -1), dim=1) + string = '' + for i, _ in enumerate(x): + if sum(_) == 0: + string += '0-0-0_' + else: + string += f'{int(ds[int(i/MAX_LAYER_PER_STAGE)])}-' + OPS2STR[torch.nonzero(torch.tensor(_)).item()] + '_' + return string[:-1] \ No newline at end of file diff --git a/MobileNetV3/analysis/arch_metrics.py b/MobileNetV3/analysis/arch_metrics.py new file mode 100644 index 0000000..6d68144 --- /dev/null +++ b/MobileNetV3/analysis/arch_metrics.py @@ -0,0 +1,114 @@ +from analysis.arch_functions import compute_arch_metrics, compute_arch_metrics_meta +from torch import Tensor +import wandb +import torch.nn as nn + + +class SamplingArchMetrics(nn.Module): + def __init__(self, config, train_ds, exp_name): + super().__init__() + + self.exp_name = exp_name + self.train_ds = train_ds + if config.data.name == 'ofa': + self.train_arch_str_list = train_ds.x_list_ + else: + self.train_arch_str_list = train_ds.arch_str_list_ + self.name = config.data.name + self.except_inout = config.data.except_inout + self.data_root = config.data.root + + + def forward(self, arch_list: list, adj, mask, this_sample_dir, test=False, timestep=None): + """_summary_ + :params arch_list: list of archs + :params adj: [batch_size, num_nodes, num_nodes] + :params mask: [batch_size, num_nodes, num_nodes] + """ + arch_metrics, all_arch_str = compute_arch_metrics( + arch_list, adj, mask, self.train_arch_str_list, self.train_ds, timestep=timestep, + name=self.name, except_inout=self.except_inout, data_root=self.data_root) + # arch_metrics + # ([validity, uniqueness, novelty], + # unique, + # dict(test_acc_list=test_acc_list, flops_list=flops_list, params_list=params_list, latency_list=latency_list), + # all_arch_str) + + if test and self.name != 'ofa': + with open(r'final_.txt', 'w') as fp: + for arch_str in all_arch_str: + # write each item on a new line + fp.write("%s\n" % arch_str) + print('All archs saved') + + if self.name != 'ofa': + valid_unique_arch = arch_metrics[1] + valid_unique_arch_prop_dict = arch_metrics[2] # test_acc, flops, params, latency + # textfile = open(f'{this_sample_dir}/archs/{name}/valid_unique_arch_step-{current_step}.txt', "w") + textfile = open(f'{this_sample_dir}/valid_unique_archs.txt', "w") + for i in range(len(valid_unique_arch)): + textfile.write(f"Arch: {valid_unique_arch[i]} \n") + textfile.write(f"Test Acc: {valid_unique_arch_prop_dict['test_acc_list'][i]} \n") + textfile.write(f"FLOPs: {valid_unique_arch_prop_dict['flops_list'][i]} \n ") + textfile.write(f"#Params: {valid_unique_arch_prop_dict['params_list'][i]} \n") + textfile.write(f"Latency: {valid_unique_arch_prop_dict['latency_list'][i]} \n \n") + textfile.writelines(valid_unique_arch) + textfile.close() + + # res_dic = { + # 'Validity': arch_metrics[0][0], 'Uniqueness': arch_metrics[0][1], 'Novelty': arch_metrics[0][2], + # 'test_acc_max': -1, 'test_acc_min':-1, 'test_acc_mean': -1, 'test_acc_std': 0, + # 'flops_max': -1, 'flops_min':-1, 'flops_mean': -1, 'flops_std': 0, + # 'params_max': -1, 'params_min':-1, 'params_mean': -1, 'params_std': 0, + # 'latency_max': -1, 'latency_min':-1, 'latency_mean': -1, 'latency_std': 0, + # } + + return arch_metrics + +class SamplingArchMetricsMeta(nn.Module): + def __init__(self, config, train_ds, exp_name, train_index=None, nasbench=None): + super().__init__() + + self.exp_name = exp_name + self.train_ds = train_ds + self.search_space = config.data.name + if self.search_space == 'ofa': + self.train_arch_str_list = None + else: + self.train_arch_str_list = [train_ds.arch_str_list[i] for i in train_ds.idx_lst['train']] + + def forward(self, arch_list: list, adj, mask, this_sample_dir, test=False, + timestep=None, check_dataname='cifar10'): + """_summary_ + :params arch_list: list of archs + :params adj: [batch_size, num_nodes, num_nodes] + :params mask: [batch_size, num_nodes, num_nodes] + """ + arch_metrics = compute_arch_metrics_meta(arch_list, adj, mask, self.train_arch_str_list, + self.train_ds, timestep=timestep, check_dataname=check_dataname, + name=self.search_space) + all_arch_str = arch_metrics[-1] + + if test: + with open(r'final_.txt', 'w') as fp: + for arch_str in all_arch_str: + # write each item on a new line + fp.write("%s\n" % arch_str) + print('All archs saved') + + valid_unique_arch = arch_metrics[1] # arch_str + valid_unique_arch_prop_dict = arch_metrics[2] # test_acc, flops, params, latency + # textfile = open(f'{this_sample_dir}/archs/{name}/valid_unique_arch_step-{current_step}.txt', "w") + if self.search_space != 'ofa': + textfile = open(f'{this_sample_dir}/valid_unique_archs.txt', "w") + for i in range(len(valid_unique_arch)): + textfile.write(f"Arch: {valid_unique_arch[i]} \n") + textfile.write(f"Arch Index: {valid_unique_arch_prop_dict['arch_idx_list'][i]} \n") + textfile.write(f"Test Acc: {valid_unique_arch_prop_dict['test_acc_list'][i]} \n") + textfile.write(f"FLOPs: {valid_unique_arch_prop_dict['flops_list'][i]} \n ") + textfile.write(f"#Params: {valid_unique_arch_prop_dict['params_list'][i]} \n") + textfile.write(f"Latency: {valid_unique_arch_prop_dict['latency_list'][i]} \n \n") + textfile.writelines(valid_unique_arch) + textfile.close() + + return arch_metrics \ No newline at end of file diff --git a/MobileNetV3/analysis/visualization.py b/MobileNetV3/analysis/visualization.py new file mode 100644 index 0000000..893a931 --- /dev/null +++ b/MobileNetV3/analysis/visualization.py @@ -0,0 +1,547 @@ +import os +import torch +import imageio +import networkx as nx +import numpy as np +# import rdkit.Chem +import wandb +import matplotlib.pyplot as plt +# import igraph +# import pygraphviz as pgv +import datasets_nas +from configs.ckpt import DATAROOT_NB201 + + +class ArchVisualization: + def __init__(self, config, remove_none=False, exp_name=None): + self.config = config + self.remove_none = remove_none + self.exp_name = exp_name + self.num_graphs_to_visualize = config.log.num_graphs_to_visualize + self.nasbench201 = torch.load(DATAROOT_NB201) + + self.labels = { + 0: 'input', + 1: 'output', + 2: 'conv3', + 3: 'sep3', + 4: 'conv5', + 5: 'sep5', + 6: 'avg3', + 7: 'max3', + } + + self.colors = ['skyblue', 'pink', 'yellow', 'orange', 'greenyellow', 'green', 'azure', 'beige'] + + + def to_networkx_directed(self, node_list, adjacency_matrix): + """ + Convert graphs to neural architectures + node_list: the nodes of a batch of nodes (bs x n) + adjacency_matrix: the adjacency_matrix of the molecule (bs x n x n) + """ + + + graph = nx.DiGraph() + # add nodes to the graph + for i in range(len(node_list)): + if node_list[i] == -1: + continue + graph.add_node(i, number=i, symbol=node_list[i], color_val=node_list[i]) + + rows, cols = np.where(torch.triu(torch.tensor(adjacency_matrix), diagonal=1).numpy() >= 1) + edges = zip(rows.tolist(), cols.tolist()) + for edge in edges: + edge_type = adjacency_matrix[edge[0]][edge[1]] + graph.add_edge(edge[0], edge[1], color=float(edge_type), weight=3 * edge_type) + + return graph + + def visualize_non_molecule(self, graph, pos, path, iterations=100, node_size=1200, largest_component=False): + if largest_component: + CGs = [graph.subgraph(c) for c in nx.connected_components(graph)] + CGs = sorted(CGs, key=lambda x: x.number_of_nodes(), reverse=True) + graph = CGs[0] + + # Plot the graph structure with colors + if pos is None: + pos = nx.nx_pydot.graphviz_layout(graph, prog="dot") + # pos = nx.multipartite_layout(graph, subset_key='number') + # pos = nx.spring_layout(graph, iterations=iterations) + + # Set node colors based on the operations + + plt.figure() + nx.draw(graph, pos=pos, labels=self.labels, arrows=True, node_shape="s", + node_size=node_size, node_color=self.colors, edge_color='grey', with_labels=True) + # nx.draw(graph, pos, font_size=5, node_size=node_size, with_labels=False, node_color=U[:, 1], + # cmap=plt.cm.coolwarm, vmin=vmin, vmax=vmax, edge_color='grey') + # import pdb; pdb.set_trace() + # plt.tight_layout() + + plt.savefig(path) + plt.close("all") + + def visualize(self, path: str, graphs: list, log='graph', adj=None): + # define path to save figures + os.makedirs(path, exist_ok=True) + + # visualize the final molecules + for i in range(self.num_graphs_to_visualize): + file_path = os.path.join(path, 'graph_{}.png'.format(i)) + graph = self.to_networkx_directed(graphs[i], adj[0].detach().cpu().numpy()) + self.visualize_non_molecule(graph, pos=None, path=file_path) + im = plt.imread(file_path) + if wandb.run and log is not None: + wandb.log({log: [wandb.Image(im, caption=file_path)]}) + + def visualize_chain(self, path, sample_list, adjacency_matrix, + r_valid_chain, r_uniqueness_chain, r_novel_chain): + import pdb; pdb.set_trace() + # convert graphs to networkx + graphs = [self.to_networkx_directed(sample_list[i], adjacency_matrix[i]) for i in range(sample_list.shape[0])] + # find the coordinates of atoms in the final molecule + final_graph = graphs[-1] + final_pos = nx.nx_pydot.graphviz_layout(final_graph, prog="dot") + # final_pos = None + + # draw gif + save_paths = [] + num_frams = sample_list + + for frame in range(num_frams): + file_name = os.path.join(path, 'frame_{}.png'.format(frame)) + self.visualize_non_molecule(graphs[frame], pos=final_pos, path=file_name) + save_paths.append(file_name) + + imgs = [imageio.imread(fn) for fn in save_paths] + gif_path = os.path.join(os.path.dirname(path), '{}.gif'.format(path.split('/')[-1])) + print(f'==> Save gif at {gif_path}') + imgs.extend([imgs[-1]] * 10) + imageio.mimsave(gif_path, imgs, subrectangles=True, fps=5) + if wandb.run: + wandb.log({'chain': [wandb.Video(gif_path, caption=gif_path, format="gif")]}) + + + def visualize_chain_vun(self, path, r_valid_chain, r_unique_chain, r_novel_chain, sde, sampling_eps, number_chain_steps=None): + + os.makedirs(path, exist_ok=True) + # timesteps = torch.linspace(sampling_eps, sde.T, sde.N) + timesteps = torch.linspace(sde.T, sampling_eps, sde.N) + + if number_chain_steps is not None: + timesteps_ = [] + n = int(sde.N / number_chain_steps) + for i, t in enumerate(timesteps): + if i % n == n - 1: + timesteps_.append(t.item()) + # timesteps_ = [t for i, t in enumerate(timesteps) if i % n == n-1] + assert len(timesteps_) == number_chain_steps + timesteps_ = timesteps_[::-1] + + else: + timesteps_ = list(timesteps.numpy())[::-1] + + # validity + plt.clf() + fig, ax = plt.subplots() + ax.plot(timesteps_, r_valid_chain, color='red') + ax.set_title(f'Validity') + ax.set_xlabel('time') + ax.set_ylabel('Validity') + plt.show() + file_path = os.path.join(path, 'validity.png') + plt.savefig(file_path) + plt.close("all") + print(f'==> Save scatter plot at {file_path}') + im = plt.imread(file_path) + if wandb.run: + wandb.log({'r_valid_chains': [wandb.Image(im, caption=file_path)]}) + + # Uniqueness + plt.clf() + fig, ax = plt.subplots() + ax.plot(timesteps_, r_unique_chain, color='green') + ax.set_title(f'Uniqueness') + ax.set_xlabel('time') + ax.set_ylabel('Uniqueness') + plt.show() + file_path = os.path.join(path, 'uniquness.png') + plt.savefig(file_path) + plt.close("all") + print(f'==> Save scatter plot at {file_path}') + im = plt.imread(file_path) + if wandb.run: + wandb.log({'r_uniqueness_chains': [wandb.Image(im, caption=file_path)]}) + + # Novelty + plt.clf() + fig, ax = plt.subplots() + ax.plot(timesteps_, r_novel_chain, color='blue') + ax.set_title(f'Novelty') + ax.set_xlabel('time') + ax.set_ylabel('Novelty') + file_path = os.path.join(path, 'novelty.png') + plt.savefig(file_path) + plt.close("all") + print(f'==> Save scatter plot at {file_path}') + im = plt.imread(file_path) + if wandb.run: + wandb.log({'r_novelty_chains': [wandb.Image(im, caption=file_path)]}) + + + def visualize_grad_norm(self, path, score_grad_norm_p, classifier_grad_norm_p, + score_grad_norm_c, classifier_grad_norm_c, sde, sampling_eps, + number_chain_steps=None): + + os.makedirs(path, exist_ok=True) + # timesteps = torch.linspace(sampling_eps, sde.T, sde.N) + timesteps = torch.linspace(sde.T, sampling_eps, sde.N) + timesteps_ = list(timesteps.numpy())[::-1] + + if len(score_grad_norm_c) == 0: + score_grad_norm_c = [-1] * len(score_grad_norm_p) + if len(classifier_grad_norm_c) == 0: + classifier_grad_norm_c = [-1] * len(classifier_grad_norm_p) + + plt.clf() + fig, ax1 = plt.subplots() + + color_1 = 'red' + ax1.set_title(f'grad_norm (predictor)') + ax1.set_xlabel('time') + ax1.set_ylabel('score_grad_norm (predictor)', color=color_1) + ax1.plot(timesteps_, score_grad_norm_p, color=color_1) + ax1.tick_params(axis='y', labelcolor=color_1) + + ax2 = ax1.twinx() + color_2 = 'blue' + ax2.set_ylabel('classifier_grad_norm (predictor)', color=color_2) + ax2.plot(timesteps_, classifier_grad_norm_p, color=color_2) + ax2.tick_params(axis='y', labelcolor=color_2) + fig.tight_layout() + plt.show() + + file_path = os.path.join(path, 'grad_norm_p.png') + plt.savefig(file_path) + plt.close("all") + print(f'==> Save scatter plot at {file_path}') + im = plt.imread(file_path) + if wandb.run: + wandb.log({'grad_norm_p': [wandb.Image(im, caption=file_path)]}) + + + plt.clf() + fig, ax1 = plt.subplots() + + color_1 = 'green' + ax1.set_title(f'grad_norm (corrector)') + ax1.set_xlabel('time') + ax1.set_ylabel('score_grad_norm (corrector)', color=color_1) + ax1.plot(timesteps_, score_grad_norm_c, color=color_1) + ax1.tick_params(axis='y', labelcolor=color_1) + + ax2 = ax1.twinx() + color_2 = 'yellow' + ax2.set_ylabel('classifier_grad_norm (corrector)', color=color_2) + ax2.plot(timesteps_, classifier_grad_norm_c, color=color_2) + ax2.tick_params(axis='y', labelcolor=color_2) + fig.tight_layout() + plt.show() + + file_path = os.path.join(path, 'grad_norm_c.png') + plt.savefig(file_path) + plt.close("all") + print(f'==> Save scatter plot at {file_path}') + im = plt.imread(file_path) + if wandb.run: + wandb.log({'grad_norm_c': [wandb.Image(im, caption=file_path)]}) + + + def visualize_scatter(self, path, + score_config, classifier_config, + sampled_arch_metric, plot_textstr=True, + x_axis='latency', y_axis='test-acc', x_label='Latency (ms)', y_label='Accuracy (%)', + log='scatter', check_dataname='cifar10-valid', + selected_arch_idx_list_topN=None, selected_arch_idx_list=None, + train_idx_list=None, return_file_path=False): + + os.makedirs(path, exist_ok=True) + + tg_dataset = classifier_config.data.tg_dataset + + train_ds_s, eval_ds_s, test_ds_s = datasets_nas.get_dataset(score_config) + if selected_arch_idx_list is None: + train_ds_c, eval_ds_c, test_ds_c = datasets_nas.get_dataset(classifier_config) + else: + train_ds_c, eval_ds_c, test_ds_c = datasets_nas.get_dataset_iter(classifier_config) + + plt.clf() + fig, ax = plt.subplots() + + # entire architectures + entire_ds_x = train_ds_s.get_unnoramlized_entire_data(x_axis, tg_dataset) + entire_ds_y = train_ds_s.get_unnoramlized_entire_data(y_axis, tg_dataset) + ax.scatter(entire_ds_x, entire_ds_y, color = 'lightgray', alpha = 0.5, label='Entire', marker=',') + + # architectures trained by the score_model + # train_ds_s_x = train_ds_s.get_unnoramlized_data(x_axis, tg_dataset) + # train_ds_s_y = train_ds_s.get_unnoramlized_data(y_axis, tg_dataset) + # ax.scatter(train_ds_s_x, train_ds_s_y, color = 'gray', alpha = 0.8, label='Trained by Score Model') + + # architectures trained by the classifier + train_ds_c_x = train_ds_c.get_unnoramlized_data(x_axis, tg_dataset) + train_ds_c_y = train_ds_c.get_unnoramlized_data(y_axis, tg_dataset) + ax.scatter(train_ds_c_x, train_ds_c_y, color = 'black', alpha = 0.8, label='Trained by Predictor Model') + + # oracle + oracle_idx = torch.argmax(torch.tensor(entire_ds_y)).item() + # oracle_idx = torch.argmax(torch.tensor(train_ds_s.get_unnoramlized_entire_data('val-acc', tg_dataset))).item() + oracle_item_x = entire_ds_x[oracle_idx] + oracle_item_y = entire_ds_y[oracle_idx] + ax.scatter(oracle_item_x, oracle_item_y, color = 'red', alpha = 1.0, label='Oracle', marker='*', s=150) + + # architectures sampled by the score_model & classifier + AXIS_TO_PROP = { + 'val-acc': 'val_acc_list', + 'test-acc': 'test_acc_list', + 'latency': 'latency_list', + 'flops': 'flops_list', + 'params': 'params_list', + } + sampled_ds_c_x = sampled_arch_metric[2][AXIS_TO_PROP[x_axis]] + sampled_ds_c_y = sampled_arch_metric[2][AXIS_TO_PROP[y_axis]] + ax.scatter(sampled_ds_c_x, sampled_ds_c_y, color = 'limegreen', alpha = 0.8, label='Sampled', marker='x') + + ax.set_title(f'{tg_dataset.upper()} Dataset') + ax.set_xlabel(x_label) + ax.set_ylabel(y_label) + + + if selected_arch_idx_list_topN is not None: + selected_arch_topN_info_dict = get_arch_acc_info_dict( + self.nasbench201, dataname=check_dataname, arch_index_list=selected_arch_idx_list_topN) + selected_topN_ds_x = selected_arch_topN_info_dict[AXIS_TO_PROP[x_axis]] + selected_topN_ds_y = selected_arch_topN_info_dict[AXIS_TO_PROP[y_axis]] + ax.scatter(selected_topN_ds_x, selected_topN_ds_y, color = 'pink', alpha = 0.8, label='Selected_topN', marker='x') + + # architectures selected by the prdictor + selected_ds_x, selected_ds_y = None, None + if selected_arch_idx_list is not None: + selected_arch_info_dict = get_arch_acc_info_dict( + self.nasbench201, dataname=check_dataname, arch_index_list=selected_arch_idx_list) + selected_ds_x = selected_arch_info_dict[AXIS_TO_PROP[x_axis]] + selected_ds_y = selected_arch_info_dict[AXIS_TO_PROP[y_axis]] + ax.scatter(selected_ds_x, selected_ds_y, color = 'blue', alpha = 0.8, label='Selected', marker='x') + + if plot_textstr: + textstr = self.get_textstr(sampled_arch_metric=sampled_arch_metric, + sampled_ds_c_x=sampled_ds_c_x, sampled_ds_c_y=sampled_ds_c_y, + x_axis=x_axis, y_axis=y_axis, + classifier_config=classifier_config, + selected_ds_x=selected_ds_x, selected_ds_y=selected_ds_y, + selected_topN_ds_x=selected_topN_ds_x, selected_topN_ds_y=selected_topN_ds_y, + oracle_idx=oracle_idx, train_idx_list=train_idx_list + ) + + props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) + ax.text(0.6, 0.4, textstr, transform=ax.transAxes, verticalalignment='bottom', bbox=props, fontsize='x-small') + # ax.text(textstr, transform=ax.transAxes, verticalalignment='bottom', bbox=props) + ax.legend(loc="lower right") + + plt.subplots_adjust(left=0, bottom=0, right=1, top=1) + plt.show() + plt.tight_layout() + + file_path = os.path.join(path, 'scatter.png') + plt.savefig(file_path) + plt.close("all") + print(f'==> Save scatter plot at {path}') + + if return_file_path: + return file_path + + im = plt.imread(file_path) + if wandb.run and log is not None: + wandb.log({log: [wandb.Image(im, caption=file_path)]}) + + # if return_selected_arch_info_dict: + # return selected_arch_info_dict, selected_arch_topN_info_dict + + def visualize_scatter_chain(self, path, score_config, classifier_config, sampled_arch_metric_chain, plot_textstr=True, + x_axis='latency', y_axis='test-acc', x_label='Latency (ms)', y_label='Accuracy (%)', + log='scatter_chain'): + + # draw gif + os.makedirs(path, exist_ok=True) + save_paths = [] + num_frames = len(sampled_arch_metric_chain) + + tg_dataset = classifier_config.data.tg_dataset + + train_ds_s, eval_ds_s, test_ds_s = datasets_nas.get_dataset(score_config) + train_ds_c, eval_ds_c, test_ds_c = datasets_nas.get_dataset(classifier_config) + + # entire architectures + entire_ds_x = train_ds_s.get_unnoramlized_entire_data(x_axis, tg_dataset) + entire_ds_y = train_ds_s.get_unnoramlized_entire_data(y_axis, tg_dataset) + + # architectures trained by the score_model + train_ds_s_x = train_ds_s.get_unnoramlized_data(x_axis, tg_dataset) + train_ds_s_y = train_ds_s.get_unnoramlized_data(y_axis, tg_dataset) + + # architectures trained by the classifier + train_ds_c_x = train_ds_c.get_unnoramlized_data(x_axis, tg_dataset) + train_ds_c_y = train_ds_c.get_unnoramlized_data(y_axis, tg_dataset) + + # oracle + # oracle_idx = torch.argmax(torch.tensor(entire_ds_y)).item() + oracle_idx = torch.argmax(torch.tensor(train_ds_s.get_unnoramlized_entire_data('val-acc', tg_dataset))).item() + oracle_item_x = entire_ds_x[oracle_idx] + oracle_item_y = entire_ds_y[oracle_idx] + + for frame in range(num_frames): + sampled_arch_metric = sampled_arch_metric_chain[frame] + + plt.clf() + fig, ax = plt.subplots() + + # entire architectures + ax.scatter(entire_ds_x, entire_ds_y, color = 'lightgray', alpha = 0.5, label='Entire', marker=',') + # architectures trained by the score_model + ax.scatter(train_ds_s_x, train_ds_s_y, color = 'gray', alpha = 0.8, label='Trained by Score Model') + # architectures trained by the classifier + ax.scatter(train_ds_c_x, train_ds_c_y, color = 'black', alpha = 0.8, label='Trained by Predictor Model') + # oracle + ax.scatter(oracle_item_x, oracle_item_y, color = 'red', alpha = 1.0, label='Oracle', marker='*', s=150) + # architectures sampled by the score_model & classifier + AXIS_TO_PROP = { + 'test-acc': 'test_acc_list', + 'latency': 'latency_list', + 'flops': 'flops_list', + 'params': 'params_list', + } + sampled_ds_c_x = sampled_arch_metric[2][AXIS_TO_PROP[x_axis]] + sampled_ds_c_y = sampled_arch_metric[2][AXIS_TO_PROP[y_axis]] + ax.scatter(sampled_ds_c_x, sampled_ds_c_y, color = 'limegreen', alpha = 0.8, label='Sampled', marker='x') + + ax.set_title(f'{tg_dataset.upper()} Dataset') + ax.set_xlabel(x_label) + ax.set_ylabel(y_label) + + if plot_textstr: + textstr = self.get_textstr(sampled_arch_metric, sampled_ds_c_x, sampled_ds_c_y, + x_axis, y_axis, classifier_config) + props = dict(boxstyle='round', facecolor='wheat', alpha=0.5) + ax.text(0.6, 0.3, textstr, transform=ax.transAxes, verticalalignment='bottom', bbox=props) + # ax.text(textstr, transform=ax.transAxes, verticalalignment='bottom', bbox=props) + ax.legend(loc="lower right") + + plt.subplots_adjust(left=0, bottom=0, right=1, top=1) + plt.show() + # plt.tight_layout() + + file_path = os.path.join(path, f'frame_{frame}.png') + plt.savefig(file_path) + plt.close("all") + print(f'==> Save scatter plot at {file_path}') + save_paths.append(file_path) + + im = plt.imread(file_path) + if wandb.run and log is not None: + wandb.log({log: [wandb.Image(im, caption=file_path)]}) + + # draw gif + imgs = [imageio.imread(fn) for fn in save_paths[::-1]] + # gif_path = os.path.join(os.path.dirname(path), '{}.gif'.format(path.split('/')[-1])) + gif_path = os.path.join(path, f'scatter.gif') + print(f'==> Save gif at {gif_path}') + imgs.extend([imgs[-1]] * 10) + # imgs.extend([imgs[0]] * 10) + imageio.mimsave(gif_path, imgs, subrectangles=True, fps=5) + if wandb.run: + wandb.log({'chain_gif': [wandb.Video(gif_path, caption=gif_path, format="gif")]}) + + def get_textstr(self, + sampled_arch_metric, + sampled_ds_c_x, sampled_ds_c_y, + x_axis='latency', y_axis='test-acc', + classifier_config=None, + selected_ds_x=None, selected_ds_y=None, + selected_topN_ds_x=None, selected_topN_ds_y=None, + oracle_idx=None, train_idx_list=None): + mean_v_x = round(np.mean(np.array(sampled_ds_c_x)), 4) + std_v_x = round(np.std(np.array(sampled_ds_c_x)), 4) + max_v_x = round(np.max(np.array(sampled_ds_c_x)), 4) + min_v_x = round(np.min(np.array(sampled_ds_c_x)), 4) + + mean_v_y = round(np.mean(np.array(sampled_ds_c_y)), 4) + std_v_y = round(np.std(np.array(sampled_ds_c_y)), 4) + max_v_y = round(np.max(np.array(sampled_ds_c_y)), 4) + min_v_y = round(np.min(np.array(sampled_ds_c_y)), 4) + + if selected_ds_x is not None: + mean_v_x_s = round(np.mean(np.array(selected_ds_x)), 4) + std_v_x_s = round(np.std(np.array(selected_ds_x)), 4) + max_v_x_s = round(np.max(np.array(selected_ds_x)), 4) + min_v_x_s = round(np.min(np.array(selected_ds_x)), 4) + + if selected_ds_y is not None: + mean_v_y_s = round(np.mean(np.array(selected_ds_y)), 4) + std_v_y_s = round(np.std(np.array(selected_ds_y)), 4) + max_v_y_s = round(np.max(np.array(selected_ds_y)), 4) + min_v_y_s = round(np.min(np.array(selected_ds_y)), 4) + + textstr = '' + r_valid, r_unique, r_novel = round(sampled_arch_metric[0][0], 4), round(sampled_arch_metric[0][1], 4), round(sampled_arch_metric[0][2], 4) + textstr += f'V-{r_valid} | U-{r_unique} | N-{r_novel} \n' + textstr += f'Predictor (Noise-aware-{str(classifier_config.training.noised)[0]}, k={self.config.sampling.classifier_scale}) \n' + textstr += f'=> Sampled {x_axis} \n' + textstr += f'Mean-{mean_v_x} | Std-{std_v_x} \n' + textstr += f'Max-{max_v_x} | Min-{min_v_x} \n' + textstr += f'=> Sampled {y_axis} \n' + textstr += f'Mean-{mean_v_y} | Std-{std_v_y} \n' + textstr += f'Max-{max_v_y} | Min-{min_v_y} \n' + if selected_ds_x is not None: + textstr += f'==> Selected {x_axis} \n' + textstr += f'Mean-{mean_v_x_s} | Std-{std_v_x_s} \n' + textstr += f'Max-{max_v_x_s} | Min-{min_v_x_s} \n' + if selected_ds_y is not None: + textstr += f'==> Selected {y_axis} \n' + textstr += f'Mean-{mean_v_y_s} | Std-{std_v_y_s} \n' + textstr += f'Max-{max_v_y_s} | Min-{min_v_y_s} \n' + if selected_topN_ds_y is not None: + textstr += f'==> Predicted TopN (10) -{str(round(max(selected_topN_ds_y[:10]), 4))} \n' + + if train_idx_list is not None and oracle_idx in train_idx_list: + textstr += f'==> Hit Oracle ({oracle_idx}) !' + + return textstr + + +def get_arch_acc_info_dict(nasbench201, dataname='cifar10-valid', arch_index_list=None): + val_acc_list = [] + test_acc_list = [] + flops_list = [] + params_list = [] + latency_list = [] + + for arch_index in arch_index_list: + val_acc = nasbench201['val-acc'][dataname][arch_index] + val_acc_list.append(val_acc) + test_acc = nasbench201['test-acc'][dataname][arch_index] + test_acc_list.append(test_acc) + flops = nasbench201['flops'][dataname][arch_index] + flops_list.append(flops) + params = nasbench201['params'][dataname][arch_index] + params_list.append(params) + latency = nasbench201['latency'][dataname][arch_index] + latency_list.append(latency) + + return { + 'val_acc_list': val_acc_list, + 'test_acc_list': test_acc_list, + 'flops_list': flops_list, + 'params_list': params_list, + 'latency_list': latency_list + } \ No newline at end of file diff --git a/MobileNetV3/configs/tr_meta_surrogate_ofa.py b/MobileNetV3/configs/tr_meta_surrogate_ofa.py new file mode 100644 index 0000000..2349bd4 --- /dev/null +++ b/MobileNetV3/configs/tr_meta_surrogate_ofa.py @@ -0,0 +1,167 @@ +import ml_collections +import torch +from all_path import SCORE_MODEL_CKPT_PATH, SCORE_MODEL_DATA_PATH + + +def get_config(): + config = ml_collections.ConfigDict() + + config.search_space = None + + # genel + config.resume = False + config.folder_name = 'DiffusionNAG' + config.task = 'tr_meta_predictor' + config.exp_name = None + config.model_type = 'meta_predictor' + config.scorenet_ckpt_path = SCORE_MODEL_CKPT_PATH + config.is_meta = True + + # training + config.training = training = ml_collections.ConfigDict() + training.sde = 'vesde' + training.continuous = True + training.reduce_mean = True + training.noised = True + + training.batch_size = 128 + training.eval_batch_size = 512 + training.n_iters = 20000 + training.snapshot_freq = 500 + training.log_freq = 500 + training.eval_freq = 500 + ## store additional checkpoints for preemption + training.snapshot_freq_for_preemption = 1000 + ## produce samples at each snapshot. + training.snapshot_sampling = True + training.likelihood_weighting = False + # training for perturbed data + training.t_spot = 1. + # training from pretrained score model + training.load_pretrained = False + training.pretrained_model_path = SCORE_MODEL_CKPT_PATH + + # sampling + config.sampling = sampling = ml_collections.ConfigDict() + sampling.method = 'pc' + sampling.predictor = 'euler_maruyama' + sampling.corrector = 'none' + # sampling.corrector = 'langevin' + sampling.rtol = 1e-5 + sampling.atol = 1e-5 + sampling.ode_method = 'dopri5' # 'rk4' + sampling.ode_step = 0.01 + + sampling.n_steps_each = 1 + sampling.noise_removal = True + sampling.probability_flow = False + sampling.snr = 0.16 + sampling.vis_row = 4 + sampling.vis_col = 4 + + # conditional + sampling.classifier_scale = 1.0 + sampling.regress = True + sampling.labels = 'max' + sampling.weight_ratio = False + sampling.weight_scheduling = False + sampling.t_spot = 1. + sampling.t_spot_end = 0. + sampling.number_chain_steps = 50 + sampling.check_dataname = 'imagenet1k' + + # evaluation + config.eval = evaluate = ml_collections.ConfigDict() + evaluate.begin_ckpt = 5 + evaluate.end_ckpt = 20 + # evaluate.batch_size = 512 + evaluate.batch_size = 128 + evaluate.enable_sampling = True + evaluate.num_samples = 1024 + evaluate.mmd_distance = 'RBF' + evaluate.max_subgraph = False + evaluate.save_graph = False + + # data + config.data = data = ml_collections.ConfigDict() + data.centered = True + data.dequantization = False + + data.root = SCORE_MODEL_DATA_PATH + data.name = 'ofa' + data.split_ratio = 0.8 + data.dataset_idx = 'random' + data.max_node = 20 + data.n_vocab = 9 + data.START_TYPE = 0 + data.END_TYPE = 1 + data.num_graphs = 100000 + data.num_channels = 1 + data.except_inout = False # ignore + data.triu_adj = True + data.connect_prev = False + data.tg_dataset = None + data.label_list = ['meta-acc'] + # aug_mask + data.aug_mask_algo = 'none' # 'long_range' | 'floyd' + # num_train + data.num_train = 150 + + # model + config.model = model = ml_collections.ConfigDict() + model.name = 'MetaPredictorCATE' + model.ema_rate = 0.9999 + model.normalization = 'GroupNorm' + model.nonlinearity = 'swish' + model.nf = 128 + model.num_gnn_layers = 4 + model.size_cond = False + model.embedding_type = 'positional' + model.rw_depth = 16 + model.graph_layer = 'PosTransLayer' + model.edge_th = -1. + model.heads = 8 + model.attn_clamp = False + ############################################################################# + # meta + model.input_type = 'DA' + model.hs = 512 + model.nz = 56 + model.num_sample = 20 + + model.num_scales = 1000 + model.beta_min = 0.1 + model.beta_max = 5.0 + model.sigma_min = 0.1 + model.sigma_max = 5.0 + model.dropout = 0.1 + # graph encoder + config.model.graph_encoder = graph_encoder = ml_collections.ConfigDict() + graph_encoder.n_layers = 2 + graph_encoder.d_model = 64 + graph_encoder.n_head = 2 + graph_encoder.d_ff = 32 + graph_encoder.dropout = 0.1 + graph_encoder.n_vocab = 9 + + # optimization + config.optim = optim = ml_collections.ConfigDict() + optim.weight_decay = 0 + optim.optimizer = 'Adam' + optim.lr = 0.001 + optim.beta1 = 0.9 + optim.eps = 1e-8 + optim.warmup = 1000 + optim.grad_clip = 1. + + config.seed = 42 + config.device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + + # log + config.log = log = ml_collections.ConfigDict() + log.use_wandb = True + log.wandb_project_name = 'DiffusionNAG' + log.log_valid_sample_prop = False + log.num_graphs_to_visualize = 20 + + return config diff --git a/MobileNetV3/configs/tr_scorenet_ofa.py b/MobileNetV3/configs/tr_scorenet_ofa.py new file mode 100644 index 0000000..365ef50 --- /dev/null +++ b/MobileNetV3/configs/tr_scorenet_ofa.py @@ -0,0 +1,141 @@ +"""Training PGSN on Community Small Dataset with GraphGDP""" + +import ml_collections +import torch + + +def get_config(): + config = ml_collections.ConfigDict() + + # general + config.resume = False + config.resume_ckpt_path = './exp' + config.folder_name = 'tr_scorenet' + config.task = 'tr_scorenet' + config.exp_name = None + + config.model_type = 'sde' + + # training + config.training = training = ml_collections.ConfigDict() + training.sde = 'vesde' + training.continuous = True + training.reduce_mean = True + + training.batch_size = 256 + training.eval_batch_size = 1000 + training.n_iters = 1000000 + training.snapshot_freq = 10000 + training.log_freq = 200 + training.eval_freq = 10000 + ## store additional checkpoints for preemption + training.snapshot_freq_for_preemption = 5000 + ## produce samples at each snapshot. + training.snapshot_sampling = True + training.likelihood_weighting = False + + # sampling + config.sampling = sampling = ml_collections.ConfigDict() + sampling.method = 'pc' + sampling.predictor = 'euler_maruyama' + sampling.corrector = 'none' + sampling.rtol = 1e-5 + sampling.atol = 1e-5 + sampling.ode_method = 'dopri5' # 'rk4' + sampling.ode_step = 0.01 + + sampling.n_steps_each = 1 + sampling.noise_removal = True + sampling.probability_flow = False + sampling.snr = 0.16 + sampling.vis_row = 4 + sampling.vis_col = 4 + sampling.alpha = 0.5 + sampling.qtype = 'threshold' + + # evaluation + config.eval = evaluate = ml_collections.ConfigDict() + evaluate.begin_ckpt = 5 + evaluate.end_ckpt = 20 + evaluate.batch_size = 1024 + evaluate.enable_sampling = True + evaluate.num_samples = 1024 + evaluate.mmd_distance = 'RBF' + evaluate.max_subgraph = False + evaluate.save_graph = False + + # data + config.data = data = ml_collections.ConfigDict() + data.centered = True + data.dequantization = False + + data.root = './data/ofa/data_score_model/ofa_database_500000.pt' + data.name = 'ofa' + data.split_ratio = 0.9 + data.dataset_idx = 'random' + data.max_node = 20 + data.n_vocab = 9 # 10 # + data.START_TYPE = 0 + data.END_TYPE = 1 + data.num_graphs = 100000 + data.num_channels = 1 + data.except_inout = False + data.triu_adj = True + data.connect_prev = False + data.label_list = None + data.tg_dataset = None + data.node_rule_type = 2 + # aug_mask + data.aug_mask_algo = 'none' + + # model + config.model = model = ml_collections.ConfigDict() + model.name = 'CATE' + model.ema_rate = 0.9999 + model.normalization = 'GroupNorm' + model.nonlinearity = 'swish' + model.nf = 128 + model.num_gnn_layers = 4 + model.size_cond = False + model.embedding_type = 'positional' + model.rw_depth = 16 + model.graph_layer = 'PosTransLayer' + model.edge_th = -1. + model.heads = 8 + model.attn_clamp = False + + model.num_scales = 1000 + model.sigma_min = 0.1 + model.sigma_max = 1.0 + model.dropout = 0.1 + model.pos_enc_type = 2 + # graph encoder + config.model.graph_encoder = graph_encoder = ml_collections.ConfigDict() + graph_encoder.n_layers = 12 + graph_encoder.d_model = 64 + graph_encoder.n_head = 8 + graph_encoder.d_ff = 128 + graph_encoder.dropout = 0.1 + graph_encoder.n_vocab = 9 #10 # 30 + + # optimization + config.optim = optim = ml_collections.ConfigDict() + optim.weight_decay = 0 + optim.optimizer = 'Adam' + optim.lr = 2e-5 + optim.beta1 = 0.9 + optim.eps = 1e-8 + optim.warmup = 1000 + optim.grad_clip = 1. + + config.seed = 42 + config.device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + + # log + config.log = log = ml_collections.ConfigDict() + log.use_wandb = True + log.wandb_project_name = 'DiffusionNAG' + log.log_valid_sample_prop = False + log.num_graphs_to_visualize = 20 + + return config diff --git a/MobileNetV3/datasets_nas.py b/MobileNetV3/datasets_nas.py new file mode 100644 index 0000000..af0a3ed --- /dev/null +++ b/MobileNetV3/datasets_nas.py @@ -0,0 +1,493 @@ +from __future__ import print_function +import torch +import os +import numpy as np +from torch.utils.data import DataLoader, Dataset + +from torch_geometric.utils import to_networkx + +from analysis.arch_functions import get_x_adj_from_opsdict_ofa, get_string_from_onehot_x +from all_path import PROCESSED_DATA_PATH, SCORE_MODEL_DATA_IDX_PATH +from analysis.arch_functions import OPS + + +def get_data_scaler(config): + """Data normalizer. Assume data are always in [0, 1].""" + + if config.data.centered: + # Rescale to [-1, 1] + return lambda x: x * 2. - 1. + else: + return lambda x: x + + +def get_data_inverse_scaler(config): + """Inverse data normalizer.""" + + if config.data.centered: + # Rescale [-1, 1] to [0, 1] + return lambda x: (x + 1.) / 2. + else: + return lambda x: x + + +def networkx_graphs(dataset): + return [to_networkx(dataset[i], to_undirected=False, remove_self_loops=True) for i in range(len(dataset))] + + +def get_dataloader(config, train_dataset, eval_dataset, test_dataset): + train_loader = DataLoader(dataset=train_dataset, + batch_size=config.training.batch_size, + shuffle=True, + collate_fn=collate_fn_ofa if config.model_type == 'meta_predictor' else None) + eval_loader = DataLoader(dataset=eval_dataset, + batch_size=config.training.batch_size, + shuffle=False, + collate_fn=collate_fn_ofa if config.model_type == 'meta_predictor' else None) + test_loader = DataLoader(dataset=test_dataset, + batch_size=config.training.batch_size, + shuffle=False, + collate_fn=collate_fn_ofa if config.model_type == 'meta_predictor' else None) + + return train_loader, eval_loader, test_loader + + +def get_dataloader_iter(config, train_dataset, eval_dataset, test_dataset): + + train_loader = DataLoader(dataset=train_dataset, + batch_size=config.training.batch_size if len(train_dataset) > config.training.batch_size else len(train_dataset), + # batch_size=8, + shuffle=True,) + eval_loader = DataLoader(dataset=eval_dataset, + batch_size=config.training.batch_size if len(eval_dataset) > config.training.batch_size else len(eval_dataset), + # batch_size=8, + shuffle=False,) + test_loader = DataLoader(dataset=test_dataset, + batch_size=config.training.batch_size if len(test_dataset) > config.training.batch_size else len(test_dataset), + # batch_size=8, + shuffle=False,) + + return train_loader, eval_loader, test_loader + + +def is_triu(mat): + is_triu_ = np.allclose(mat, np.triu(mat)) + return is_triu_ + + +def collate_fn_ofa(batch): + # x, adj, label_dict, task + x = torch.stack([item[0] for item in batch]) + adj = torch.stack([item[1] for item in batch]) + label_dict = {} + for item in batch: + for k, v in item[2].items(): + if not k in label_dict.keys(): + label_dict[k] = [] + label_dict[k].append(v) + for k, v in label_dict.items(): + label_dict[k] = torch.tensor(v) + task = [item[3] for item in batch] + return x, adj, label_dict, task + + +def get_dataset(config): + """Create data loaders for training and evaluation. + + Args: + config: A ml_collection.ConfigDict parsed from config files. + + Returns: + train_ds, eval_ds, test_ds + """ + num_train = config.data.num_train if 'num_train' in config.data else None + NASDataset = OFADataset + + train_dataset = NASDataset( + config.data.root, + config.data.split_ratio, + config.data.except_inout, + config.data.triu_adj, + config.data.connect_prev, + 'train', + config.data.label_list, + config.data.tg_dataset, + config.data.dataset_idx, + num_train, + node_rule_type=config.data.node_rule_type) + eval_dataset = NASDataset( + config.data.root, + config.data.split_ratio, + config.data.except_inout, + config.data.triu_adj, + config.data.connect_prev, + 'eval', + config.data.label_list, + config.data.tg_dataset, + config.data.dataset_idx, + num_train, + node_rule_type=config.data.node_rule_type) + + test_dataset = NASDataset( + config.data.root, + config.data.split_ratio, + config.data.except_inout, + config.data.triu_adj, + config.data.connect_prev, + 'test', + config.data.label_list, + config.data.tg_dataset, + config.data.dataset_idx, + num_train, + node_rule_type=config.data.node_rule_type) + + + return train_dataset, eval_dataset, test_dataset + + +def get_meta_dataset(config): + database = MetaTrainDatabaseOFA + data_path = PROCESSED_DATA_PATH + + train_dataset = database( + data_path, + config.model.num_sample, + config.data.label_list, + True, + config.data.except_inout, + config.data.triu_adj, + config.data.connect_prev, + 'train') + eval_dataset = database( + data_path, + config.model.num_sample, + config.data.label_list, + True, + config.data.except_inout, + config.data.triu_adj, + config.data.connect_prev, + 'val') + # test_dataset = MetaTestDataset() + test_dataset = None + return train_dataset, eval_dataset, test_dataset + +def get_meta_dataloader(config ,train_dataset, eval_dataset, test_dataset): + if config.data.name == 'ofa': + train_loader = DataLoader(dataset=train_dataset, + batch_size=config.training.batch_size, + shuffle=True,) + # collate_fn=collate_fn_ofa) + eval_loader = DataLoader(dataset=eval_dataset, + batch_size=config.training.batch_size,) + # collate_fn=collate_fn_ofa) + else: + train_loader = DataLoader(dataset=train_dataset, + batch_size=config.training.batch_size, + shuffle=True) + eval_loader = DataLoader(dataset=eval_dataset, + batch_size=config.training.batch_size, + shuffle=False) + # test_loader = DataLoader(dataset=test_dataset, + # batch_size=config.training.batch_size, + # shuffle=False) + test_loader = None + return train_loader, eval_loader, test_loader + + +class MetaTestDataset(Dataset): + def __init__(self, data_path, data_name, num_sample, num_class=None): + self.num_sample = num_sample + self.data_name = data_name + + num_class_dict = { + 'cifar100': 100, + 'cifar10': 10, + 'mnist': 10, + 'svhn': 10, + 'aircraft': 30, + 'pets': 37 + } + + if num_class is not None: + self.num_class = num_class + else: + self.num_class = num_class_dict[data_name] + self.x = torch.load(os.path.join(data_path, f'aircraft100bylabel.pt' if 'ofa' in data_path and data_name == 'aircraft' else f'{data_name}bylabel.pt' )) + + def __len__(self): + return 1000000 + + def __getitem__(self, index): + data = [] + classes = list(range(self.num_class)) + for cls in classes: + cx = self.x[cls][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + x = torch.cat(data) + return x + + +class MetaTrainDatabaseOFA(Dataset): + # def __init__(self, data_path, num_sample, is_pred=False): + def __init__( + self, + data_path, + num_sample, + label_list, + is_pred=True, + except_inout=False, + triu_adj=True, + connect_prev=False, + mode='train'): + + self.ops_decoder = list(OPS.keys()) + self.mode = mode + self.acc_norm = True + self.num_sample = num_sample + self.x = torch.load(os.path.join(data_path, 'imgnet32bylabel.pt')) + + if is_pred: + self.dpath = f'{data_path}/predictor/processed/' + else: + raise NotImplementedError + + self.dname = 'database_219152_14.0K' + data = torch.load(self.dpath + f'{self.dname}_{self.mode}.pt') + self.net = data['net'] + self.x_list = [] + self.adj_list = [] + self.arch_str_list = [] + for net in self.net: + x, adj = get_x_adj_from_opsdict_ofa(net) + # ---------- matrix ---------- # + self.x_list.append(x) + self.adj_list.append(torch.tensor(adj)) + # ---------- arch_str ---------- # + self.arch_str_list.append(get_string_from_onehot_x(x)) + # ---------- labels ---------- # + self.label_list = label_list + if self.label_list is not None: + self.flops_list = data['flops'] + self.params_list = None + self.latency_list = None + + self.acc_list = data['acc'] + self.mean = data['mean'] + self.std = data['std'] + self.task_lst = data['class'] + + def __len__(self): + return len(self.acc_list) + + def __getitem__(self, index): + data = [] + classes = self.task_lst[index] + acc = self.acc_list[index] + graph = self.net[index] + + # ---------- x ----------- + x = self.x_list[index] + # ---------- adj ---------- + adj = self.adj_list[index] + acc = self.acc_list[index] + + for i, cls in enumerate(classes): + cx = self.x[cls.item()][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + task = torch.cat(data) + if self.acc_norm: + acc = ((acc - self.mean) / self.std) / 100.0 + else: + acc = acc / 100.0 + + label_dict = {} + if self.label_list is not None: + assert type(self.label_list) == list + for label in self.label_list: + if label == 'meta-acc': + label_dict[f"{label}"] = acc + else: + raise ValueError + return x, adj, label_dict, task + + +class OFADataset(Dataset): + def __init__( + self, + data_path, + split_ratio=0.8, + except_inout=False, + triu_adj=True, + connect_prev=False, + mode='train', + label_list=None, + tg_dataset=None, + dataset_idx='random', + num_train=None, + node_rule_type=None): + + # ---------- entire dataset ---------- # + self.data = torch.load(data_path) + self.except_inout = except_inout + self.triu_adj = triu_adj + self.connect_prev = connect_prev + self.node_rule_type = node_rule_type + + # ---------- x ---------- # + self.x_list = self.data['x_none2zero'] + + # ---------- adj ---------- # + assert self.connect_prev == False + self.n_adj = len(self.data['node_type'][0]) + const_adj = self.get_not_connect_prev_adj() + self.adj_list = [const_adj] * len(self.x_list) + + # ---------- arch_str ---------- # + self.arch_str_list = self.data['net_setting'] + # ---------- labels ---------- # + self.label_list = label_list + if self.label_list is not None: + raise NotImplementedError + + # ----------- split dataset ---------- # + self.ds_idx = list(torch.load(SCORE_MODEL_DATA_IDX_PATH)) + + self.split_ratio = split_ratio + if num_train is None: + num_train = int(len(self.x_list) * self.split_ratio) + num_test = len(self.x_list) - num_train + else: + num_train = num_train + num_test = len(self.x_list) - num_train + # ----------- compute mean and std w/ training dataset ---------- # + if self.label_list is not None: + self.train_idx_list = self.ds_idx[:num_train] + print('Computing mean and std of the training set...') + from collections import defaultdict + LABEL_TO_MEAN_STD = defaultdict(dict) + assert type(self.label_list) == list + for label in self.label_list: + if label == 'test-acc': + self.test_acc_list_tr = [self.test_acc_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.test_acc_list_tr)) + elif label == 'flops': + self.flops_list_tr = [self.flops_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.flops_list_tr)) + elif label == 'params': + self.params_list_tr = [self.params_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.params_list_tr)) + elif label == 'latency': + self.latency_list_tr = [self.latency_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.latency_list_tr)) + else: + raise ValueError + + self.mode = mode + if self.mode in ['train']: + self.idx_list = self.ds_idx[:num_train] + elif self.mode in ['eval']: + self.idx_list = self.ds_idx[:num_test] + elif self.mode in ['test']: + self.idx_list = self.ds_idx[num_train:] + + self.x_list_ = [self.x_list[i] for i in self.idx_list] + self.adj_list_ = [self.adj_list[i] for i in self.idx_list] + self.arch_str_list_ = [self.arch_str_list[i] for i in self.idx_list] + + if self.label_list is not None: + assert type(self.label_list) == list + for label in self.label_list: + if label == 'test-acc': + self.test_acc_list_ = [self.test_acc_list[i] for i in self.idx_list] + self.test_acc_list_ = self.normalize(self.test_acc_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + elif label == 'flops': + self.flops_list_ = [self.flops_list[i] for i in self.idx_list] + self.flops_list_ = self.normalize(self.flops_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + elif label == 'params': + self.params_list_ = [self.params_list[i] for i in self.idx_list] + self.params_list_ = self.normalize(self.params_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + elif label == 'latency': + self.latency_list_ = [self.latency_list[i] for i in self.idx_list] + self.latency_list_ = self.normalize(self.latency_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + else: + raise ValueError + + def normalize(self, original, mean, std): + return [(i-mean)/std for i in original] + + def get_not_connect_prev_adj(self): + _adj = torch.zeros(self.n_adj, self.n_adj) + for i in range(self.n_adj-1): + _adj[i, i+1] = 1 + _adj = _adj.to(torch.float32).to('cpu') # torch.tensor(_adj, dtype=torch.float32, device=torch.device('cpu')) + # if self.except_inout: + # _adj = _adj[1:-1, 1:-1] + return _adj + + @property + def adj(self): + return self.adj_list_[0] + + # @property + def mask(self, algo='floyd', data='ofa'): + from utils import aug_mask + return aug_mask(self.adj, algo=algo, data=data)[0] + + def get_unnoramlized_entire_data(self, label, tg_dataset): + entire_test_acc_list = self.data['test-acc'][tg_dataset] + entire_flops_list = self.data['flops'][tg_dataset] + entire_params_list = self.data['params'][tg_dataset] + entire_latency_list = self.data['latency'][tg_dataset] + + if label == 'test-acc': + return entire_test_acc_list + elif label == 'flops': + return entire_flops_list + elif label == 'params': + return entire_params_list + elif label == 'latency': + return entire_latency_list + else: + raise ValueError + + + def get_unnoramlized_data(self, label, tg_dataset): + entire_test_acc_list = self.data['test-acc'][tg_dataset] + entire_flops_list = self.data['flops'][tg_dataset] + entire_params_list = self.data['params'][tg_dataset] + entire_latency_list = self.data['latency'][tg_dataset] + + if label == 'test-acc': + return [entire_test_acc_list[i] for i in self.idx_list] + elif label == 'flops': + return [entire_flops_list[i] for i in self.idx_list] + elif label == 'params': + return [entire_params_list[i] for i in self.idx_list] + elif label == 'latency': + return [entire_latency_list[i] for i in self.idx_list] + else: + raise ValueError + + def __len__(self): + return len(self.x_list_) + + def __getitem__(self, index): + + label_dict = {} + if self.label_list is not None: + assert type(self.label_list) == list + for label in self.label_list: + if label == 'test-acc': + label_dict[f"{label}"] = self.test_acc_list_[index] + elif label == 'flops': + label_dict[f"{label}"] = self.flops_list_[index] + elif label == 'params': + label_dict[f"{label}"] = self.params_list_[index] + elif label == 'latency': + label_dict[f"{label}"] = self.latency_list_[index] + else: + raise ValueError + + return self.x_list_[index], self.adj_list_[index], label_dict \ No newline at end of file diff --git a/MobileNetV3/evaluation/__init__.py b/MobileNetV3/evaluation/__init__.py new file mode 100755 index 0000000..ae2f042 --- /dev/null +++ b/MobileNetV3/evaluation/__init__.py @@ -0,0 +1 @@ +from .evaluator import get_stats_eval, get_nn_eval diff --git a/MobileNetV3/evaluation/evaluator.py b/MobileNetV3/evaluation/evaluator.py new file mode 100644 index 0000000..43e456c --- /dev/null +++ b/MobileNetV3/evaluation/evaluator.py @@ -0,0 +1,58 @@ +import networkx as nx +from .structure_evaluator import mmd_eval +from .gin_evaluator import nn_based_eval +from torch_geometric.utils import to_networkx +import torch +import torch.nn.functional as F +import dgl + + +def get_stats_eval(config): + + if config.eval.mmd_distance.lower() == 'rbf': + method = [('degree', 1., 'argmax'), ('cluster', 0.1, 'argmax'), + ('spectral', 1., 'argmax')] + else: + raise ValueError + + def eval_stats_fn(test_dataset, pred_graph_list): + pred_G = [nx.from_numpy_matrix(pred_adj) for pred_adj in pred_graph_list] + sub_pred_G = [] + if config.eval.max_subgraph: + for G in pred_G: + CGs = [G.subgraph(c) for c in nx.connected_components(G)] + CGs = sorted(CGs, key=lambda x: x.number_of_nodes(), reverse=True) + sub_pred_G += [CGs[0]] + pred_G = sub_pred_G + + test_G = [to_networkx(test_dataset[i], to_undirected=True, remove_self_loops=True) + for i in range(len(test_dataset))] + results = mmd_eval(test_G, pred_G, method) + return results + + return eval_stats_fn + + +def get_nn_eval(config): + + if hasattr(config.eval, "N_gin"): + N_gin = config.eval.N_gin + else: + N_gin = 10 + + def nn_eval_fn(test_dataset, pred_graph_list): + pred_G = [nx.from_numpy_matrix(pred_adj) for pred_adj in pred_graph_list] + sub_pred_G = [] + if config.eval.max_subgraph: + for G in pred_G: + CGs = [G.subgraph(c) for c in nx.connected_components(G)] + CGs = sorted(CGs, key=lambda x: x.number_of_nodes(), reverse=True) + sub_pred_G += [CGs[0]] + pred_G = sub_pred_G + test_G = [to_networkx(test_dataset[i], to_undirected=True, remove_self_loops=True) + for i in range(len(test_dataset))] + + results = nn_based_eval(test_G, pred_G, N_gin) + return results + + return nn_eval_fn diff --git a/MobileNetV3/evaluation/gin.py b/MobileNetV3/evaluation/gin.py new file mode 100644 index 0000000..411d0ff --- /dev/null +++ b/MobileNetV3/evaluation/gin.py @@ -0,0 +1,311 @@ +"""Modified from https://github.com/uoguelph-mlrg/GGM-metrics""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +import dgl.function as fn +from dgl.utils import expand_as_pair +from dgl.nn import SumPooling, AvgPooling, MaxPooling + + +class GINConv(nn.Module): + def __init__(self, + apply_func, + aggregator_type, + init_eps=0, + learn_eps=False): + super(GINConv, self).__init__() + self.apply_func = apply_func + self._aggregator_type = aggregator_type + if aggregator_type == 'sum': + self._reducer = fn.sum + elif aggregator_type == 'max': + self._reducer = fn.max + elif aggregator_type == 'mean': + self._reducer = fn.mean + else: + raise KeyError('Aggregator type {} not recognized.'.format(aggregator_type)) + # to specify whether eps is trainable or not. + if learn_eps: + self.eps = torch.nn.Parameter(torch.FloatTensor([init_eps])) + else: + self.register_buffer('eps', torch.FloatTensor([init_eps])) + + def forward(self, graph, feat, edge_weight=None): + r""" + Description + ----------- + Compute Graph Isomorphism Network layer. + Parameters + ---------- + graph : DGLGraph + The graph. + feat : torch.Tensor or pair of torch.Tensor + If a torch.Tensor is given, the input feature of shape :math:`(N, D_{in})` where + :math:`D_{in}` is size of input feature, :math:`N` is the number of nodes. + If a pair of torch.Tensor is given, the pair must contain two tensors of shape + :math:`(N_{in}, D_{in})` and :math:`(N_{out}, D_{in})`. + If ``apply_func`` is not None, :math:`D_{in}` should + fit the input dimensionality requirement of ``apply_func``. + edge_weight : torch.Tensor, optional + Optional tensor on the edge. If given, the convolution will weight + with regard to the message. + Returns + ------- + torch.Tensor + The output feature of shape :math:`(N, D_{out})` where + :math:`D_{out}` is the output dimensionality of ``apply_func``. + If ``apply_func`` is None, :math:`D_{out}` should be the same + as input dimensionality. + """ + with graph.local_scope(): + aggregate_fn = self.concat_edge_msg + # aggregate_fn = fn.copy_src('h', 'm') + if edge_weight is not None: + assert edge_weight.shape[0] == graph.number_of_edges() + graph.edata['_edge_weight'] = edge_weight + aggregate_fn = fn.u_mul_e('h', '_edge_weight', 'm') + + feat_src, feat_dst = expand_as_pair(feat, graph) + graph.srcdata['h'] = feat_src + graph.update_all(aggregate_fn, self._reducer('m', 'neigh')) + + + diff = torch.tensor(graph.dstdata['neigh'].shape[1: ]) - torch.tensor(feat_dst.shape[1: ]) + zeros = torch.zeros(feat_dst.shape[0], *diff).to(feat_dst.device) + feat_dst = torch.cat([feat_dst, zeros], dim=1) + rst = (1 + self.eps) * feat_dst + graph.dstdata['neigh'] + if self.apply_func is not None: + rst = self.apply_func(rst) + return rst + + def concat_edge_msg(self, edges): + if self.edge_feat_loc not in edges.data: + return {'m': edges.src['h']} + else: + m = torch.cat([edges.src['h'], edges.data[self.edge_feat_loc]], dim=1) + return {'m': m} + + +class ApplyNodeFunc(nn.Module): + """Update the node feature hv with MLP, BN and ReLU.""" + def __init__(self, mlp): + super(ApplyNodeFunc, self).__init__() + self.mlp = mlp + self.bn = nn.BatchNorm1d(self.mlp.output_dim) + + def forward(self, h): + h = self.mlp(h) + h = self.bn(h) + h = F.relu(h) + return h + + +class MLP(nn.Module): + """MLP with linear output""" + def __init__(self, num_layers, input_dim, hidden_dim, output_dim): + """MLP layers construction + + Paramters + --------- + num_layers: int + The number of linear layers + input_dim: int + The dimensionality of input features + hidden_dim: int + The dimensionality of hidden units at ALL layers + output_dim: int + The number of classes for prediction + + """ + super(MLP, self).__init__() + self.linear_or_not = True # default is linear model + self.num_layers = num_layers + self.output_dim = output_dim + + if num_layers < 1: + raise ValueError("number of layers should be positive!") + elif num_layers == 1: + # Linear model + self.linear = nn.Linear(input_dim, output_dim) + + else: + # Multi-layer model + self.linear_or_not = False + self.linears = torch.nn.ModuleList() + self.batch_norms = torch.nn.ModuleList() + + self.linears.append(nn.Linear(input_dim, hidden_dim)) + for layer in range(num_layers - 2): + self.linears.append(nn.Linear(hidden_dim, hidden_dim)) + self.linears.append(nn.Linear(hidden_dim, output_dim)) + + for layer in range(num_layers - 1): + self.batch_norms.append(nn.BatchNorm1d((hidden_dim))) + + def forward(self, x): + if self.linear_or_not: + # If linear model + return self.linear(x) + else: + # If MLP + h = x + for i in range(self.num_layers - 1): + h = F.relu(self.batch_norms[i](self.linears[i](h))) + return self.linears[-1](h) + + +class GIN(nn.Module): + """GIN model""" + def __init__(self, num_layers, num_mlp_layers, input_dim, hidden_dim, + graph_pooling_type, neighbor_pooling_type, edge_feat_dim=0, + final_dropout=0.0, learn_eps=False, output_dim=1, **kwargs): + """model parameters setting + + Paramters + --------- + num_layers: int + The number of linear layers in the neural network + num_mlp_layers: int + The number of linear layers in mlps + input_dim: int + The dimensionality of input features + hidden_dim: int + The dimensionality of hidden units at ALL layers + output_dim: int + The number of classes for prediction + final_dropout: float + dropout ratio on the final linear layer + learn_eps: boolean + If True, learn epsilon to distinguish center nodes from neighbors + If False, aggregate neighbors and center nodes altogether. + neighbor_pooling_type: str + how to aggregate neighbors (sum, mean, or max) + graph_pooling_type: str + how to aggregate entire nodes in a graph (sum, mean or max) + """ + + super().__init__() + + def init_weights_orthogonal(m): + if isinstance(m, nn.Linear): + torch.nn.init.orthogonal_(m.weight) + elif isinstance(m, MLP): + if hasattr(m, 'linears'): + m.linears.apply(init_weights_orthogonal) + else: + m.linear.apply(init_weights_orthogonal) + elif isinstance(m, nn.ModuleList): + pass + else: + raise Exception() + + self.num_layers = num_layers + self.learn_eps = learn_eps + + # List of MLPs + self.ginlayers = torch.nn.ModuleList() + self.batch_norms = torch.nn.ModuleList() + + # self.preprocess_nodes = PreprocessNodeAttrs( + # node_attrs=node_preprocess, output_dim=node_preprocess_output_dim) + # print(input_dim) + for layer in range(self.num_layers - 1): + if layer == 0: + mlp = MLP(num_mlp_layers, input_dim + edge_feat_dim, hidden_dim, hidden_dim) + else: + mlp = MLP(num_mlp_layers, hidden_dim + edge_feat_dim, hidden_dim, hidden_dim) + if kwargs['init'] == 'orthogonal': + init_weights_orthogonal(mlp) + + self.ginlayers.append( + GINConv(ApplyNodeFunc(mlp), neighbor_pooling_type, 0, self.learn_eps)) + self.batch_norms.append(nn.BatchNorm1d(hidden_dim)) + + # Linear function for graph poolings of output of each layer + # which maps the output of different layers into a prediction score + self.linears_prediction = torch.nn.ModuleList() + + for layer in range(num_layers): + if layer == 0: + self.linears_prediction.append( + nn.Linear(input_dim, output_dim)) + else: + self.linears_prediction.append( + nn.Linear(hidden_dim, output_dim)) + + if kwargs['init'] == 'orthogonal': + # print('orthogonal') + self.linears_prediction.apply(init_weights_orthogonal) + + self.drop = nn.Dropout(final_dropout) + + if graph_pooling_type == 'sum': + self.pool = SumPooling() + elif graph_pooling_type == 'mean': + self.pool = AvgPooling() + elif graph_pooling_type == 'max': + self.pool = MaxPooling() + else: + raise NotImplementedError + + def forward(self, g, h): + # list of hidden representation at each layer (including input) + hidden_rep = [h] + + # h = self.preprocess_nodes(h) + for i in range(self.num_layers - 1): + h = self.ginlayers[i](g, h) + h = self.batch_norms[i](h) + h = F.relu(h) + hidden_rep.append(h) + + score_over_layer = 0 + + # perform pooling over all nodes in each graph in every layer + for i, h in enumerate(hidden_rep): + pooled_h = self.pool(g, h) + score_over_layer += self.drop(self.linears_prediction[i](pooled_h)) + return score_over_layer + + def get_graph_embed(self, g, h): + self.eval() + with torch.no_grad(): + # return self.forward(g, h).detach().numpy() + hidden_rep = [] + # h = self.preprocess_nodes(h) + for i in range(self.num_layers - 1): + h = self.ginlayers[i](g, h) + h = self.batch_norms[i](h) + h = F.relu(h) + hidden_rep.append(h) + + # perform pooling over all nodes in each graph in every layer + graph_embed = torch.Tensor([]).to(self.device) + for i, h in enumerate(hidden_rep): + pooled_h = self.pool(g, h) + graph_embed = torch.cat([graph_embed, pooled_h], dim = 1) + + return graph_embed + + def get_graph_embed_no_cat(self, g, h): + self.eval() + with torch.no_grad(): + hidden_rep = [] + # h = self.preprocess_nodes(h) + for i in range(self.num_layers - 1): + h = self.ginlayers[i](g, h) + h = self.batch_norms[i](h) + h = F.relu(h) + hidden_rep.append(h) + + return self.pool(g, hidden_rep[-1]).to(self.device) + + @property + def edge_feat_loc(self): + return self.ginlayers[0].edge_feat_loc + + @edge_feat_loc.setter + def edge_feat_loc(self, loc): + for layer in self.ginlayers: + layer.edge_feat_loc = loc diff --git a/MobileNetV3/evaluation/gin_evaluator.py b/MobileNetV3/evaluation/gin_evaluator.py new file mode 100644 index 0000000..f9be998 --- /dev/null +++ b/MobileNetV3/evaluation/gin_evaluator.py @@ -0,0 +1,292 @@ +"""Evaluation on random GIN features. Modified from https://github.com/uoguelph-mlrg/GGM-metrics""" + +import torch +import numpy as np +import sklearn +import sklearn.metrics +from sklearn.preprocessing import StandardScaler +import time +import dgl + +from .gin import GIN + + +def load_feature_extractor( + device, num_layers=3, hidden_dim=35, neighbor_pooling_type='sum', + graph_pooling_type='sum', input_dim=1, edge_feat_dim=0, + dont_concat=False, num_mlp_layers=2, output_dim=1, + node_feat_loc='attr', edge_feat_loc='attr', init='orthogonal', + **kwargs): + + model = GIN(num_layers=num_layers, hidden_dim=hidden_dim, neighbor_pooling_type=neighbor_pooling_type, + graph_pooling_type=graph_pooling_type, input_dim=input_dim, edge_feat_dim=edge_feat_dim, + num_mlp_layers=num_mlp_layers, output_dim=output_dim, init=init) + + model.node_feat_loc = node_feat_loc + model.edge_feat_loc = edge_feat_loc + + model.eval() + + if dont_concat: + model.forward = model.get_graph_embed_no_cat + else: + model.forward = model.get_graph_embed + + model.device = device + return model.to(device) + + +def time_function(func): + def wrapper(*args, **kwargs): + start = time.time() + results = func(*args, **kwargs) + end = time.time() + return results, end - start + return wrapper + + +class GINMetric(): + def __init__(self, model): + self.feat_extractor = model + self.get_activations = self.get_activations_gin + + @time_function + def get_activations_gin(self, generated_dataset, reference_dataset): + return self._get_activations(generated_dataset, reference_dataset) + + def _get_activations(self, generated_dataset, reference_dataset): + gen_activations = self.__get_activations_single_dataset(generated_dataset) + ref_activations = self.__get_activations_single_dataset(reference_dataset) + + scaler = StandardScaler() + scaler.fit(ref_activations) + ref_activations = scaler.transform(ref_activations) + gen_activations = scaler.transform(gen_activations) + + return gen_activations, ref_activations + + def __get_activations_single_dataset(self, dataset): + + node_feat_loc = self.feat_extractor.node_feat_loc + edge_feat_loc = self.feat_extractor.edge_feat_loc + + ndata = [node_feat_loc] if node_feat_loc in dataset[0].ndata else '__ALL__' + edata = [edge_feat_loc] if edge_feat_loc in dataset[0].edata else '__ALL__' + graphs = dgl.batch(dataset, ndata=ndata, edata=edata).to(self.feat_extractor.device) + + if node_feat_loc not in graphs.ndata: # Use degree as features + feats = graphs.in_degrees() + graphs.out_degrees() + feats = feats.unsqueeze(1).type(torch.float32) + else: + feats = graphs.ndata[node_feat_loc] + + graph_embeds = self.feat_extractor(graphs, feats) + return graph_embeds.cpu().detach().numpy() + + def evaluate(self, *args, **kwargs): + raise Exception('Must be implemented by child class') + + +class MMDEvaluation(GINMetric): + def __init__(self, model, kernel='rbf', sigma='range', multiplier='mean'): + super().__init__(model) + + if multiplier == 'mean': + self.__get_sigma_mult_factor = self.__mean_pairwise_distance + elif multiplier == 'median': + self.__get_sigma_mult_factor = self.__median_pairwise_distance + elif multiplier is None: + self.__get_sigma_mult_factor = lambda *args, **kwargs: 1 + else: + raise Exception(multiplier) + + if 'rbf' in kernel: + if sigma == 'range': + self.base_sigmas = np.array([0.01, 0.1, 0.25, 0.5, 0.75, 1.0, 2.5, 5.0, 7.5, 10.0]) + + if multiplier == 'mean': + self.name = 'mmd_rbf' + elif multiplier == 'median': + self.name = 'mmd_rbf_adaptive_median' + else: + self.name = 'mmd_rbf_adaptive' + elif sigma == 'one': + self.base_sigmas = np.array([1]) + + if multiplier == 'mean': + self.name = 'mmd_rbf_single_mean' + elif multiplier == 'median': + self.name = 'mmd_rbf_single_median' + else: + self.name = 'mmd_rbf_single' + else: + raise Exception(sigma) + + self.evaluate = self.calculate_MMD_rbf_quadratic + + elif 'linear' in kernel: + self.evaluate = self.calculate_MMD_linear_kernel + + else: + raise Exception() + + def __get_pairwise_distances(self, generated_dataset, reference_dataset): + return sklearn.metrics.pairwise_distances(reference_dataset, generated_dataset, metric='euclidean', n_jobs=8)**2 + + def __mean_pairwise_distance(self, dists_GR): + return np.sqrt(dists_GR.mean()) + + def __median_pairwise_distance(self, dists_GR): + return np.sqrt(np.median(dists_GR)) + + def get_sigmas(self, dists_GR): + mult_factor = self.__get_sigma_mult_factor(dists_GR) + return self.base_sigmas * mult_factor + + @time_function + def calculate_MMD_rbf_quadratic(self, generated_dataset=None, reference_dataset=None): + # https://github.com/djsutherland/opt-mmd/blob/master/two_sample/mmd.py + + if not isinstance(generated_dataset, torch.Tensor) and not isinstance(generated_dataset, np.ndarray): + (generated_dataset, reference_dataset), _ = self.get_activations(generated_dataset, reference_dataset) + + GG = self.__get_pairwise_distances(generated_dataset, generated_dataset) + GR = self.__get_pairwise_distances(generated_dataset, reference_dataset) + RR = self.__get_pairwise_distances(reference_dataset, reference_dataset) + + max_mmd = 0 + sigmas = self.get_sigmas(GR) + + for sigma in sigmas: + gamma = 1 / (2 * sigma**2) + + K_GR = np.exp(-gamma * GR) + K_GG = np.exp(-gamma * GG) + K_RR = np.exp(-gamma * RR) + + mmd = K_GG.mean() + K_RR.mean() - 2 * K_GR.mean() + max_mmd = mmd if mmd > max_mmd else max_mmd + + return {self.name: max_mmd} + + @time_function + def calculate_MMD_linear_kernel(self, generated_dataset=None, reference_dataset=None): + # https://github.com/djsutherland/opt-mmd/blob/master/two_sample/mmd.py + if not isinstance(generated_dataset, torch.Tensor) and not isinstance(generated_dataset, np.ndarray): + (generated_dataset, reference_dataset), _ = self.get_activations(generated_dataset, reference_dataset) + + G_bar = generated_dataset.mean(axis=0) + R_bar = reference_dataset.mean(axis=0) + Z_bar = G_bar - R_bar + mmd = Z_bar.dot(Z_bar) + mmd = mmd if mmd >= 0 else 0 + return {'mmd_linear': mmd} + + +class prdcEvaluation(GINMetric): + # From PRDC github: https://github.com/clovaai/generative-evaluation-prdc/blob/master/prdc/prdc.py#L54 + def __init__(self, *args, use_pr=False, **kwargs): + super().__init__(*args, **kwargs) + self.use_pr = use_pr + + @time_function + def evaluate(self, generated_dataset=None, reference_dataset=None, nearest_k=5): + """ Computes precision, recall, density, and coverage given two manifolds. """ + + if not isinstance(generated_dataset, torch.Tensor) and not isinstance(generated_dataset, np.ndarray): + (generated_dataset, reference_dataset), _ = self.get_activations(generated_dataset, reference_dataset) + + real_nearest_neighbour_distances = self.__compute_nearest_neighbour_distances(reference_dataset, nearest_k) + distance_real_fake = self.__compute_pairwise_distance(reference_dataset, generated_dataset) + + if self.use_pr: + fake_nearest_neighbour_distances = self.__compute_nearest_neighbour_distances(generated_dataset, nearest_k) + precision = ( + distance_real_fake <= np.expand_dims(real_nearest_neighbour_distances, axis=1) + ).any(axis=0).mean() + + recall = ( + distance_real_fake <= np.expand_dims(fake_nearest_neighbour_distances, axis=0) + ).any(axis=1).mean() + + f1_pr = 2 / ((1 / (precision + 1e-8)) + (1 / (recall + 1e-8))) + result = dict(precision=precision, recall=recall, f1_pr=f1_pr) + else: + density = (1. / float(nearest_k)) * ( + distance_real_fake <= np.expand_dims(real_nearest_neighbour_distances, axis=1)).sum(axis=0).mean() + + coverage = (distance_real_fake.min(axis=1) <= real_nearest_neighbour_distances).mean() + + f1_dc = 2 / ((1 / (density + 1e-8)) + (1 / (coverage + 1e-8))) + result = dict(density=density, coverage=coverage, f1_dc=f1_dc) + return result + + def __compute_pairwise_distance(self, data_x, data_y=None): + """ + Args: + data_x: numpy.ndarray([N, feature_dim], dtype=np.float32) + data_y: numpy.ndarray([N, feature_dim], dtype=np.float32) + Return: + numpy.ndarray([N, N], dtype=np.float32) of pairwise distances. + """ + if data_y is None: + data_y = data_x + dists = sklearn.metrics.pairwise_distances(data_x, data_y, metric='euclidean', n_jobs=8) + return dists + + def __get_kth_value(self, unsorted, k, axis=-1): + """ + Args: + unsorted: numpy.ndarray of any dimensionality. + k: int + Return: + kth values along the designated axis. + """ + indices = np.argpartition(unsorted, k, axis=axis)[..., :k] + k_smallest = np.take_along_axis(unsorted, indices, axis=axis) + kth_values = k_smallest.max(axis=axis) + return kth_values + + def __compute_nearest_neighbour_distances(self, input_features, nearest_k): + """ + Args: + input_features: numpy.ndarray([N, feature_dim], dtype=np.float32) + nearest_k: int + Return: + Distances to kth nearest neighbours. + """ + distances = self.__compute_pairwise_distance(input_features) + radii = self.__get_kth_value(distances, k=nearest_k + 1, axis=-1) + return radii + + +def nn_based_eval(graph_ref_list, graph_pred_list, N_gin=10): + device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') + + evaluators = [] + for _ in range(N_gin): + gin = load_feature_extractor(device) + evaluators.append(MMDEvaluation(model=gin, kernel='rbf', sigma='range', multiplier='mean')) + evaluators.append(prdcEvaluation(model=gin, use_pr=True)) + evaluators.append(prdcEvaluation(model=gin, use_pr=False)) + + ref_graphs = [dgl.from_networkx(g).to(device) for g in graph_ref_list] + gen_graphs = [dgl.from_networkx(g).to(device) for g in graph_pred_list] + + metrics = { + 'mmd_rbf': [], + 'f1_pr': [], + 'f1_dc': [] + } + for evaluator in evaluators: + res, time = evaluator.evaluate(generated_dataset=gen_graphs, reference_dataset=ref_graphs) + for key in list(res.keys()): + if key in metrics: + metrics[key].append(res[key]) + + results = { + 'MMD_RBF': (np.mean(metrics['mmd_rbf']), np.std(metrics['mmd_rbf'])), + 'F1_PR': (np.mean(metrics['f1_pr']), np.std(metrics['f1_pr'])), + 'F1_DC': (np.mean(metrics['f1_dc']), np.std(metrics['f1_dc'])) + } + return results diff --git a/MobileNetV3/evaluation/structure_evaluator.py b/MobileNetV3/evaluation/structure_evaluator.py new file mode 100644 index 0000000..a037460 --- /dev/null +++ b/MobileNetV3/evaluation/structure_evaluator.py @@ -0,0 +1,209 @@ +"""MMD Evaluation on graph structure statistics. Modified from https://github.com/uoguelph-mlrg/GGM-metrics""" + +import numpy as np +import networkx as nx +import numpy as np +# from scipy.linalg import toeplitz +# import pyemd +import concurrent.futures +from scipy.linalg import eigvalsh +from functools import partial + + +class Descriptor(): + def __init__(self, is_parallel=False, bins=100, kernel='rbf', sigma_type='single', **kwargs): + self.is_parallel = is_parallel + self.bins = bins + self.max_workers = kwargs.get('max_workers') + + if kernel == 'rbf': + self.distance = self.l2 + self.name += '_rbf' + else: + ValueError + + if sigma_type == 'argmax': + log_sigmas = np.linspace(-5., 5., 50) + # the first 30 sigma values is usually enough + log_sigmas = log_sigmas[:30] + self.sigmas = [np.exp(log_sigma) for log_sigma in log_sigmas] + elif sigma_type == 'single': + self.sigmas = kwargs['sigma'] + else: + raise ValueError + + def evaluate(self, graph_ref_list, graph_pred_list): + """Compute the distance between the distributions of two unordered sets of graphs. + Args: + graph_ref_list, graph_pred_list: two lists of networkx graphs to be evaluated. + """ + + graph_pred_list = [G for G in graph_pred_list if not G.number_of_nodes() == 0] + + sample_pred = self.extract_features(graph_pred_list) + sample_ref = self.extract_features(graph_ref_list) + + GG = self.disc(sample_pred, sample_pred, distance_scaling=self.distance_scaling) + GR = self.disc(sample_pred, sample_ref, distance_scaling=self.distance_scaling) + RR = self.disc(sample_ref, sample_ref, distance_scaling=self.distance_scaling) + + sigmas = self.sigmas + max_mmd = 0 + mmd_dict = [] + for sigma in sigmas: + gamma = 1 / (2 * sigma ** 2) + + K_GR = np.exp(-gamma * GR) + K_GG = np.exp(-gamma * GG) + K_RR = np.exp(-gamma * RR) + + mmd = K_GG.mean() + K_RR.mean() - (2 * K_GR.mean()) + mmd_dict.append((sigma, mmd)) + max_mmd = mmd if mmd > max_mmd else max_mmd + + # print(self.name, mmd_dict) + + return max_mmd + + def pad_histogram(self, x, y): + # convert histogram values x and y to float, and pad them for equal length + support_size = max(len(x), len(y)) + x = x.astype(np.float) + y = y.astype(np.float) + if len(x) < len(y): + x = np.hstack((x, [0.] * (support_size - len(x)))) + elif len(y) < len(x): + y = np.hstack((y, [0.] * (support_size - len(y)))) + + return x, y + + # def emd(self, x, y, distance_scaling=1.0): + # support_size = max(len(x), len(y)) + # x, y = self.pad_histogram(x, y) + # + # d_mat = toeplitz(range(support_size)).astype(np.float) + # distance_mat = d_mat / distance_scaling + # + # dist = pyemd.emd(x, y, distance_mat) + # return dist ** 2 + + def l2(self, x, y, **kwargs): + # gaussian rbf + x, y = self.pad_histogram(x, y) + dist = np.linalg.norm(x - y, 2) + return dist ** 2 + + def kernel_parallel_unpacked(self, x, samples2, kernel): + dist = [] + for s2 in samples2: + dist += [kernel(x, s2)] + return dist + + def kernel_parallel_worker(self, t): + return self.kernel_parallel_unpacked(*t) + + def disc(self, samples1, samples2, **kwargs): + # Discrepancy between 2 samples + tot_dist = [] + if not self.is_parallel: + for s1 in samples1: + for s2 in samples2: + tot_dist += [self.distance(s1, s2)] + else: + with concurrent.futures.ProcessPoolExecutor(max_workers=self.max_workers) as executor: + for dist in executor.map(self.kernel_parallel_worker, + [(s1, samples2, partial(self.distance, **kwargs)) for s1 in samples1]): + tot_dist += [dist] + return np.array(tot_dist) + + +class degree(Descriptor): + def __init__(self, *args, **kwargs): + self.name = 'degree' + self.sigmas = [kwargs.get('sigma', 1.0)] + self.distance_scaling = 1.0 + super().__init__(*args, **kwargs) + + def extract_features(self, dataset): + res = [] + if self.is_parallel: + with concurrent.futures.ProcessPoolExecutor(max_workers=self.max_workers) as executor: + for deg_hist in executor.map(self.degree_worker, dataset): + res.append(deg_hist) + else: + for g in dataset: + degree_hist = self.degree_worker(g) + res.append(degree_hist) + + res = [s1 / np.sum(s1) for s1 in res] + return res + + def degree_worker(self, G): + return np.array(nx.degree_histogram(G)) + + +class cluster(Descriptor): + def __init__(self, *args, **kwargs): + self.name = 'cluster' + self.sigmas = [kwargs.get('sigma', [1.0 / 10])] + super().__init__(*args, **kwargs) + self.distance_scaling = self.bins + + def extract_features(self, dataset): + res = [] + if self.is_parallel: + with concurrent.futures.ProcessPoolExecutor(max_workers=self.max_workers) as executor: + for clustering_hist in executor.map(self.clustering_worker, [(G, self.bins) for G in dataset]): + res.append(clustering_hist) + else: + for g in dataset: + clustering_hist = self.clustering_worker((g, self.bins)) + res.append(clustering_hist) + + res = [s1 / np.sum(s1) for s1 in res] + return res + + def clustering_worker(self, param): + G, bins = param + clustering_coeffs_list = list(nx.clustering(G).values()) + hist, _ = np.histogram( + clustering_coeffs_list, bins=bins, range=(0.0, 1.0), density=False) + return hist + + +class spectral(Descriptor): + def __init__(self, *args, **kwargs): + self.name = 'spectral' + self.sigmas = [kwargs.get('sigma', 1.0)] + self.distance_scaling = 1 + super().__init__(*args, **kwargs) + + def extract_features(self, dataset): + res = [] + if self.is_parallel: + with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor: + for spectral_density in executor.map(self.spectral_worker, dataset): + res.append(spectral_density) + else: + for g in dataset: + spectral_temp = self.spectral_worker(g) + res.append(spectral_temp) + return res + + def spectral_worker(self, G): + eigs = eigvalsh(nx.normalized_laplacian_matrix(G).todense()) + spectral_pmf, _ = np.histogram(eigs, bins=200, range=(-1e-5, 2), density=False) + spectral_pmf = spectral_pmf / spectral_pmf.sum() + return spectral_pmf + + +def mmd_eval(graph_ref_list, graph_pred_list, methods): + evaluators = [] + for (method, sigma, sigma_type) in methods: + evaluators.append(eval(method)(sigma=sigma, sigma_type=sigma_type)) + + results = {} + for evaluator in evaluators: + results[evaluator.name] = evaluator.evaluate(graph_ref_list, graph_pred_list) + + return results diff --git a/MobileNetV3/logger.py b/MobileNetV3/logger.py new file mode 100644 index 0000000..427bdfa --- /dev/null +++ b/MobileNetV3/logger.py @@ -0,0 +1,180 @@ +import os +import wandb +import torch +import numpy as np + + +class Logger: + def __init__( + self, + exp_name, + log_dir=None, + exp_suffix="", + write_textfile=True, + use_wandb=False, + wandb_project_name=None, + entity='hysh', + config=None + ): + + self.log_dir = log_dir + self.write_textfile = write_textfile + self.use_wandb = use_wandb + + self.logs_for_save = {} + self.logs = {} + + if self.write_textfile: + self.f = open(os.path.join(log_dir, 'logs.txt'), 'w') + + if self.use_wandb: + exp_suffix = "_".join(exp_suffix.split("/")[:-1]) + wandb.init( + config=config if config is not None else wandb.config, + entity=entity, + project=wandb_project_name, + name=exp_name + "_" + exp_suffix, + group=exp_name, + reinit=True) + + def write_str(self, log_str): + self.f.write(log_str+'\n') + self.f.flush() + + def update_config(self, v, is_args=False): + if is_args: + self.logs_for_save.update({'args': v}) + else: + self.logs_for_save.update(v) + if self.use_wandb: + wandb.config.update(v, allow_val_change=True) + + def write_log_nohead(self, element, step): + log_str = f"{step} | " + log_dict = {} + for key, val in element.items(): + if not key in self.logs_for_save: + self.logs_for_save[key] = [] + self.logs_for_save[key].append(val) + log_str += f'{key} {val} | ' + log_dict[f'{key}'] = val + + if self.write_textfile: + self.f.write(log_str+'\n') + self.f.flush() + + if self.use_wandb: + wandb.log(log_dict, step=step) + + def write_log(self, element, step, return_log_dict=False): + log_str = f"{step} | " + log_dict = {} + for head, keys in element.items(): + for k in keys: + if k in self.logs: + v = self.logs[k].avg + if not k in self.logs_for_save: + self.logs_for_save[k] = [] + self.logs_for_save[k].append(v) + log_str += f'{k} {v}| ' + log_dict[f'{head}/{k}'] = v + + if self.write_textfile: + self.f.write(log_str+'\n') + self.f.flush() + + if return_log_dict: + return log_dict + + if self.use_wandb: + wandb.log(log_dict, step=step) + + def log_sample(self, sample_x): + wandb.log({"sampled_x": [wandb.Image(x.unsqueeze(-1).cpu().numpy()) for x in sample_x]}) + + def log_valid_sample_prop(self, arch_metric, x_axis, y_axis): + assert x_axis in ['test_acc', 'flops', 'params', 'latency'] + assert y_axis in ['test_acc', 'flops', 'params', 'latency'] + + data = [[x, y] for (x, y) in zip(arch_metric[2][f'{x_axis}_list'], arch_metric[2][f'{y_axis}_list'])] + table = wandb.Table(data=data, columns = [x_axis, y_axis]) + wandb.log({f"valid_sample ({x_axis}-{y_axis})" : wandb.plot.scatter(table, x_axis, y_axis)}) + + def save_log(self, name=None): + name = 'logs.pt' if name is None else name + torch.save(self.logs_for_save, os.path.join(self.log_dir, name)) + + def update(self, key, v, n=1): + if not key in self.logs: + self.logs[key] = AverageMeter() + self.logs[key].update(v, n) + + def reset(self, keys=None, except_keys=[]): + if keys is not None: + if isinstance(keys, list): + for key in keys: + self.logs[key] = AverageMeter() + else: + self.logs[keys] = AverageMeter() + else: + for key in self.logs.keys(): + if not key in except_keys: + self.logs[key] = AverageMeter() + + def avg(self, keys=None, except_keys=[]): + if keys is not None: + if isinstance(keys, list): + return {key: self.logs[key].avg for key in keys if key in self.logs.keys()} + else: + return self.logs[keys].avg + else: + avg_dict = {} + for key in self.logs.keys(): + if not key in except_keys: + avg_dict[key] = self.logs[key].avg + return avg_dict + + +class AverageMeter(object): + """ + Computes and stores the average and current value + Copied from: https://github.com/pytorch/examples/blob/master/imagenet/main.py + """ + + def __init__(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def get_metrics(g_embeds, x_embeds, logit_scale, prefix='train'): + metrics = {} + logits_per_g = (logit_scale * g_embeds @ x_embeds.t()).detach().cpu() + logits_per_x = logits_per_g.t().detach().cpu() + + logits = {"g_to_x": logits_per_g, "x_to_g": logits_per_x} + ground_truth = torch.arange(len(x_embeds)).view(-1, 1) + + for name, logit in logits.items(): + ranking = torch.argsort(logit, descending=True) + preds = torch.where(ranking == ground_truth)[1] + preds = preds.detach().cpu().numpy() + metrics[f"{prefix}_{name}_mean_rank"] = preds.mean() + 1 + metrics[f"{prefix}_{name}_median_rank"] = np.floor(np.median(preds)) + 1 + for k in [1, 5, 10]: + metrics[f"{prefix}_{name}_R@{k}"] = np.mean(preds < k) + + return metrics \ No newline at end of file diff --git a/MobileNetV3/losses.py b/MobileNetV3/losses.py new file mode 100644 index 0000000..1d912c6 --- /dev/null +++ b/MobileNetV3/losses.py @@ -0,0 +1,584 @@ +"""All functions related to loss computation and optimization.""" + +import torch +import torch.optim as optim +import numpy as np +from models import utils as mutils +from sde_lib import VPSDE, VESDE + + +def get_optimizer(config, params): + """Return a flax optimizer object based on `config`.""" + if config.optim.optimizer == 'Adam': + optimizer = optim.Adam(params, lr=config.optim.lr, betas=(config.optim.beta1, 0.999), eps=config.optim.eps, + weight_decay=config.optim.weight_decay) + else: + raise NotImplementedError( + f'Optimizer {config.optim.optimizer} not supported yet!' + ) + return optimizer + + +def optimization_manager(config): + """Return an optimize_fn based on `config`.""" + + def optimize_fn(optimizer, params, step, lr=config.optim.lr, + warmup=config.optim.warmup, + grad_clip=config.optim.grad_clip): + """Optimize with warmup and gradient clipping (disabled if negative).""" + if warmup > 0: + for g in optimizer.param_groups: + g['lr'] = lr * np.minimum(step / warmup, 1.0) + if grad_clip >= 0: + torch.nn.utils.clip_grad_norm_(params, max_norm=grad_clip) + optimizer.step() + + return optimize_fn + + +def get_sde_loss_fn_nas(sde, train, reduce_mean=True, continuous=True, likelihood_weighting=True, eps=1e-5): + """Create a loss function for training with arbitrary SDEs. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + train: `True` for training loss and `False` for evaluation loss. + reduce_mean: If `True`, average the loss across data dimensions. Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + Otherwise, it requires ad-hoc interpolation to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according + to https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended in Score SDE paper. + eps: A `float` number. The smallest time step to sample from. + + Returns: + A loss function. + """ + + # reduce_op = torch.mean if reduce_mean else lambda *args, **kwargs: 0.5 * torch.sum(*args, **kwargs) + + def loss_fn(model, batch): + """Compute the loss function. + + Args: + model: A score model. + batch: A mini-batch of training data, including adjacency matrices and mask. + + Returns: + loss: A scalar that represents the average loss value across the mini-batch. + """ + x, adj, mask = batch + # adj, mask: [32, 1, 20, 20] + score_fn = mutils.get_score_fn(sde, model, train=train, continuous=continuous) + t = torch.rand(x.shape[0], device=adj.device) * (sde.T - eps) + eps + + z = torch.randn_like(x) # [B, C, N, N] + # z = torch.tril(z, -1) + # z = z + z.transpose(2, 3) + + mean, std = sde.marginal_prob(x, t) + # mean = torch.tril(mean, -1) + # mean = mean + mean.transpose(2, 3) + + perturbed_data = mean + std[:, None, None] * z + score = score_fn(perturbed_data, t, mask) + + # mask = torch.tril(mask, -1) + # mask = mask + mask.transpose(2, 3) + # mask = mask.reshape(mask.shape[0], -1) # low triangular part of adj matrices + + if not likelihood_weighting: + losses = torch.square(score * std[:, None, None] + z) + losses = losses.reshape(losses.shape[0], -1) + if reduce_mean: + # losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + losses = torch.mean(losses, dim=-1) + else: + losses = 0.5 * torch.sum(losses, dim=-1) + loss = losses.mean() + else: + g2 = sde.sde(torch.zeros_like(x), t)[1] ** 2 + losses = torch.square(score + z / std[:, None, None]) + losses = losses.reshape(losses.shape[0], -1) + if reduce_mean: + # losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + losses = torch.mean(losses, dim=-1) + else: + losses = 0.5 * torch.sum(losses, dim=-1) + loss = (losses * g2).mean() + + return loss + + return loss_fn + + +def get_predictor_loss_fn_nas_binary(sde, train, reduce_mean=True, continuous=True, + likelihood_weighting=True, eps=1e-5, label_list=None, + noised=True, t_spot=None): + """Create a loss function for training with arbitrary SDEs. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + train: `True` for training loss and `False` for evaluation loss. + reduce_mean: If `True`, average the loss across data dimensions. Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + Otherwise, it requires ad-hoc interpolation to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according + to https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended in Score SDE paper. + eps: A `float` number. The smallest time step to sample from. + + Returns: + A loss function. + """ + + # reduce_op = torch.mean if reduce_mean else lambda *args, **kwargs: 0.5 * torch.sum(*args, **kwargs) + + def loss_fn(model, batch): + """Compute the loss function. + + Args: + model: A score model. + batch: A mini-batch of training data, including adjacency matrices and mask. + + Returns: + loss: A scalar that represents the average loss value across the mini-batch. + """ + x, adj, mask, extra = batch + # adj, mask: [32, 1, 20, 20] + # score_fn = mutils.get_score_fn(sde, model, train=train, continuous=continuous) + predictor_fn = mutils.get_predictor_fn(sde, model, train=train, continuous=continuous) + if noised: + if t_spot < 1: + t = torch.rand(x.shape[0], device=adj.device) * (t_spot - eps) + eps # torch.rand: [0, 1) + else: + t = torch.rand(x.shape[0], device=adj.device) * (sde.T - eps) + eps + + z = torch.randn_like(x) # [B, C, N, N] + # z = torch.tril(z, -1) + # z = z + z.transpose(2, 3) + + mean, std = sde.marginal_prob(x, t) + # mean = torch.tril(mean, -1) + # mean = mean + mean.transpose(2, 3) + + perturbed_data = mean + std[:, None, None] * z + # score = score_fn(perturbed_data, t, mask) + pred = predictor_fn(perturbed_data, t, mask) + else: + t = eps * torch.ones(x.shape[0], device=adj.device) + pred = predictor_fn(x, t, mask) + + labels = extra[f"{label_list}"][1] + labels = labels.to(pred.device).unsqueeze(1).type(pred.dtype) + # mask = torch.tril(mask, -1) + # mask = mask + mask.transpose(2, 3) + # mask = mask.reshape(mask.shape[0], -1) # low triangular part of adj matrices + # loss = torch.nn.MSELoss()(pred, labels) + loss = torch.nn.BCEWithLogitsLoss()(pred, labels) + + # if not likelihood_weighting: + # losses = torch.square(score * std[:, None, None] + z) + # losses = losses.reshape(losses.shape[0], -1) + # if reduce_mean: + # # losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + # losses = torch.mean(losses, dim=-1) + # else: + # losses = 0.5 * torch.sum(losses, dim=-1) + # loss = losses.mean() + # else: + # g2 = sde.sde(torch.zeros_like(x), t)[1] ** 2 + # losses = torch.square(score + z / std[:, None, None]) + # losses = losses.reshape(losses.shape[0], -1) + # if reduce_mean: + # # losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + # losses = torch.mean(losses, dim=-1) + # else: + # losses = 0.5 * torch.sum(losses, dim=-1) + # loss = (losses * g2).mean() + + return loss, pred, labels + + return loss_fn + + + +def get_predictor_loss_fn_nas(sde, train, reduce_mean=True, continuous=True, + likelihood_weighting=True, eps=1e-5, label_list=None, + noised=True, t_spot=None): + """Create a loss function for training with arbitrary SDEs. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + train: `True` for training loss and `False` for evaluation loss. + reduce_mean: If `True`, average the loss across data dimensions. Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + Otherwise, it requires ad-hoc interpolation to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according + to https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended in Score SDE paper. + eps: A `float` number. The smallest time step to sample from. + + Returns: + A loss function. + """ + + # reduce_op = torch.mean if reduce_mean else lambda *args, **kwargs: 0.5 * torch.sum(*args, **kwargs) + + def loss_fn(model, batch): + """Compute the loss function. + + Args: + model: A score model. + batch: A mini-batch of training data, including adjacency matrices and mask. + + Returns: + loss: A scalar that represents the average loss value across the mini-batch. + """ + x, adj, mask, extra = batch + # adj, mask: [32, 1, 20, 20] + # score_fn = mutils.get_score_fn(sde, model, train=train, continuous=continuous) + predictor_fn = mutils.get_predictor_fn(sde, model, train=train, continuous=continuous) + if noised: + if t_spot < 1: + t = torch.rand(x.shape[0], device=adj.device) * (t_spot - eps) + eps # torch.rand: [0, 1) + else: + t = torch.rand(x.shape[0], device=adj.device) * (sde.T - eps) + eps + + z = torch.randn_like(x) # [B, C, N, N] + # z = torch.tril(z, -1) + # z = z + z.transpose(2, 3) + + mean, std = sde.marginal_prob(x, t) + # mean = torch.tril(mean, -1) + # mean = mean + mean.transpose(2, 3) + + perturbed_data = mean + std[:, None, None] * z + # score = score_fn(perturbed_data, t, mask) + pred = predictor_fn(perturbed_data, t, mask) + else: + t = eps * torch.ones(x.shape[0], device=adj.device) + pred = predictor_fn(x, t, mask) + + labels = extra[f"{label_list[-1]}"] + labels = labels.to(pred.device).unsqueeze(1).type(pred.dtype) + # mask = torch.tril(mask, -1) + # mask = mask + mask.transpose(2, 3) + # mask = mask.reshape(mask.shape[0], -1) # low triangular part of adj matrices + loss = torch.nn.MSELoss()(pred, labels) + + # if not likelihood_weighting: + # losses = torch.square(score * std[:, None, None] + z) + # losses = losses.reshape(losses.shape[0], -1) + # if reduce_mean: + # # losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + # losses = torch.mean(losses, dim=-1) + # else: + # losses = 0.5 * torch.sum(losses, dim=-1) + # loss = losses.mean() + # else: + # g2 = sde.sde(torch.zeros_like(x), t)[1] ** 2 + # losses = torch.square(score + z / std[:, None, None]) + # losses = losses.reshape(losses.shape[0], -1) + # if reduce_mean: + # # losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + # losses = torch.mean(losses, dim=-1) + # else: + # losses = 0.5 * torch.sum(losses, dim=-1) + # loss = (losses * g2).mean() + + return loss, pred, labels + + return loss_fn + + +def get_meta_predictor_loss_fn_nas(sde, train, reduce_mean=True, continuous=True, + likelihood_weighting=True, eps=1e-5, label_list=None, + noised=True, t_spot=None): + """Create a loss function for training with arbitrary SDEs. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + train: `True` for training loss and `False` for evaluation loss. + reduce_mean: If `True`, average the loss across data dimensions. Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + Otherwise, it requires ad-hoc interpolation to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according + to https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended in Score SDE paper. + eps: A `float` number. The smallest time step to sample from. + + Returns: + A loss function. + """ + + # reduce_op = torch.mean if reduce_mean else lambda *args, **kwargs: 0.5 * torch.sum(*args, **kwargs) + + def loss_fn(model, batch): + """Compute the loss function. + + Args: + model: A score model. + batch: A mini-batch of training data, including adjacency matrices and mask. + + Returns: + loss: A scalar that represents the average loss value across the mini-batch. + """ + x, adj, mask, extra, task = batch + predictor_fn = mutils.get_predictor_fn(sde, model, train=train, continuous=continuous) + if noised: + if t_spot < 1: + t = torch.rand(x.shape[0], device=adj.device) * (t_spot - eps) + eps # torch.rand: [0, 1) + else: + t = torch.rand(x.shape[0], device=adj.device) * (sde.T - eps) + eps + + z = torch.randn_like(x) # [B, C, N, N] + + mean, std = sde.marginal_prob(x, t) + + perturbed_data = mean + std[:, None, None] * z + # score = score_fn(perturbed_data, t, mask) + pred = predictor_fn(perturbed_data, t, mask, task) + else: + t = eps * torch.ones(x.shape[0], device=adj.device) + pred = predictor_fn(x, t, mask, task) + labels = extra[f"{label_list[-1]}"] + labels = labels.to(pred.device).unsqueeze(1).type(pred.dtype) + + loss = torch.nn.MSELoss()(pred, labels) + + return loss, pred, labels + + return loss_fn + + +def get_sde_loss_fn(sde, train, reduce_mean=True, continuous=True, likelihood_weighting=True, eps=1e-5): + """Create a loss function for training with arbitrary SDEs. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + train: `True` for training loss and `False` for evaluation loss. + reduce_mean: If `True`, average the loss across data dimensions. Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + Otherwise, it requires ad-hoc interpolation to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according + to https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended in Score SDE paper. + eps: A `float` number. The smallest time step to sample from. + + Returns: + A loss function. + """ + + # reduce_op = torch.mean if reduce_mean else lambda *args, **kwargs: 0.5 * torch.sum(*args, **kwargs) + + def loss_fn(model, batch): + """Compute the loss function. + + Args: + model: A score model. + batch: A mini-batch of training data, including adjacency matrices and mask. + + Returns: + loss: A scalar that represents the average loss value across the mini-batch. + """ + adj, mask = batch + # adj, mask: [32, 1, 20, 20] + score_fn = mutils.get_score_fn(sde, model, train=train, continuous=continuous) + t = torch.rand(adj.shape[0], device=adj.device) * (sde.T - eps) + eps + + z = torch.randn_like(adj) # [B, C, N, N] + z = torch.tril(z, -1) + z = z + z.transpose(2, 3) + + mean, std = sde.marginal_prob(adj, t) + mean = torch.tril(mean, -1) + mean = mean + mean.transpose(2, 3) + + perturbed_data = mean + std[:, None, None, None] * z + score = score_fn(perturbed_data, t, mask=mask) + + mask = torch.tril(mask, -1) + mask = mask + mask.transpose(2, 3) + mask = mask.reshape(mask.shape[0], -1) # low triangular part of adj matrices + + if not likelihood_weighting: + losses = torch.square(score * std[:, None, None, None] + z) + losses = losses.reshape(losses.shape[0], -1) + if reduce_mean: + losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + else: + losses = 0.5 * torch.sum(losses * mask, dim=-1) + loss = losses.mean() + else: + g2 = sde.sde(torch.zeros_like(adj), t)[1] ** 2 + losses = torch.square(score + z / std[:, None, None, None]) + losses = losses.reshape(losses.shape[0], -1) + if reduce_mean: + losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + else: + losses = 0.5 * torch.sum(losses * mask, dim=-1) + loss = (losses * g2).mean() + + return loss + + return loss_fn + + +def get_step_fn(sde, train, optimize_fn=None, reduce_mean=False, continuous=True, + likelihood_weighting=False, data='NASBench201'): + """Create a one-step training/evaluation function. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + Tuple (`sde_lib.SDE`, `sde_lib.SDE`) that represents the forward node SDE and edge SDE. + optimize_fn: An optimization function. + reduce_mean: If `True`, average the loss across data dimensions. + Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according to + https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended by score-sde. + + Returns: + A one-step function for training or evaluation. + """ + + if continuous: + if isinstance(sde, tuple): + loss_fn = get_multi_sde_loss_fn(sde[0], sde[1], train, reduce_mean=reduce_mean, continuous=True, + likelihood_weighting=likelihood_weighting) + else: + if data in ['NASBench201', 'ofa']: + loss_fn = get_sde_loss_fn_nas(sde, train, reduce_mean=reduce_mean, + continuous=True, likelihood_weighting=likelihood_weighting) + else: + loss_fn = get_sde_loss_fn(sde, train, reduce_mean=reduce_mean, + continuous=True, likelihood_weighting=likelihood_weighting) + else: + assert not likelihood_weighting, "Likelihood weighting is not supported for original SMLD/DDPM training." + if isinstance(sde, VESDE): + loss_fn = get_smld_loss_fn(sde, train, reduce_mean=reduce_mean) + elif isinstance(sde, VPSDE): + loss_fn = get_ddpm_loss_fn(sde, train, reduce_mean=reduce_mean) + elif isinstance(sde, tuple): + raise ValueError("Discrete training for multi sde is not recommended.") + else: + raise ValueError(f"Discrete training for {sde.__class__.__name__} is not recommended.") + + def step_fn(state, batch): + """Running one step of training or evaluation. + + For jax version: This function will undergo `jax.lax.scan` so that multiple steps can be pmapped and + jit-compiled together for faster execution. + + Args: + state: A dictionary of training information, containing the score model, optimizer, + EMA status, and number of optimization steps. + batch: A mini-batch of training/evaluation data, including min-batch adjacency matrices and mask. + + Returns: + loss: The average loss value of this state. + """ + model = state['model'] + if train: + optimizer = state['optimizer'] + optimizer.zero_grad() + loss = loss_fn(model, batch) + loss.backward() + optimize_fn(optimizer, model.parameters(), step=state['step']) + state['step'] += 1 + state['ema'].update(model.parameters()) + else: + with torch.no_grad(): + ema = state['ema'] + ema.store(model.parameters()) + ema.copy_to(model.parameters()) + loss = loss_fn(model, batch) + ema.restore(model.parameters()) + + return loss + + return step_fn + + +def get_step_fn_predictor(sde, train, optimize_fn=None, reduce_mean=False, continuous=True, + likelihood_weighting=False, data='NASBench201', label_list=None, noised=True, + t_spot=None, is_meta=False, is_binary=False): + """Create a one-step training/evaluation function. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + Tuple (`sde_lib.SDE`, `sde_lib.SDE`) that represents the forward node SDE and edge SDE. + optimize_fn: An optimization function. + reduce_mean: If `True`, average the loss across data dimensions. + Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according to + https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended by score-sde. + + Returns: + A one-step function for training or evaluation. + """ + + if continuous: + if isinstance(sde, tuple): + loss_fn = get_multi_sde_loss_fn(sde[0], sde[1], train, reduce_mean=reduce_mean, continuous=True, + likelihood_weighting=likelihood_weighting) + else: + if data in ['NASBench201', 'ofa']: + if is_meta: + loss_fn = get_meta_predictor_loss_fn_nas(sde, train, reduce_mean=reduce_mean, + continuous=True, likelihood_weighting=likelihood_weighting, + label_list=label_list, noised=noised, t_spot=t_spot) + elif is_binary: + loss_fn = get_predictor_loss_fn_nas_binary(sde, train, reduce_mean=reduce_mean, + continuous=True, likelihood_weighting=likelihood_weighting, + label_list=label_list, noised=noised, t_spot=t_spot) + else: + loss_fn = get_predictor_loss_fn_nas(sde, train, reduce_mean=reduce_mean, + continuous=True, likelihood_weighting=likelihood_weighting, + label_list=label_list, noised=noised, t_spot=t_spot) + else: + loss_fn = get_sde_loss_fn(sde, train, reduce_mean=reduce_mean, + continuous=True, likelihood_weighting=likelihood_weighting) + else: + assert not likelihood_weighting, "Likelihood weighting is not supported for original SMLD/DDPM training." + if isinstance(sde, VESDE): + loss_fn = get_smld_loss_fn(sde, train, reduce_mean=reduce_mean) + elif isinstance(sde, VPSDE): + loss_fn = get_ddpm_loss_fn(sde, train, reduce_mean=reduce_mean) + elif isinstance(sde, tuple): + raise ValueError("Discrete training for multi sde is not recommended.") + else: + raise ValueError(f"Discrete training for {sde.__class__.__name__} is not recommended.") + + def step_fn(state, batch): + """Running one step of training or evaluation. + + For jax version: This function will undergo `jax.lax.scan` so that multiple steps can be pmapped and + jit-compiled together for faster execution. + + Args: + state: A dictionary of training information, containing the score model, optimizer, + EMA status, and number of optimization steps. + batch: A mini-batch of training/evaluation data, including min-batch adjacency matrices and mask. + + Returns: + loss: The average loss value of this state. + """ + model = state['model'] + if train: + model.train() + optimizer = state['optimizer'] + optimizer.zero_grad() + loss, pred, labels = loss_fn(model, batch) + loss.backward() + optimize_fn(optimizer, model.parameters(), step=state['step']) + state['step'] += 1 + # state['ema'].update(model.parameters()) + else: + model.eval() + with torch.no_grad(): + # ema = state['ema'] + # ema.store(model.parameters()) + # ema.copy_to(model.parameters()) + loss, pred, labels = loss_fn(model, batch) + # ema.restore(model.parameters()) + + return loss, pred, labels + + return step_fn \ No newline at end of file diff --git a/MobileNetV3/main.py b/MobileNetV3/main.py new file mode 100644 index 0000000..3148962 --- /dev/null +++ b/MobileNetV3/main.py @@ -0,0 +1,40 @@ +"""Training and evaluation""" + +import run_lib +from absl import app, flags +from ml_collections.config_flags import config_flags +import logging +import os + +FLAGS = flags.FLAGS + +config_flags.DEFINE_config_file( + 'config', None, 'Training configuration.', lock_config=True +) +config_flags.DEFINE_config_file( + 'classifier_config_nf', None, 'Training configuration.', lock_config=True +) +flags.DEFINE_string('workdir', None, 'Work directory.') +flags.DEFINE_enum('mode', None, ['train', 'eval'], + 'Running mode: train or eval') +flags.DEFINE_string('eval_folder', 'eval', 'The folder name for storing evaluation results') +flags.mark_flags_as_required(['config', 'mode']) + + +def main(argv): + # Set random seed + run_lib.set_random_seed(FLAGS.config) + + if FLAGS.mode == 'train': + logger = logging.getLogger() + logger.setLevel('INFO') + # Run the training pipeline + run_lib.train(FLAGS.config) + elif FLAGS.mode == 'eval': + run_lib.evaluate(FLAGS.config) + else: + raise ValueError(f"Mode {FLAGS.mode} not recognized.") + + +if __name__ == '__main__': + app.run(main) diff --git a/MobileNetV3/main_exp/diffusion/run_lib.py b/MobileNetV3/main_exp/diffusion/run_lib.py new file mode 100644 index 0000000..7dde260 --- /dev/null +++ b/MobileNetV3/main_exp/diffusion/run_lib.py @@ -0,0 +1,329 @@ +import torch +import numpy as np +import sys +from scipy.stats import pearsonr, spearmanr +from torch.utils.data import DataLoader +sys.path.append('.') +import sampling + +import datasets_nas +from models import pgsn +from models import digcn +from models import cate +from models import dagformer +from models import digcn +from models import digcn_meta +from models import regressor +from models.GDSS import scorenetx +from models import utils as mutils +from models.ema import ExponentialMovingAverage +import sde_lib +from utils import * +import losses + +from analysis.arch_functions import BasicArchMetricsOFA +import losses +from analysis.arch_functions import NUM_STAGE, MAX_LAYER_PER_STAGE +from all_path import * + + +def get_sampling_fn(config, p=1, prod_w=False, weight_ratio_abs=False): + # Setup SDEs + if config.training.sde.lower() == 'vpsde': + sde = sde_lib.VPSDE( + beta_min=config.model.beta_min, + beta_max=config.model.beta_max, + N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'subvpsde': + sde = sde_lib.subVPSDE( + beta_min=config.model.beta_min, + beta_max=config.model.beta_max, + N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'vesde': + sde = sde_lib.VESDE( + sigma_min=config.model.sigma_min, + sigma_max=config.model.sigma_max, + N=config.model.num_scales) + sampling_eps = 1e-5 + else: + raise NotImplementedError(f"SDE {config.training.sde} unknown.") + + # create data normalizer and its inverse + inverse_scaler = datasets_nas.get_data_inverse_scaler(config) + + sampling_shape = ( + config.eval.batch_size, config.data.max_node, config.data.n_vocab) # ofa: 1024, 20, 28 + sampling_fn = sampling.get_sampling_fn( + config, sde, sampling_shape, inverse_scaler, + sampling_eps, config.data.name, conditional=True, + p=p, prod_w=prod_w, weight_ratio_abs=weight_ratio_abs) + + return sampling_fn, sde + + +def get_sampling_fn_meta(config, p=1, prod_w=False, weight_ratio_abs=False, init=False, n_init=5): + # Setup SDEs + if config.training.sde.lower() == 'vpsde': + sde = sde_lib.VPSDE( + beta_min=config.model.beta_min, + beta_max=config.model.beta_max, + N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'subvpsde': + sde = sde_lib.subVPSDE( + beta_min=config.model.beta_min, + beta_max=config.model.beta_max, + N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'vesde': + sde = sde_lib.VESDE( + sigma_min=config.model.sigma_min, + sigma_max=config.model.sigma_max, + N=config.model.num_scales) + sampling_eps = 1e-5 + else: + raise NotImplementedError(f"SDE {config.training.sde} unknown.") + + # create data normalizer and its inverse + inverse_scaler = datasets_nas.get_data_inverse_scaler(config) + + if init: + sampling_shape = ( + n_init, config.data.max_node, config.data.n_vocab) + else: + sampling_shape = ( + config.eval.batch_size, config.data.max_node, config.data.n_vocab) # ofa: 1024, 20, 28 + sampling_fn = sampling.get_sampling_fn( + config, sde, sampling_shape, inverse_scaler, + sampling_eps, config.data.name, conditional=True, + is_meta=True, data_name=config.sampling.check_dataname, + num_sample=config.model.num_sample) + + return sampling_fn, sde + + +def get_score_model(config, pos_enc_type=2): + # Build sampling functions and Load pre-trained score network & predictor network + score_config = torch.load(config.scorenet_ckpt_path)['config'] + ckpt_path = config.scorenet_ckpt_path + score_config.sampling.corrector = 'langevin' + score_config.model.pos_enc_type = pos_enc_type + + score_model = mutils.create_model(score_config) + score_ema = ExponentialMovingAverage( + score_model.parameters(), decay=score_config.model.ema_rate) + score_state = dict( + model=score_model, ema=score_ema, step=0, config=score_config) + score_state = restore_checkpoint( + ckpt_path, score_state, + device=config.device, resume=True) + score_ema.copy_to(score_model.parameters()) + return score_model, score_ema, score_config + + +def get_predictor(config): + classifier_model = mutils.create_model(config) + + return classifier_model + + +def get_adj(data_name, except_inout): + if data_name == 'NASBench201': + _adj = np.asarray( + [[0, 1, 1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 1, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 0]] + ) + _adj = torch.tensor(_adj, dtype=torch.float32, device=torch.device('cpu')) + if except_inout: + _adj = _adj[1:-1, 1:-1] + elif data_name == 'ofa': + assert except_inout + num_nodes = NUM_STAGE * MAX_LAYER_PER_STAGE + _adj = torch.zeros(num_nodes, num_nodes) + for i in range(num_nodes-1): + _adj[i, i+1] = 1 + return _adj + return _adj + +def generate_archs( + config, sampling_fn, score_model, score_ema, classifier_model, + num_samples, patient_factor, batch_size=512, classifier_scale=None, + task=None): + + metrics = BasicArchMetricsOFA() + # algo = 'none' + adj_s = get_adj(config.data.name, config.data.except_inout) + mask_s = aug_mask(adj_s, algo=config.data.aug_mask_algo)[0] + adj_c = get_adj(config.data.name, config.data.except_inout) + mask_c = aug_mask(adj_c, algo=config.data.aug_mask_algo)[0] + assert (adj_s == adj_c).all() and (mask_s == mask_c).all() + adj_s, mask_s, adj_c, mask_c = \ + adj_s.to(config.device), mask_s.to(config.device), adj_c.to(config.device), mask_c.to(config.device) + + # Generate and save samples + score_ema.copy_to(score_model.parameters()) + if num_samples > batch_size: + num_sampling_rounds = int(np.ceil(num_samples / batch_size) * patient_factor) + else: + num_sampling_rounds = int(patient_factor) + print(f'==> Sampling for {num_sampling_rounds} rounds...') + + r = 0 + all_samples = [] + classifier_scales = list(range(100000, 0, -int(classifier_scale))) + + while True and r < num_sampling_rounds: + classifier_scale = classifier_scales[r] + print(f'==> round {r} classifier_scale {classifier_scale}') + sample, _, sample_chain, (score_grad_norm_p, classifier_grad_norm_p, score_grad_norm_c, classifier_grad_norm_c) \ + = sampling_fn(score_model, mask_s, classifier_model, + eval_chain=True, + number_chain_steps=config.sampling.number_chain_steps, + classifier_scale=classifier_scale, + task=task, sample_bs=num_samples) + try: + sample_list = quantize(sample, adj_s) # quantization + _, validity, valid_arch_str, _, _ = metrics.compute_validity(sample_list, adj_s, mask_s) + except: + import pdb; pdb.set_trace() + validity = 0. + valid_arch_str = [] + print(f' ==> [Validity]: {round(validity, 4)}') + + if len(valid_arch_str) > 0: + all_samples += valid_arch_str + print(f' ==> [# Unique Arch]: {len(set(all_samples))}') + + if (len(set(all_samples)) >= num_samples): + break + + r += 1 + + return list(set(all_samples))[:num_samples] + + +def noise_aware_meta_predictor_fit(config, + predictor_model=None, + xtrain=None, + seed=None, + sde=None, + batch_size=5, + epochs=50, + save_best_p_corr=False, + save_path=None,): + assert save_best_p_corr + reset_seed(seed) + + data_loader = DataLoader(xtrain, + batch_size=batch_size, + shuffle=True, + drop_last=True) + + # create data normalizer and its inverse + scaler = datasets_nas.get_data_scaler(config) + + # Initialize model. + optimizer = losses.get_optimizer(config, predictor_model.parameters()) + state = dict(optimizer=optimizer, + model=predictor_model, + step=0, + config=config) + + # Build one-step training and evaluation functions + optimize_fn = losses.optimization_manager(config) + continuous = config.training.continuous + reduce_mean = config.training.reduce_mean + likelihood_weighting = config.training.likelihood_weighting + train_step_fn = losses.get_step_fn_predictor(sde, train=True, optimize_fn=optimize_fn, + reduce_mean=reduce_mean, continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name, label_list=config.data.label_list, + noised=config.training.noised, + t_spot=config.training.t_spot, + is_meta=True) + + # temp + # epochs = len(xtrain) * 100 + is_best = False + best_p_corr = -1 + ckpt_dir = os.path.join(save_path, 'loop') + print(f'==> Training for {epochs} epochs') + for epoch in range(epochs): + pred_list, labels_list = list(), list() + for step, batch in enumerate(data_loader): + x = batch['x'].to(config.device) # (5, 5, 20, 9)??? + adj = get_adj(config.data.name, config.data.except_inout) + task = batch['task'] + extra = batch + mask = aug_mask(adj, + algo=config.data.aug_mask_algo, + data=config.data.name) + x = scaler(x.to(config.device)) + adj = adj.to(config.device) + mask = mask.to(config.device) + task = task.to(config.device) + batch = (x, adj, mask, extra, task) + # Execute one training step + loss, pred, labels = train_step_fn(state, batch) + pred_list += [v.detach().item() for v in pred.squeeze()] + labels_list += [v.detach().item() for v in labels.squeeze()] + p_corr = pearsonr(np.array(pred_list), np.array(labels_list))[0] + s_corr = spearmanr(np.array(pred_list), np.array(labels_list))[0] + if epoch % 50 == 0: print(f'==> [Epoch-{epoch}] P corr: {round(p_corr, 4)} | S corr: {round(s_corr, 4)}') + + if save_best_p_corr: + if p_corr > best_p_corr: + is_best = True + best_p_corr = p_corr + os.makedirs(ckpt_dir, exist_ok=True) + save_checkpoint(ckpt_dir, state, epoch, is_best) + if save_best_p_corr: + loaded_state = torch.load(os.path.join(ckpt_dir, 'model_best.pth.tar'), map_location=config.device) + predictor_model.load_state_dict(loaded_state['model']) + + +def save_checkpoint(ckpt_dir, state, epoch, is_best): + saved_state = {} + for k in state: + if k in ['optimizer', 'model', 'ema']: + saved_state.update({k: state[k].state_dict()}) + else: + saved_state.update({k: state[k]}) + os.makedirs(ckpt_dir, exist_ok=True) + torch.save(saved_state, os.path.join(ckpt_dir, f'checkpoint_{epoch}.pth.tar')) + if is_best: + shutil.copy(os.path.join(ckpt_dir, f'checkpoint_{epoch}.pth.tar'), os.path.join(ckpt_dir, 'model_best.pth.tar')) + # remove the ckpt except is_best state + for ckpt_file in sorted(os.listdir(ckpt_dir)): + if not ckpt_file.startswith('checkpoint'): + continue + if os.path.join(ckpt_dir, ckpt_file) != os.path.join(ckpt_dir, 'model_best.pth.tar'): + os.remove(os.path.join(ckpt_dir, ckpt_file)) + + +def restore_checkpoint(ckpt_dir, state, device, resume=False): + if not resume: + os.makedirs(os.path.dirname(ckpt_dir), exist_ok=True) + return state + elif not os.path.exists(ckpt_dir): + if not os.path.exists(os.path.dirname(ckpt_dir)): + os.makedirs(os.path.dirname(ckpt_dir)) + logging.warning(f"No checkpoint found at {ckpt_dir}. " + f"Returned the same state as input") + return state + else: + loaded_state = torch.load(ckpt_dir, map_location=device) + for k in state: + if k in ['optimizer', 'model', 'ema']: + state[k].load_state_dict(loaded_state[k]) + else: + state[k] = loaded_state[k] + return state diff --git a/MobileNetV3/main_exp/get_files/get_aircraft.py b/MobileNetV3/main_exp/get_files/get_aircraft.py new file mode 100644 index 0000000..9d02170 --- /dev/null +++ b/MobileNetV3/main_exp/get_files/get_aircraft.py @@ -0,0 +1,63 @@ +""" +@author: Hayeon Lee +2020/02/19 +Script for downloading, and reorganizing aircraft +for few shot classification +Run this file as follows: + python get_data.py +""" + +import pickle +import os +import numpy as np +from tqdm import tqdm +import requests +import tarfile +from PIL import Image +import glob +import shutil +import pickle +import collections +import sys +sys.path.append(os.path.join(os.getcwd(), 'main_exp')) +from all_path import RAW_DATA_PATH + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +dir_path = RAW_DATA_PATH +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, 'fgvc-aircraft-2013b.tar.gz') + +if not os.path.exists(file_name): + print(f"Downloading {file_name}\n") + download_file( + 'http://www.robots.ox.ac.uk/~vgg/data/fgvc-aircraft/archives/fgvc-aircraft-2013b.tar.gz', + file_name) + print("\nDownloading done.\n") +else: + print("fgvc-aircraft-2013b.tar.gz has already been downloaded. Did not download twice.\n") + +untar_file_name = os.path.join(dir_path, 'aircraft') +if not os.path.exists(untar_file_name): + tarname = file_name + print("Untarring: {}".format(tarname)) + tar = tarfile.open(tarname) + tar.extractall(untar_file_name) + tar.close() +else: + print(f"{untar_file_name} folder already exists. Did not untarring twice\n") +os.remove(file_name) diff --git a/MobileNetV3/main_exp/get_files/get_pets.py b/MobileNetV3/main_exp/get_files/get_pets.py new file mode 100644 index 0000000..1a43e7d --- /dev/null +++ b/MobileNetV3/main_exp/get_files/get_pets.py @@ -0,0 +1,50 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile +import sys +sys.path.append(os.path.join(os.getcwd(), 'main_exp')) +from all_path import RAW_DATA_PATH + + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm(unit="B", total=int(r.headers['Content-Length'])) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update(len(chunk)) + f.write(chunk) + return filename + + +dir_path = os.path.join(RAW_DATA_PATH, 'pets') +if not os.path.exists(dir_path): + os.makedirs(dir_path) + +full_name = os.path.join(dir_path, 'test15.pth') +if not os.path.exists(full_name): + print(f"Downloading {full_name}\n") + download_file( + 'https://www.dropbox.com/s/kzmrwyyk5iaugv0/test15.pth?dl=1', full_name) + print("Downloading done.\n") +else: + print(f"{full_name} has already been downloaded. Did not download twice.\n") + +full_name = os.path.join(dir_path, 'train85.pth') +if not os.path.exists(full_name): + print(f"Downloading {full_name}\n") + download_file( + 'https://www.dropbox.com/s/w7mikpztkamnw9s/train85.pth?dl=1', full_name) + print("Downloading done.\n") +else: + print(f"{full_name} has already been downloaded. Did not download twice.\n") diff --git a/MobileNetV3/main_exp/get_files/get_preprocessed_data.py b/MobileNetV3/main_exp/get_files/get_preprocessed_data.py new file mode 100644 index 0000000..60a95ce --- /dev/null +++ b/MobileNetV3/main_exp/get_files/get_preprocessed_data.py @@ -0,0 +1,46 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +from all_path import PROCESSED_DATA_PATH + +dir_path = PROCESSED_DATA_PATH +if not os.path.exists(dir_path): + os.makedirs(dir_path) + + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + + +def get_preprocessed_data(file_name, url): + print(f"Downloading {file_name} datasets\n") + full_name = os.path.join(dir_path, file_name) + download_file(url, full_name) + print("Downloading done.\n") + + +for file_name, url in [ + ('aircraftbylabel.pt', 'https://www.dropbox.com/s/nn6mlrk1jijg108/aircraft100bylabel.pt?dl=1'), + ('cifar100bylabel.pt', 'https://www.dropbox.com/s/nn6mlrk1jijg108/aircraft100bylabel.pt?dl=1'), + ('cifar10bylabel.pt', 'https://www.dropbox.com/s/wt1pcwi991xyhwr/cifar10bylabel.pt?dl=1'), + ('imgnet32bylabel.pt', 'https://www.dropbox.com/s/7r3hpugql8qgi9d/imgnet32bylabel.pt?dl=1'), + ('petsbylabel.pt', 'https://www.dropbox.com/s/mxh6qz3grhy7wcn/petsbylabel.pt?dl=1'), + ]: + + get_preprocessed_data(file_name, url) diff --git a/MobileNetV3/main_exp/get_files/get_preprocessed_score_model_data.py b/MobileNetV3/main_exp/get_files/get_preprocessed_score_model_data.py new file mode 100644 index 0000000..543cb80 --- /dev/null +++ b/MobileNetV3/main_exp/get_files/get_preprocessed_score_model_data.py @@ -0,0 +1,44 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests + + +DATA_PATH = "./data/ofa/data_score_model" +dir_path = DATA_PATH +if not os.path.exists(dir_path): + os.makedirs(dir_path) + + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + + +def get_preprocessed_data(file_name, url): + print(f"Downloading {file_name} datasets\n") + full_name = os.path.join(dir_path, file_name) + download_file(url, full_name) + print("Downloading done.\n") + + +for file_name, url in [ + ('ofa_database_500000.pt', 'https://www.dropbox.com/scl/fi/0asz5qnvakf6ggucuynkk/ofa_database_500000.pt?rlkey=lqa1y4d6mikgzznevtanl2ybx&dl=1'), + ('ridx-500000.pt', 'https://www.dropbox.com/scl/fi/ambrm9n5efdkyydmsli0h/ridx-500000.pt?rlkey=b6iliyuiaxya4ropms8chsa7c&dl=1'), + ]: + + get_preprocessed_data(file_name, url) diff --git a/MobileNetV3/main_exp/nag.py b/MobileNetV3/main_exp/nag.py new file mode 100644 index 0000000..d1c1946 --- /dev/null +++ b/MobileNetV3/main_exp/nag.py @@ -0,0 +1,390 @@ +from __future__ import print_function +import torch +import os +import gc +import sys +from tqdm import tqdm +import numpy as np +import time +import os + +from torch import optim +from torch.optim.lr_scheduler import ReduceLROnPlateau +from scipy.stats import pearsonr + +from transfer_nag_lib.MetaD2A_mobilenetV3.metad2a_utils import load_graph_config, decode_ofa_mbv3_str_to_igraph +from transfer_nag_lib.MetaD2A_mobilenetV3.metad2a_utils import get_log +from transfer_nag_lib.MetaD2A_mobilenetV3.metad2a_utils import save_model, mean_confidence_interval + +from transfer_nag_lib.MetaD2A_mobilenetV3.loader import get_meta_train_loader, MetaTestDataset + +from transfer_nag_lib.encoder_FSBO_ofa import EncoderFSBO as PredictorModel +from transfer_nag_lib.MetaD2A_mobilenetV3.predictor import Predictor as MetaD2APredictor +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.train import train_single_model + +from diffusion.run_lib import generate_archs +from diffusion.run_lib import get_sampling_fn_meta +from diffusion.run_lib import get_score_model +from diffusion.run_lib import get_predictor + +sys.path.append(os.path.join(os.getcwd())) +from all_path import * +from utils import restore_checkpoint + + +class NAG: + def __init__(self, args, dgp_arch=[99, 50, 179, 194], bohb=False): + self.args = args + self.batch_size = args.batch_size + self.num_sample = args.num_sample + self.max_epoch = args.max_epoch + self.save_epoch = args.save_epoch + self.save_path = args.save_path + self.search_space = args.search_space + self.model_name = 'predictor' + self.test = args.test + self.device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") + self.max_corr_dict = {'corr': -1, 'epoch': -1} + self.train_arch = args.train_arch + self.use_metad2a_predictor_selec = args.use_metad2a_predictor_selec + + self.raw_data_path = RAW_DATA_PATH + self.model_path = UNNOISE_META_PREDICTOR_CKPT_PATH + self.data_path = PROCESSED_DATA_PATH + self.classifier_ckpt_path = NOISE_META_PREDICTOR_CKPT_PATH + self.load_diffusion_model(self.args.n_training_samples, args.pos_enc_type) + + graph_config = load_graph_config( + args.graph_data_name, args.nvt, self.data_path) + + self.model = PredictorModel(args, graph_config, dgp_arch=dgp_arch) + self.metad2a_model = MetaD2APredictor(args).model + + if self.test: + self.data_name = args.data_name + self.num_class = args.num_class + self.load_epoch = args.load_epoch + self.n_training_samples = self.args.n_training_samples + self.n_gen_samples = args.n_gen_samples + self.folder_name = args.folder_name + self.unique = args.unique + + model_state_dict = self.model.state_dict() + load_max_pt = 'ckpt_max_corr.pt' + ckpt_path = os.path.join(self.model_path, load_max_pt) + ckpt = torch.load(ckpt_path) + for k, v in ckpt.items(): + if k in model_state_dict.keys(): + model_state_dict[k] = v + self.model.cpu() + self.model.load_state_dict(model_state_dict) + self.model.to(self.device) + + self.optimizer = optim.Adam(self.model.parameters(), lr=args.lr) + self.scheduler = ReduceLROnPlateau(self.optimizer, 'min', + factor=0.1, patience=1000, verbose=True) + self.mtrloader = get_meta_train_loader( + self.batch_size, self.data_path, self.num_sample, is_pred=True) + + self.acc_mean = self.mtrloader.dataset.mean + self.acc_std = self.mtrloader.dataset.std + + + def forward(self, x, arch, labels=None, train=False, matrix=False, metad2a=False): + if metad2a: + D_mu = self.metad2a_model.set_encode(x.to(self.device)) + G_mu = self.metad2a_model.graph_encode(arch) + y_pred = self.metad2a_model.predict(D_mu, G_mu) + return y_pred + else: + D_mu = self.model.set_encode(x.to(self.device)) + G_mu = self.model.graph_encode(arch, matrix=matrix) + y_pred, y_dist = self.model.predict(D_mu, G_mu, labels=labels, train=train) + return y_pred, y_dist + + def meta_train(self): + sttime = time.time() + for epoch in range(1, self.max_epoch + 1): + self.mtrlog.ep_sttime = time.time() + loss, corr = self.meta_train_epoch(epoch) + self.scheduler.step(loss) + self.mtrlog.print_pred_log(loss, corr, 'train', epoch) + valoss, vacorr = self.meta_validation(epoch) + if self.max_corr_dict['corr'] < vacorr or epoch==1: + self.max_corr_dict['corr'] = vacorr + self.max_corr_dict['epoch'] = epoch + self.max_corr_dict['loss'] = valoss + save_model(epoch, self.model, self.model_path, max_corr=True) + + self.mtrlog.print_pred_log( + valoss, vacorr, 'valid', max_corr_dict=self.max_corr_dict) + + if epoch % self.save_epoch == 0: + save_model(epoch, self.model, self.model_path) + + self.mtrlog.save_time_log() + self.mtrlog.max_corr_log(self.max_corr_dict) + + def meta_train_epoch(self, epoch): + self.model.to(self.device) + self.model.train() + + self.mtrloader.dataset.set_mode('train') + + dlen = len(self.mtrloader.dataset) + trloss = 0 + y_all, y_pred_all = [], [] + pbar = tqdm(self.mtrloader) + + for x, g, acc in pbar: + self.optimizer.zero_grad() + y_pred, y_dist = self.forward(x, g, labels=acc, train=True, matrix=False) + y = acc.to(self.device).double() + print(y.double()) + print(y_dist) + loss = -self.model.mll(y_dist, y) + loss.backward() + self.optimizer.step() + + y = y.tolist() + y_pred = y_pred.squeeze().tolist() + y_all += y + y_pred_all += y_pred + pbar.set_description(get_log( + epoch, loss, y_pred, y, self.acc_std, self.acc_mean)) + trloss += float(loss) + + return trloss / dlen, pearsonr(np.array(y_all), + np.array(y_pred_all))[0] + + def meta_validation(self, epoch): + self.model.to(self.device) + self.model.eval() + + valoss = 0 + self.mtrloader.dataset.set_mode('valid') + dlen = len(self.mtrloader.dataset) + y_all, y_pred_all = [], [] + pbar = tqdm(self.mtrloader) + + with torch.no_grad(): + for x, g, acc in pbar: + y_pred, y_dist = self.forward(x, g, labels=acc, train=False, matrix=False) + y = acc.to(self.device) + loss = -self.model.mll(y_dist, y) + + y = y.tolist() + y_pred = y_pred.squeeze().tolist() + y_all += y + y_pred_all += y_pred + pbar.set_description(get_log( + epoch, loss, y_pred, y, self.acc_std, self.acc_mean, tag='val')) + valoss += float(loss) + try: + pearson_corr = pearsonr(np.array(y_all), np.array(y_pred_all))[0] + except Exception as e: + pearson_corr = 0 + + return valoss / dlen, pearson_corr + + def meta_test(self): + if self.data_name == 'all': + for data_name in ['cifar10', 'cifar100', 'aircraft', 'pets']: + acc = self.meta_test_per_dataset(data_name) + else: + acc = self.meta_test_per_dataset(self.data_name) + return acc + + + def meta_test_per_dataset(self, data_name): + self.test_dataset = MetaTestDataset( + self.data_path, data_name, self.num_sample, self.num_class) + + meta_test_path = self.args.exp_name + os.makedirs(meta_test_path, exist_ok=True) + f_arch_str = open(os.path.join(meta_test_path, 'architecture.txt'), 'w') + f = open(os.path.join(meta_test_path, 'accuracy.txt'), 'w') + + elasped_time = [] + + print(f'==> select top architectures for {data_name} by meta-predictor...') + + gen_arch_str = self.get_gen_arch_str() + + gen_arch_igraph = [decode_ofa_mbv3_str_to_igraph(_) for _ in gen_arch_str] + + y_pred_all = [] + self.metad2a_model.eval() + self.metad2a_model.to(self.device) + + # MetaD2A ver. prediction + sttime = time.time() + with torch.no_grad(): + for i, arch_igraph in enumerate(gen_arch_igraph): + x, g = self.collect_data(arch_igraph) + y_pred = self.forward(x, g, metad2a=True) + y_pred = torch.mean(y_pred) + y_pred_all.append(y_pred.cpu().detach().item()) + + if self.use_metad2a_predictor_selec: + top_arch_lst = self.select_top_arch( + data_name, torch.tensor(y_pred_all), gen_arch_str, self.n_training_samples) + else: + top_arch_lst = gen_arch_str[:self.n_training_samples] + + elasped = time.time() - sttime + elasped_time.append(elasped) + + for _, arch_str in enumerate(top_arch_lst): + f_arch_str.write(f'{arch_str}\n'); print(f'neural architecture config: {arch_str}') + + support = top_arch_lst + x_support = [] + y_support = [] + seeds = [777, 888, 999] + y_support_per_seed = { + _: [] for _ in seeds + } + net_info = { + 'params': [], + 'flops': [], + } + best_acc = 0.0 + best_sampe_num = 0 + + print("Data name: %s" % data_name) + for i, arch_str in enumerate(support): + save_path = os.path.join(meta_test_path, arch_str) + os.makedirs(save_path, exist_ok=True) + acc_runs = [] + for seed in seeds: + print(f'==> train for {data_name} {arch_str} ({seed})') + valid_acc, max_valid_acc, params, flops = train_single_model(save_path=save_path, + workers=8, + datasets=data_name, + xpaths=f'{self.raw_data_path}/{data_name}', + splits=[0], + use_less=False, + seed=seed, + model_str=arch_str, + device='cuda', + lr=0.01, + momentum=0.9, + weight_decay=4e-5, + report_freq=50, + epochs=20, + grad_clip=5, + cutout=True, + cutout_length=16, + autoaugment=True, + drop=0.2, + drop_path=0.2, + img_size=224) + acc_runs.append(valid_acc) + y_support_per_seed[seed].append(valid_acc) + + for r, acc in enumerate(acc_runs): + msg = f'run {r + 1} {acc:.2f} (%)' + f.write(msg + '\n') + f.flush() + print(msg) + m, h = mean_confidence_interval(acc_runs) + + if m > best_acc: + best_acc = m + best_sampe_num = i + msg = f'Avg {m:.3f}+-{h.item():.2f} (%) (best acc {best_acc:.3f} - #{i})' + f.write(msg + '\n') + print(msg) + y_support.append(np.mean(acc_runs)) + x_support.append(arch_str) + net_info['params'].append(params) + net_info['flops'].append(flops) + torch.save({'y_support': y_support, 'x_support': x_support, + 'y_support_per_seed': y_support_per_seed, + 'net_info': net_info, + 'best_acc': best_acc, + 'best_sample_num': best_sampe_num}, + meta_test_path+'/result.pt') + + + return None + + + def train_single_arch(self, data_name, arch_str, meta_test_path): + save_path = os.path.join(meta_test_path, arch_str) + seeds = (777, 888, 999) + train_single_model(save_path=save_path, + workers=24, + datasets=[data_name], + xpaths=[f'{self.raw_data_path}/{data_name}'], + splits=[0], + use_less=False, + seeds=seeds, + model_str=arch_str, + arch_config={'channel': 16, 'num_cells': 5}) + # Changed training time from 49/199 + epoch = 49 if data_name == 'mnist' else 199 + test_acc_lst = [] + for seed in seeds: + result = torch.load(os.path.join(save_path, f'seed-0{seed}.pth')) + test_acc_lst.append(result[data_name]['valid_acc1es'][f'x-test@{epoch}']) + return test_acc_lst + + + def select_top_arch( + self, data_name, y_pred_all, gen_arch_str, N): + _, sorted_idx = torch.sort(y_pred_all, descending=True) + sotred_gen_arch_str = [gen_arch_str[_] for _ in sorted_idx] + final_str = sotred_gen_arch_str[:N] + return final_str + + def collect_data_only(self): + x_batch = [] + x_batch.append(self.test_dataset[0]) + return torch.stack(x_batch).to(self.device) + + def collect_data(self, arch_igraph): + x_batch, g_batch = [], [] + for _ in range(10): + x_batch.append(self.test_dataset[0]) + g_batch.append(arch_igraph) + return torch.stack(x_batch).to(self.device), g_batch + + def load_diffusion_model(self, n_training_samples, pos_enc_type): + self.config = torch.load(CONFIG_PATH) + self.config.data.root = SCORE_MODEL_DATA_PATH + self.config.scorenet_ckpt_path = SCORE_MODEL_CKPT_PATH + torch.save(self.config, CONFIG_PATH) + + self.sampling_fn, self.sde = get_sampling_fn_meta(self.config) + self.sampling_fn_training_samples, _ = get_sampling_fn_meta(self.config, init=True, n_init=n_training_samples) + self.score_model, self.score_ema, self.score_config \ + = get_score_model(self.config, pos_enc_type=pos_enc_type) + + def get_gen_arch_str(self): + classifier_config = torch.load(self.classifier_ckpt_path)['config'] + # Load meta-predictor + classifier_model = get_predictor(classifier_config) + classifier_state = dict(model=classifier_model, step=0, config=classifier_config) + classifier_state = restore_checkpoint(self.classifier_ckpt_path, + classifier_state, device=self.config.device, resume=True) + print(f'==> load checkpoint for our predictor: {self.classifier_ckpt_path}...') + + with torch.no_grad(): + x = self.collect_data_only() + + generated_arch_str = generate_archs( + self.config, + self.sampling_fn, + self.score_model, + self.score_ema, + classifier_model, + num_samples=self.n_gen_samples, + patient_factor=self.args.patient_factor, + batch_size=self.args.eval_batch_size, + classifier_scale=self.args.classifier_scale, + task=x if self.args.fix_task else None) + + gc.collect() + return generated_arch_str diff --git a/MobileNetV3/main_exp/run_transfer_nag.py b/MobileNetV3/main_exp/run_transfer_nag.py new file mode 100644 index 0000000..c8d0690 --- /dev/null +++ b/MobileNetV3/main_exp/run_transfer_nag.py @@ -0,0 +1,154 @@ +import os +import sys +import random +import numpy as np +import argparse +import torch +import os +from nag import NAG +# sys.path.append(os.getcwd()) +# from utils import str2bool + + + +def str2bool(v): + return v.lower() in ['t', 'true', True] + +# save_path = "results" +# data_path = os.path.join('MetaD2A_nas_bench_201', 'data') +# model_load_path = '/home/data/GTAD/baselines/transferNAS' + + +def get_parser(): + parser = argparse.ArgumentParser() + # general settings + parser.add_argument('--seed', type=int, default=444) + parser.add_argument('--gpu', type=str, default='0', + help='set visible gpus') + parser.add_argument('--search_space', type=str, default='ofa') + parser.add_argument('--save-path', type=str, + default=None, help='the path of save directory') + parser.add_argument('--data-path', type=str, + default=None, help='the path of save directory') + parser.add_argument('--model-load-path', type=str, + default=None, help='') + parser.add_argument('--save-epoch', type=int, default=20, + help='how many epochs to wait each time to save model states') + parser.add_argument('--max-epoch', type=int, default=50, + help='number of epochs to train') + parser.add_argument('--batch_size', type=int, + default=1024, help='batch size for generator') + parser.add_argument('--graph-data-name', + default='ofa', help='graph dataset name') + parser.add_argument('--nvt', type=int, default=27, + help='number of different node types') + # set encoder + parser.add_argument('--num-sample', type=int, default=20, + help='the number of images as input for set encoder') + # graph encoder + parser.add_argument('--hs', type=int, default=512, + help='hidden size of GRUs') + parser.add_argument('--nz', type=int, default=56, + help='the number of dimensions of latent vectors z') + # test + parser.add_argument('--test', action='store_true', + default=True, help='turn on test mode') + parser.add_argument('--load-epoch', type=int, default=100, + help='checkpoint epoch loaded for meta-test') + parser.add_argument('--data-name', type=str, + default='pets', help='meta-test dataset name') + parser.add_argument('--trials', type=int, default=5) + + parser.add_argument('--num-class', type=int, default=None, + help='the number of class of dataset') + parser.add_argument('--num-gen-arch', type=int, default=500, + help='the number of candidate architectures generated by the generator') + parser.add_argument('--train-arch', type=str2bool, default=True, + help='whether to train the searched architecture') + parser.add_argument('--n_training_samples', type=int, default=5) + parser.add_argument('--N', type=int, default=10) + parser.add_argument('--use_gp', type=str2bool, default=False) + parser.add_argument('--sorting', type=str2bool, default=True) + parser.add_argument('--use_metad2a_predictor_selec', type=str2bool, default=True) + parser.add_argument('--use_ensemble_selec', type=str2bool, default=False) + + # ---------- For diffusion NAG ------------ # + parser.add_argument('--folder_name', type=str, default='DiffusionNAG') + parser.add_argument('--task', type=str, default='mtst') + parser.add_argument('--exp_name', type=str, default='') + parser.add_argument('--wandb_exp_name', type=str, default='') + parser.add_argument('--wandb_project_name', type=str, default='DiffusionNAG') + parser.add_argument('--use_wandb', type=str2bool, default=False) + parser.add_argument('--classifier_scale', type=int, default=10000.0, help='classifier scale') + parser.add_argument('--eval_batch_size', type=int, default=256) + parser.add_argument('--predictor', type=str, default='euler_maruyama', + choices=['euler_maruyama', 'reverse_diffusion', 'none']) + parser.add_argument('--corrector', type=str, default='langevin', + choices=['none', 'langevin']) + parser.add_argument('--weight_ratio', type=str2bool, default=False) + parser.add_argument('--weight_scheduling', type=str2bool, default=False) + parser.add_argument('--weight_ratio_abs', type=str2bool, default=False) + parser.add_argument('--p', type=int, default=1) + parser.add_argument('--prod_w', type=str2bool, default=False) + parser.add_argument('--t_spot', type=float, default=1.0) + parser.add_argument('--t_spot_end', type=float, default=0.0) + # Train + parser.add_argument('--lr', type=float, default=0.001, help='learning rate') + parser.add_argument('--epochs', type=int, default=500) + parser.add_argument('--save_best_p_corr', type=str2bool, default=True) + parser.add_argument('--unique', type=str2bool, default=True) + parser.add_argument('--patient_factor', type=int, default=20) + parser.add_argument('--n_gen_samples', type=int, default=50) + ################ OFA #################### + parser.add_argument('--ofa_path', type=str, default='/home/hayeon/imagenet1k', help='') + parser.add_argument('--ofa_batch_size', type=int, default=256, help='') + parser.add_argument('--ofa_workers', type=int, default=4, help='') + ################ Diffusion ############## + parser.add_argument('--diffusion_lr', type=float, default=1e-3, help='') + parser.add_argument('--noise_aware_acc_norm', type=int, default=-1) + parser.add_argument('--fix_task', type=str2bool, default=True) + ################ BO #################### + parser.add_argument('--bo_loop_max_epoch', type=int, default=30) + parser.add_argument('--bo_loop_acc_norm', type=int, default=1) + parser.add_argument('--gp_model_acc_norm', type=int, default=1) + parser.add_argument('--num_ensemble', type=int, default=3) + parser.add_argument('--explore_type', type=str, default='ei') + ################ BO #################### + # parser.add_argument('--multi_proc', type=str2bool, default=False) + parser.add_argument('--eps', type=float, default=0.) + parser.add_argument('--beta', type=float, default=0.5) + parser.add_argument('--pos_enc_type', type=int, default=4) + args = parser.parse_args() + + return args + +def set_exp_name(args): + exp_name = f'./exp/{args.task}/{args.folder_name}/data-{args.data_name}' + wandb_exp_name = f'./exp/{args.task}/{args.folder_name}/{args.data_name}' + + os.makedirs(exp_name, exist_ok=True) + args.exp_name = exp_name + args.wandb_exp_name = wandb_exp_name + + +def main(): + args = get_parser() + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + torch.cuda.manual_seed(args.seed) + torch.manual_seed(args.seed) + np.random.seed(args.seed) + random.seed(args.seed) + + set_exp_name(args) + + p = NAG(args) + + if args.test: + p.meta_test() + else: + p.meta_train() + + +if __name__ == '__main__': + main() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/DeepKernelGPHelpers.py b/MobileNetV3/main_exp/transfer_nag_lib/DeepKernelGPHelpers.py new file mode 100644 index 0000000..95e23e8 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/DeepKernelGPHelpers.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Tue Jul 6 14:02:53 2021 + +@author: hsjomaa +""" +import numpy as np +from scipy.stats import norm +import pandas as pd +from torch import autograd as ag +import torch +from sklearn.preprocessing import PowerTransformer + + +def regret(output,response): + incumbent = output[0] + best_output = [] + for _ in output: + incumbent = _ if _ > incumbent else incumbent + best_output.append(incumbent) + opt = max(response) + orde = list(np.sort(np.unique(response))[::-1]) + tmp = pd.DataFrame(best_output,columns=['regret_validation']) + + tmp['rank_valid'] = tmp['regret_validation'].map(lambda x : orde.index(x)) + tmp['regret_validation'] = opt - tmp['regret_validation'] + return tmp + +def EI(incumbent, model_fn,support,queries,return_variance, return_score=False): + mu, stddev = model_fn(queries) + mu = mu.reshape(-1,) + stddev = stddev.reshape(-1,) + if return_variance: + stddev = np.sqrt(stddev) + with np.errstate(divide='warn'): + imp = mu - incumbent + Z = imp / stddev + score = imp * norm.cdf(Z) + stddev * norm.pdf(Z) + if not return_score: + score[support] = 0 + return np.argmax(score) + else: + return score + + +class Metric(object): + def __init__(self,prefix='train: '): + self.reset() + self.message=prefix + "loss: {loss:.2f} - noise: {log_var:.2f} - mse: {mse:.2f}" + + def update(self,loss,noise,mse): + self.loss.append(np.asscalar(loss)) + self.noise.append(np.asscalar(noise)) + self.mse.append(np.asscalar(mse)) + + def reset(self,): + self.loss = [] + self.noise = [] + self.mse = [] + + def report(self): + return self.message.format(loss=np.mean(self.loss), + log_var=np.mean(self.noise), + mse=np.mean(self.mse)) + + def get(self): + return {"loss":np.mean(self.loss), + "noise":np.mean(self.noise), + "mse":np.mean(self.mse)} + +def totorch(x,device): + if type(x) is tuple: + return tuple([ag.Variable(torch.Tensor(e)).to(device) for e in x]) + return torch.Tensor(x).to(device) + + +def prepare_data(indexes, support, Lambda, response, metafeatures=None, output_transform=False): + # Generate indexes of the batch + X,E,Z,y,r = [],[],[],[],[] + #### get support data + for dim in indexes: + if metafeatures is not None: + Z.append(metafeatures) + E.append(Lambda[support]) + X.append(Lambda[dim]) + r_ = response[support,np.newaxis] + y_ = response[dim] + if output_transform: + power = PowerTransformer(method="yeo-johnson") + r_ = power.fit_transform(r_) + y_ = power.transform(y_.reshape(-1,1)).reshape(-1,) + r.append(r_) + y.append(y_) + X = np.array(X) + E = np.array(E) + Z = np.array(Z) + y = np.array(y) + r = np.array(r) + return (np.expand_dims(E, axis=-1), r, np.expand_dims(X, axis=-1), Z), y diff --git a/MobileNetV3/main_exp/transfer_nag_lib/DeepKernelGPModules.py b/MobileNetV3/main_exp/transfer_nag_lib/DeepKernelGPModules.py new file mode 100644 index 0000000..48e7e59 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/DeepKernelGPModules.py @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Tue Jul 6 14:03:42 2021 + +@author: hsjomaa +""" +## Original packages +import torch +import torch.nn as nn +from sklearn.preprocessing import MinMaxScaler +import copy +import numpy as np +import os +# from torch.utils.tensorboard import SummaryWriter +import json +import time +## Our packages +import gpytorch +import logging +from transfer_nag_lib.DeepKernelGPHelpers import totorch,prepare_data, Metric, EI +from transfer_nag_lib.MetaD2A_nas_bench_201.generator import Generator +from transfer_nag_lib.MetaD2A_nas_bench_201.main import get_parser +np.random.seed(1203) +RandomQueryGenerator= np.random.RandomState(413) +RandomSupportGenerator= np.random.RandomState(413) +RandomTaskGenerator = np.random.RandomState(413) + + +class DeepKernelGP(nn.Module): + + def __init__(self,X,Y,Z,kernel,backbone_fn, config, support,log_dir,seed): + super(DeepKernelGP, self).__init__() + torch.manual_seed(seed) + ## GP parameters + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.X,self.Y,self.Z = X,Y,Z + self.feature_extractor = backbone_fn().to(self.device) + self.config=config + self.get_model_likelihood_mll(len(support),kernel,backbone_fn) + + logging.basicConfig(filename=log_dir, level=logging.DEBUG) + + def get_model_likelihood_mll(self, train_size,kernel,backbone_fn): + + train_x=torch.ones(train_size, self.feature_extractor.out_features).to(self.device) + train_y=torch.ones(train_size).to(self.device) + + likelihood = gpytorch.likelihoods.GaussianLikelihood() + model = ExactGPLayer(train_x=train_x, train_y=train_y, likelihood=likelihood, config=self.config, + dims=self.feature_extractor.out_features) + self.model = model.to(self.device) + self.likelihood = likelihood.to(self.device) + self.mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model).to(self.device) + + def set_forward(self, x, is_feature=False): + pass + + def set_forward_loss(self, x): + pass + + def train(self, support, load_model,optimizer, checkpoint=None,epochs=1000, verbose = False): + + if load_model: + assert(checkpoint is not None) + print("KEYS MATCHED") + self.load_checkpoint(os.path.join(checkpoint,"weights")) + + inputs,labels = prepare_data(support,support,self.X,self.Y,self.Z) + inputs,labels = totorch(inputs,device=self.device), totorch(labels.reshape(-1,),device=self.device) + losses = [np.inf] + best_loss = np.inf + starttime = time.time() + initial_weights = copy.deepcopy(self.state_dict()) + patience=0 + max_patience = self.config["patience"] + for _ in range(epochs): + optimizer.zero_grad() + z = self.feature_extractor(inputs) + self.model.set_train_data(inputs=z, targets=labels) + predictions = self.model(z) + try: + loss = -self.mll(predictions, self.model.train_targets) + loss.backward() + optimizer.step() + except Exception as ada: + logging.info(f"Exception {ada}") + break + + if verbose: + print("Iter {iter}/{epochs} - Loss: {loss:.5f} noise: {noise:.5f}".format( + iter=_+1,epochs=epochs,loss=loss.item(),noise=self.likelihood.noise.item())) + losses.append(loss.detach().to("cpu").item()) + if best_loss>losses[-1]: + best_loss = losses[-1] + weights = copy.deepcopy(self.state_dict()) + if np.allclose(losses[-1],losses[-2],atol=self.config["loss_tol"]): + patience+=1 + else: + patience=0 + if patience>max_patience: + break + self.load_state_dict(weights) + logging.info(f"Current Iteration: {len(support)} | Incumbent {max(self.Y[support])} | Duration {np.round(time.time()-starttime)} | Epochs {_} | Noise {self.likelihood.noise.item()}") + return losses,weights,initial_weights + + def load_checkpoint(self, checkpoint): + ckpt = torch.load(checkpoint,map_location=torch.device(self.device)) + self.model.load_state_dict(ckpt['gp'],strict=False) + self.likelihood.load_state_dict(ckpt['likelihood'],strict=False) + self.feature_extractor.load_state_dict(ckpt['net'],strict=False) + + + def predict(self,support, query_range=None, noise_fn=None): + + card = len(self.Y) + if noise_fn: + self.Y = noise_fn(self.Y) + x_support,y_support = prepare_data(support,support, + self.X,self.Y,self.Z) + if query_range is None: + x_query,_ = prepare_data(np.arange(card),support, + self.X,self.Y,self.Z) + else: + x_query,_ = prepare_data(query_range,support, + self.X,self.Y,self.Z) + self.model.eval() + self.feature_extractor.eval() + self.likelihood.eval() + + z_support = self.feature_extractor(totorch(x_support,self.device)).detach() + self.model.set_train_data(inputs=z_support, targets=totorch(y_support.reshape(-1,),self.device), strict=False) + + with torch.no_grad(): + z_query = self.feature_extractor(totorch(x_query,self.device)).detach() + pred = self.likelihood(self.model(z_query)) + + + mu = pred.mean.detach().to("cpu").numpy().reshape(-1,) + stddev = pred.stddev.detach().to("cpu").numpy().reshape(-1,) + + return mu,stddev + +class DKT(nn.Module): + def __init__(self, train_data,valid_data, kernel,backbone_fn, config): + super(DKT, self).__init__() + ## GP parameters + self.train_data = train_data + self.valid_data = valid_data + self.fixed_context_size = config["fixed_context_size"] + self.minibatch_size = config["minibatch_size"] + self.n_inner_steps = config["n_inner_steps"] + self.checkpoint_path = config["checkpoint_path"] + os.makedirs(self.checkpoint_path,exist_ok=False) + json.dump(config, open(os.path.join(self.checkpoint_path,"configuration.json"),"w")) + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + logging.basicConfig(filename=os.path.join(self.checkpoint_path,"log.txt"), level=logging.DEBUG) + self.feature_extractor = backbone_fn().to(self.device) + self.config=config + self.get_model_likelihood_mll(self.fixed_context_size,kernel,backbone_fn) + self.mse = nn.MSELoss() + self.curr_valid_loss = np.inf + self.get_tasks() + self.setup_writers() + + self.train_metrics = Metric() + self.valid_metrics = Metric(prefix="valid: ") + print(self) + + + def setup_writers(self,): + train_log_dir = os.path.join(self.checkpoint_path,"train") + os.makedirs(train_log_dir,exist_ok=True) + self.train_summary_writer = SummaryWriter(train_log_dir) + + valid_log_dir = os.path.join(self.checkpoint_path,"valid") + os.makedirs(valid_log_dir,exist_ok=True) + self.valid_summary_writer = SummaryWriter(valid_log_dir) + + def get_tasks(self,): + pairs = [] + for space in self.train_data.keys(): + for task in self.train_data[space].keys(): + pairs.append([space,task]) + self.tasks = pairs + ########## + pairs = [] + for space in self.valid_data.keys(): + for task in self.valid_data[space].keys(): + pairs.append([space,task]) + self.valid_tasks = pairs + + + def get_model_likelihood_mll(self, train_size,kernel,backbone_fn): + + train_x=torch.ones(train_size, self.feature_extractor.out_features).to(self.device) + train_y=torch.ones(train_size).to(self.device) + + likelihood = gpytorch.likelihoods.GaussianLikelihood() + model = ExactGPLayer(train_x=train_x, train_y=train_y, likelihood=likelihood, config=self.config,dims = self.feature_extractor.out_features) + self.model = model.to(self.device) + self.likelihood = likelihood.to(self.device) + self.mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model).to(self.device) + + def set_forward(self, x, is_feature=False): + pass + + def set_forward_loss(self, x): + pass + + def epoch_end(self): + RandomTaskGenerator.shuffle(self.tasks) + + def train_loop(self, epoch, optimizer, scheduler_fn=None): + if scheduler_fn: + scheduler = scheduler_fn(optimizer,len(self.tasks)) + self.epoch_end() + assert(self.training) + for task in self.tasks: + inputs, labels = self.get_batch(task) + for _ in range(self.n_inner_steps): + optimizer.zero_grad() + z = self.feature_extractor(inputs) + self.model.set_train_data(inputs=z, targets=labels, strict=False) + predictions = self.model(z) + loss = -self.mll(predictions, self.model.train_targets) + loss.backward() + optimizer.step() + mse = self.mse(predictions.mean, labels) + self.train_metrics.update(loss,self.model.likelihood.noise,mse) + if scheduler_fn: + scheduler.step() + + training_results = self.train_metrics.get() + for k,v in training_results.items(): + self.train_summary_writer.add_scalar(k, v, epoch) + for task in self.valid_tasks: + mse,loss = self.test_loop(task,train=False) + self.valid_metrics.update(loss,np.array(0),mse,) + + logging.info(self.train_metrics.report() + " " + self.valid_metrics.report()) + validation_results = self.valid_metrics.get() + for k,v in validation_results.items(): + self.valid_summary_writer.add_scalar(k, v, epoch) + self.feature_extractor.train() + self.likelihood.train() + self.model.train() + + if validation_results["loss"] < self.curr_valid_loss: + self.save_checkpoint(os.path.join(self.checkpoint_path,"weights")) + self.curr_valid_loss = validation_results["loss"] + self.valid_metrics.reset() + self.train_metrics.reset() + + def test_loop(self, task, train, optimizer=None): # no optimizer needed for GP + (x_support, y_support),(x_query,y_query) = self.get_support_and_queries(task,train) + z_support = self.feature_extractor(x_support).detach() + self.model.set_train_data(inputs=z_support, targets=y_support, strict=False) + self.model.eval() + self.feature_extractor.eval() + self.likelihood.eval() + + with torch.no_grad(): + z_query = self.feature_extractor(x_query).detach() + pred = self.likelihood(self.model(z_query)) + loss = -self.mll(pred, y_query) + lower, upper = pred.confidence_region() #2 standard deviations above and below the mean + + mse = self.mse(pred.mean, y_query) + + return mse,loss + + def get_batch(self,task): + # we want to fit the gp given context info to new observations + # task is an algorithm/dataset pair + space,task = task + Lambda,response = np.array(self.train_data[space][task]["X"]), MinMaxScaler().fit_transform(np.array(self.train_data[space][task]["y"])).reshape(-1,) + + card, dim = Lambda.shape + + support = RandomSupportGenerator.choice(np.arange(card), + replace=False,size=self.fixed_context_size) + remaining = np.setdiff1d(np.arange(card),support) + indexes = RandomQueryGenerator.choice( + remaining,replace=False,size=self.minibatch_size if len(remaining)>self.minibatch_size else len(remaining)) + + inputs,labels = prepare_data(support,indexes,Lambda,response,np.zeros(32)) + inputs,labels = totorch(inputs,device=self.device), totorch(labels.reshape(-1,),device=self.device) + return inputs, labels + + def get_support_and_queries(self,task, train=False): + + # task is an algorithm/dataset pair + space,task = task + + hpo_data = self.valid_data if not train else self.train_data + Lambda,response = np.array(hpo_data[space][task]["X"]), MinMaxScaler().fit_transform(np.array(hpo_data[space][task]["y"])).reshape(-1,) + card, dim = Lambda.shape + + support = RandomSupportGenerator.choice(np.arange(card), + replace=False,size=self.fixed_context_size) + indexes = RandomQueryGenerator.choice( + np.setdiff1d(np.arange(card),support),replace=False,size=self.minibatch_size) + + support_x,support_y = prepare_data(support,support,Lambda,response,np.zeros(32)) + query_x,query_y = prepare_data(support,indexes,Lambda,response,np.zeros(32)) + + return (totorch(support_x,self.device),totorch(support_y.reshape(-1,),self.device)),\ + (totorch(query_x,self.device),totorch(query_y.reshape(-1,),self.device)) + + def save_checkpoint(self, checkpoint): + # save state + gp_state_dict = self.model.state_dict() + likelihood_state_dict = self.likelihood.state_dict() + nn_state_dict = self.feature_extractor.state_dict() + torch.save({'gp': gp_state_dict, 'likelihood': likelihood_state_dict, 'net':nn_state_dict}, checkpoint) + + def load_checkpoint(self, checkpoint): + ckpt = torch.load(checkpoint) + self.model.load_state_dict(ckpt['gp']) + self.likelihood.load_state_dict(ckpt['likelihood']) + self.feature_extractor.load_state_dict(ckpt['net']) + +class ExactGPLayer(gpytorch.models.ExactGP): + def __init__(self, train_x, train_y, likelihood,config,dims ): + super(ExactGPLayer, self).__init__(train_x, train_y, likelihood) + self.mean_module = gpytorch.means.ConstantMean() + + ## RBF kernel + if(config["kernel"]=='rbf' or config["kernel"]=='RBF'): + self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.RBFKernel(ard_num_dims=dims if config["ard"] else None)) + elif(config["kernel"]=='52'): + self.covar_module = gpytorch.kernels.ScaleKernel(gpytorch.kernels.MaternKernel(nu=config["nu"],ard_num_dims=dims if config["ard"] else None)) + ## Spectral kernel + else: + raise ValueError("[ERROR] the kernel '" + str(config["kernel"]) + "' is not supported for regression, use 'rbf' or 'spectral'.") + + def forward(self, x): + mean_x = self.mean_module(x) + covar_x = self.covar_module(x) + return gpytorch.distributions.MultivariateNormal(mean_x, covar_x) + + +class batch_mlp(nn.Module): + def __init__(self, d_in, output_sizes, nonlinearity="relu",dropout=0.0): + + super(batch_mlp, self).__init__() + assert(nonlinearity=="relu") + self.nonlinearity = nn.ReLU() + + self.fc = nn.ModuleList([nn.Linear(in_features=d_in, out_features=output_sizes[0])]) + for d_out in output_sizes[1:]: + self.fc.append(nn.Linear(in_features=self.fc[-1].out_features, out_features=d_out)) + self.out_features = output_sizes[-1] + self.dropout = nn.Dropout(dropout) + def forward(self,x): + + for fc in self.fc[:-1]: + x = fc(x) + x = self.dropout(x) + x = self.nonlinearity(x) + x = self.fc[-1](x) + x = self.dropout(x) + return x + +class StandardDeepGP(nn.Module): + def __init__(self, configuration): + + super(StandardDeepGP, self).__init__() + self.A = batch_mlp(configuration["dim"], configuration["output_size_A"],dropout=configuration["dropout"]) + self.out_features = configuration["output_size_A"][-1] + + + def forward(self, x): + # e,r,x,z = x + hidden = self.A(x.squeeze(dim=-1)) ### NxA + return hidden + + +class DKTNAS(nn.Module): + def __init__(self, kernel, backbone_fn, config, pretrained_encoder=True, GP_only=False): + super(DKTNAS, self).__init__() + ## GP parameters + + self.fixed_context_size = config["fixed_context_size"] + self.minibatch_size = config["minibatch_size"] + self.n_inner_steps = config["n_inner_steps"] + self.set_encoder_args = get_parser() + if not os.path.exists(self.set_encoder_args.save_path): + os.makedirs(self.set_encoder_args.save_path) + self.set_encoder_args.model_path = os.path.join(self.set_encoder_args.save_path, + self.set_encoder_args.model_name, 'model') + if not os.path.exists(self.set_encoder_args.model_path): + os.makedirs(self.set_encoder_args.model_path) + self.set_encoder = Generator(self.set_encoder_args) + if pretrained_encoder: + self.dataset_enc, self.arch, self.acc = self.set_encoder.train_dgp(encode=False) + self.dataset_enc_val, self.acc_val = self.set_encoder.test_dgp(data_name='cifar100', encode=False) + else: # In case we want to train the set-encoder from scratch + self.dataset_enc = np.load("train_data_path.npy") + self.acc = np.load("train_acc.npy") + self.dataset_enc_val = np.load("cifar100_data_path.npy") + self.acc_val = np.load("cifar100_acc.npy") + self.valid_data = self.dataset_enc_val + self.checkpoint_path = config["checkpoint_path"] + os.makedirs(self.checkpoint_path, exist_ok=False) + json.dump(config, open(os.path.join(self.checkpoint_path, "configuration.json"), "w")) + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + logging.basicConfig(filename=os.path.join(self.checkpoint_path, "log.txt"), level=logging.DEBUG) + self.feature_extractor = backbone_fn().to(self.device) + self.config = config + self.GP_only = GP_only + self.get_model_likelihood_mll(self.fixed_context_size, kernel, backbone_fn) + self.mse = nn.MSELoss() + self.curr_valid_loss = np.inf + # self.get_tasks() + self.setup_writers() + + self.train_metrics = Metric() + self.valid_metrics = Metric(prefix="valid: ") + self.tasks = len(self.dataset_enc) + + print(self) + + def setup_writers(self, ): + train_log_dir = os.path.join(self.checkpoint_path, "train") + os.makedirs(train_log_dir, exist_ok=True) + # self.train_summary_writer = SummaryWriter(train_log_dir) + + valid_log_dir = os.path.join(self.checkpoint_path, "valid") + os.makedirs(valid_log_dir, exist_ok=True) + # self.valid_summary_writer = SummaryWriter(valid_log_dir) + + + def get_model_likelihood_mll(self, train_size, kernel, backbone_fn): + if not self.GP_only: + train_x = torch.ones(train_size, self.feature_extractor.out_features).to(self.device) + train_y = torch.ones(train_size).to(self.device) + + likelihood = gpytorch.likelihoods.GaussianLikelihood() + + model = ExactGPLayer(train_x=None, train_y=None, likelihood=likelihood, config=self.config, + dims=self.feature_extractor.out_features) + else: + train_x = torch.ones(train_size, self.fixed_context_size).to(self.device) + train_y = torch.ones(train_size).to(self.device) + + likelihood = gpytorch.likelihoods.GaussianLikelihood() + + model = ExactGPLayer(train_x=None, train_y=None, likelihood=likelihood, config=self.config, + dims=self.fixed_context_size) + self.model = model.to(self.device) + self.likelihood = likelihood.to(self.device) + self.mll = gpytorch.mlls.ExactMarginalLogLikelihood(likelihood, model).to(self.device) + + def set_forward(self, x, is_feature=False): + pass + + def set_forward_loss(self, x): + pass + + def epoch_end(self): + RandomTaskGenerator.shuffle([1]) + + def train_loop(self, epoch, optimizer, scheduler_fn=None): + if scheduler_fn: + scheduler = scheduler_fn(optimizer, 1) + self.epoch_end() + assert (self.training) + for task in range(self.tasks): + inputs, labels = self.get_batch(task) + for _ in range(self.n_inner_steps): + optimizer.zero_grad() + z = self.feature_extractor(inputs) + self.model.set_train_data(inputs=z, targets=labels, strict=False) + predictions = self.model(z) + loss = -self.mll(predictions, self.model.train_targets) + loss.backward() + optimizer.step() + mse = self.mse(predictions.mean, labels) + self.train_metrics.update(loss, self.model.likelihood.noise, mse) + if scheduler_fn: + scheduler.step() + + training_results = self.train_metrics.get() + for k, v in training_results.items(): + self.train_summary_writer.add_scalar(k, v, epoch) + mse, loss = self.test_loop(train=False) + self.valid_metrics.update(loss, np.array(0), mse, ) + + logging.info(self.train_metrics.report() + " " + self.valid_metrics.report()) + validation_results = self.valid_metrics.get() + for k, v in validation_results.items(): + self.valid_summary_writer.add_scalar(k, v, epoch) + self.feature_extractor.train() + self.likelihood.train() + self.model.train() + + if validation_results["loss"] < self.curr_valid_loss: + self.save_checkpoint(os.path.join(self.checkpoint_path, "weights")) + self.curr_valid_loss = validation_results["loss"] + self.valid_metrics.reset() + self.train_metrics.reset() + + def test_loop(self, train=None, optimizer=None): # no optimizer needed for GP + (x_support, y_support), (x_query, y_query) = self.get_support_and_queries(train) + z_support = self.feature_extractor(x_support).detach() + self.model.set_train_data(inputs=z_support, targets=y_support, strict=False) + self.model.eval() + self.feature_extractor.eval() + self.likelihood.eval() + + with torch.no_grad(): + z_query = self.feature_extractor(x_query).detach() + pred = self.likelihood(self.model(z_query)) + loss = -self.mll(pred, y_query) + lower, upper = pred.confidence_region() # 2 standard deviations above and below the mean + + mse = self.mse(pred.mean, y_query) + + return mse, loss + + def get_batch(self, task, valid=False): + + # we want to fit the gp given context info to new observations + #TODO: scale the response as in FSBO(needed for train) + Lambda, response = np.array(self.dataset_enc), np.array(self.acc) + + inputs, labels = Lambda[task], response[task] + inputs, labels = totorch([inputs], device=self.device), totorch([labels], device=self.device) + return inputs, labels + + def get_support_and_queries(self, task, train=False): + + # TODO: scale the response as in FSBO(not necessary for test) + Lambda, response = np.array(self.dataset_enc_val), np.array(self.acc_val) + card, dim = Lambda.shape + + support = RandomSupportGenerator.choice(np.arange(card), + replace=False, size=self.fixed_context_size) + indexes = RandomQueryGenerator.choice( + np.setdiff1d(np.arange(card), support), replace=False, size=self.minibatch_size) + + support_x, support_y = Lambda[support], response[support] + query_x, query_y = Lambda[indexes], response[indexes] + + return (totorch(support_x, self.device), totorch(support_y.reshape(-1, ), self.device)), \ + (totorch(query_x, self.device), totorch(query_y.reshape(-1, ), self.device)) + + def save_checkpoint(self, checkpoint): + # save state + gp_state_dict = self.model.state_dict() + likelihood_state_dict = self.likelihood.state_dict() + nn_state_dict = self.feature_extractor.state_dict() + torch.save({'gp': gp_state_dict, 'likelihood': likelihood_state_dict, 'net': nn_state_dict}, checkpoint) + + def load_checkpoint(self, checkpoint): + ckpt = torch.load(checkpoint) + self.model.load_state_dict(ckpt['gp']) + self.likelihood.load_state_dict(ckpt['likelihood']) + self.feature_extractor.load_state_dict(ckpt['net']) + + def predict(self, x_support, y_support, x_query, y_query, GP_only=False): + if not GP_only: + z_support = self.feature_extractor(x_support).detach() + else: + z_support = x_support + self.model.set_train_data(inputs=z_support, targets=y_support, strict=False) + self.model.eval() + self.feature_extractor.eval() + self.likelihood.eval() + + with torch.no_grad(): + if not GP_only: + z_query = self.feature_extractor(x_query).detach() + else: + z_query = x_query + pred = self.likelihood(self.model(z_query)) + mu = pred.mean.detach().to("cpu").numpy().reshape(-1, ) + stddev = pred.stddev.detach().to("cpu").numpy().reshape(-1, ) + return mu, stddev diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/README.md b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/README.md new file mode 100644 index 0000000..0595fab --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/README.md @@ -0,0 +1,168 @@ +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets +This code is for MobileNetV3 Search Space experiments + + +## Prerequisites +- Python 3.6 (Anaconda) +- PyTorch 1.6.0 +- CUDA 10.2 +- python-igraph==0.8.2 +- tqdm==4.50.2 +- torchvision==0.7.0 +- python-igraph==0.8.2 +- scipy==1.5.2 +- ofa==0.0.4-2007200808 + + +## MobileNetV3 Search Space +Go to the folder for MobileNetV3 experiments (i.e. ```MetaD2A_mobilenetV3```) + +The overall flow is summarized as follows: +- Building database for Predictor +- Meta-Training Predictor +- Building database for Generator with trained Predictor +- Meta-Training Generator +- Meta-Testing (Searching) +- Evaluating the Searched architecture + + +## Data Preparation +To download preprocessed data files, run ```get_files/get_preprocessed_data.py```: +```shell script +$ python get_files/get_preprocessed_data.py +``` +It will take some time to download and preprocess each dataset. + + +## Meta Test and Evaluation +### Meta-Test + +You can download trained checkpoint files for generator and predictor +```shell script +$ python get_files/get_generator_checkpoint.py +$ python get_files/get_predictor_checkpoint.py +``` + +If you want to meta-test with your own dataset, please first make your own preprocessed data, +by modifying ```process_dataset.py``` . +```shell script +$ process_dataset.py +``` + +This code automatically generates neural architecturess and then +selects high-performing architectures among the candidates. +By setting ```--data-name``` as the name of dataset (i.e. ```cifar10```, ```cifar100```, ```aircraft100```, ```pets```), +you can evaluate the specific dataset. + +```shell script +# Meta-testing +$ python main.py --gpu 0 --model generator --hs 56 --nz 56 --test --load-epoch 120 --num-gen-arch 200 --data-name {DATASET_NAME} +``` + +### Arhictecture Evaluation (MetaD2A vs NSGANetV2) +##### Dataset Preparation +You need to download Oxford-IIIT Pet dataset to evaluate on ```--data-name pets``` +```shell script +$ python get_files/get_pets.py +``` +Every others ```cifar10```, ```cifar100```, ```aircraft100``` will be downloaded automatically. + +##### evaluation +You can run the searched architecture by running ```evaluation/main```. Codes are based on NSGANetV2. + +Go to the evaluation folder (i.e. ```evaluation```) +```shell script +$ cd evaluation +``` + +This automatically run the top 1 predicted architecture derived by MetaD2A. +```shell script +python main.py --data-name cifar10 --num-gen-arch 200 +``` +You can also give flop constraint by using ```bound``` option. +```shell script +python main.py --data-name cifar10 --num-gen-arch 200 --bound 300 +``` + +You can compare MetaD2A with NSGANetV2 +but you need to download some files provided +by [NSGANetV2](https://github.com/human-analysis/nsganetv2) + +```shell script +python main.py --data-name cifar10 --num-gen-arch 200 --model-config flops@232 +``` + + +## Meta-Training MetaD2A Model +To build database for Meta-training, you need to set ```IMGNET_PATH```, which is a directory of ILSVRC2021. + +### Database Building for Predictor +We recommend you to run the multiple ```create_database.sh``` simultaneously to build fast. +You need to set ```IMGNET_PATH``` in the shell script. +```shell script +# Examples +bash create_database.sh 0,1,2,3 0 49 predictor +bash create_database.sh all 50 99 predictor +... +``` +After enough dataset is gathered, run ```build_database.py``` to collect them as one file. +```shell script +python build_database.py --model_name predictor --collect +``` + +We also provide the database we use. To download database, run ```get_files/get_predictor_database.py```: +```shell script +$ python get_files/get_predictor_database.py +``` + +### Meta-Train Predictor +You can train the predictor as follows +```shell script +# Meta-training for predictor +$ python main.py --gpu 0 --model predictor --hs 512 --nz 56 +``` +### Database Building for Generator +We recommend you to run the multiple ```create_database.sh``` simultaneously to build fast. +```shell script +# Examples +bash create_database.sh 4,5,6,7 0 49 generator +bash create_database.sh all 50 99 generator +... +``` +After enough dataset is gathered, run ```build_database.py``` to collect them as one. +```shell script +python build_database.py --model_name generator --collect +``` + +We also provide the database we use. To download database, run ```get_files/get_generator_database.py``` +```shell script +$ python get_files/get_generator_database.py +``` + + +### Meta-Train Generator +You can train the generator as follows +```shell script +# Meta-training for generator +$ python main.py --gpu 0 --model generator --hs 56 --nz 56 +``` + + + +## Citation +If you found the provided code useful, please cite our work. +``` +@inproceedings{ + lee2021rapid, + title={Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets}, + author={Hayeon Lee and Eunyoung Hyung and Sung Ju Hwang}, + booktitle={ICLR}, + year={2021} +} +``` + +## Reference +- [Set Transformer: A Framework for Attention-based Permutation-Invariant Neural Networks (ICML2019)](https://github.com/juho-lee/set_transformer) +- [D-VAE: A Variational Autoencoder for Directed Acyclic Graphs, Advances in Neural Information Processing Systems (NeurIPS2019)](https://github.com/muhanzhang/D-VAE) +- [Once for All: Train One Network and Specialize it for Efficient Deployment (ICLR2020)](https://github.com/mit-han-lab/once-for-all) +- [NSGANetV2: Evolutionary Multi-Objective Surrogate-Assisted Neural Architecture Search (ECCV2020)](https://github.com/human-analysis/nsganetv2) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/build_database.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/build_database.py new file mode 100644 index 0000000..1353ee4 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/build_database.py @@ -0,0 +1,49 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +import random +import numpy as np +import torch +from parser import get_parser +from predictor import PredictorModel +from database import DatabaseOFA +from utils import load_graph_config + +def main(): + args = get_parser() + + if args.gpu == 'all': + device_list = range(torch.cuda.device_count()) + args.gpu = ','.join(str(_) for _ in device_list) + else: + device_list = [int(_) for _ in args.gpu.split(',')] + os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu + args.device = torch.device("cuda:0") + args.batch_size = args.batch_size * max(len(device_list), 1) + + torch.cuda.manual_seed(args.seed) + torch.manual_seed(args.seed) + np.random.seed(args.seed) + random.seed(args.seed) + + args.model_path = os.path.join(args.save_path, args.model_name, 'model') + + if args.model_name == 'generator': + graph_config = load_graph_config( + args.graph_data_name, args.nvt, args.data_path) + model = PredictorModel(args, graph_config) + d = DatabaseOFA(args, model) + else: + d = DatabaseOFA(args) + + if args.collect: + d.collect_db() + else: + assert args.index is not None + assert args.imgnet is not None + d.make_db() + +if __name__ == '__main__': + main() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/create_database.sh b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/create_database.sh new file mode 100644 index 0000000..6352456 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/create_database.sh @@ -0,0 +1,15 @@ +#bash create_database.sh all predictor 0 49 + +IMGNET_PATH='/w14/dataset/ILSVRC2012' # PUT YOUR ILSVRC2012 DIR + +for ((ind=$2;ind<=$3;ind++)) +do + python build_database.py --gpu $1 \ + --model_name $4 \ + --index $ind \ + --imgnet $IMGNET_PATH \ + --hs 512 \ + --nz 56 +done + + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/__init__.py new file mode 100644 index 0000000..765d442 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/__init__.py @@ -0,0 +1,5 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from .db_ofa import DatabaseOFA diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/base_provider.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/base_provider.py new file mode 100644 index 0000000..8861c63 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/base_provider.py @@ -0,0 +1,57 @@ +###################################################################################### +# Copyright (c) Han Cai, Once for All, ICLR 2020 [GitHub OFA] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### + +import numpy as np +import torch + +__all__ = ['DataProvider'] + + +class DataProvider: + SUB_SEED = 937162211 # random seed for sampling subset + VALID_SEED = 2147483647 # random seed for the validation set + + @staticmethod + def name(): + """ Return name of the dataset """ + raise NotImplementedError + + @property + def data_shape(self): + """ Return shape as python list of one data entry """ + raise NotImplementedError + + @property + def n_classes(self): + """ Return `int` of num classes """ + raise NotImplementedError + + @property + def save_path(self): + """ local path to save the data """ + raise NotImplementedError + + @property + def data_url(self): + """ link to download the data """ + raise NotImplementedError + + @staticmethod + def random_sample_valid_set(train_size, valid_size): + assert train_size > valid_size + + g = torch.Generator() + g.manual_seed(DataProvider.VALID_SEED) # set random seed before sampling validation set + rand_indexes = torch.randperm(train_size, generator=g).tolist() + + valid_indexes = rand_indexes[:valid_size] + train_indexes = rand_indexes[valid_size:] + return train_indexes, valid_indexes + + @staticmethod + def labels_to_one_hot(n_classes, labels): + new_labels = np.zeros((labels.shape[0], n_classes), dtype=np.float32) + new_labels[range(labels.shape[0]), labels] = np.ones(labels.shape) + return new_labels diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/db_ofa.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/db_ofa.py new file mode 100644 index 0000000..cd7c56a --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/db_ofa.py @@ -0,0 +1,107 @@ +import os +import torch +import time +import copy +import glob +from .imagenet import ImagenetDataProvider +from .imagenet_loader import ImagenetRunConfig +from .run_manager import RunManager +from ofa.model_zoo import ofa_net + + +class DatabaseOFA: + def __init__(self, args, predictor=None): + self.path = f'{args.data_path}/{args.model_name}' + self.model_name = args.model_name + self.index = args.index + self.args = args + self.predictor = predictor + ImagenetDataProvider.DEFAULT_PATH = args.imgnet + + if not os.path.exists(self.path): + os.makedirs(self.path) + + def make_db(self): + self.ofa_network = ofa_net('ofa_mbv3_d234_e346_k357_w1.0', pretrained=True) + self.run_config = ImagenetRunConfig(test_batch_size=self.args.batch_size, + n_worker=20) + database = [] + st_time = time.time() + f = open(f'{self.path}/txt_{self.index}.txt', 'w') + for dn in range(10000): + best_pp = -1 + best_info = None + dls = None + with torch.no_grad(): + if self.model_name == 'generator': + for i in range(10): + net_setting = self.ofa_network.sample_active_subnet() + subnet = self.ofa_network.get_active_subnet(preserve_weight=True) + if i == 0: + run_manager = RunManager('.tmp/eval_subnet', self.args, subnet, + self.run_config, init=False, pp=self.predictor) + self.run_config.data_provider.assign_active_img_size(224) + dls = {j: copy.deepcopy(run_manager.data_loader) for j in range(1, 10)} + else: + run_manager = RunManager('.tmp/eval_subnet', self.args, subnet, + self.run_config, + init=False, data_loader=dls[i], pp=self.predictor) + run_manager.reset_running_statistics(net=subnet) + + loss, (top1, top5), pred_acc \ + = run_manager.validate(net=subnet, net_setting=net_setting) + + if best_pp < pred_acc: + best_pp = pred_acc + print('[%d] class=%d,\t loss=%.5f,\t top1=%.1f,\t top5=%.1f' % ( + dn, len(run_manager.cls_lst), loss, top1, top5)) + info_dict = {'loss': loss, + 'top1': top1, + 'top5': top5, + 'net': net_setting, + 'class': run_manager.cls_lst, + 'params': run_manager.net_info['params'], + 'flops': run_manager.net_info['flops'], + 'test_transform': run_manager.test_transform + } + best_info = info_dict + elif self.model_name == 'predictor': + net_setting = self.ofa_network.sample_active_subnet() + subnet = self.ofa_network.get_active_subnet(preserve_weight=True) + run_manager = RunManager('.tmp/eval_subnet', self.args, subnet, self.run_config, init=False) + self.run_config.data_provider.assign_active_img_size(224) + run_manager.reset_running_statistics(net=subnet) + + loss, (top1, top5), _ = run_manager.validate(net=subnet) + print('[%d] class=%d,\t loss=%.5f,\t top1=%.1f,\t top5=%.1f' % ( + dn, len(run_manager.cls_lst), loss, top1, top5)) + best_info = {'loss': loss, + 'top1': top1, + 'top5': top5, + 'net': net_setting, + 'class': run_manager.cls_lst, + 'params': run_manager.net_info['params'], + 'flops': run_manager.net_info['flops'], + 'test_transform': run_manager.test_transform + } + database.append(best_info) + if (len(database)) % 10 == 0: + msg = f'{(time.time() - st_time) / 60.0:0.2f}(min) save {len(database)} database, {self.index} id' + print(msg) + f.write(msg + '\n') + f.flush() + torch.save(database, f'{self.path}/database_{self.index}.pt') + + def collect_db(self): + if not os.path.exists(self.path + f'/processed'): + os.makedirs(self.path + f'/processed') + + database = [] + dlst = glob.glob(self.path + '/*.pt') + for filepath in dlst: + database += torch.load(filepath) + + assert len(database) != 0 + + print(f'The number of database: {len(database)}') + torch.save(database, self.path + f'/processed/collected_database.pt') diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/imagenet.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/imagenet.py new file mode 100644 index 0000000..552f23d --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/imagenet.py @@ -0,0 +1,240 @@ +###################################################################################### +# Copyright (c) Han Cai, Once for All, ICLR 2020 [GitHub OFA] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import warnings +import os +import torch +import math +import numpy as np +import torch.utils.data +import torchvision.transforms as transforms +import torchvision.datasets as datasets + +from ofa_local.imagenet_classification.data_providers.base_provider import DataProvider +from ofa_local.utils.my_dataloader import MyRandomResizedCrop, MyDistributedSampler +from .metaloader import MetaImageNetDataset, EpisodeSampler, MetaDataLoader + + +__all__ = ['ImagenetDataProvider'] + + +class ImagenetDataProvider(DataProvider): + DEFAULT_PATH = '/dataset/imagenet' + + def __init__(self, save_path=None, train_batch_size=256, test_batch_size=512, valid_size=None, n_worker=32, + resize_scale=0.08, distort_color=None, image_size=224, + num_replicas=None, rank=None): + warnings.filterwarnings('ignore') + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = 'None' if distort_color is None else distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + from ofa.utils.my_dataloader import MyDataLoader + assert isinstance(self.image_size, list) + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) # active resolution for test + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + + ########################## modification ######################## + train_dataset = self.train_dataset(self.build_train_transform()) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset), valid_size) + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, True, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, True, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + # test_dataset = self.test_dataset(valid_transforms) + test_dataset = self.meta_test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + # self.test = torch.utils.data.DataLoader( + # test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + # ) + sampler = EpisodeSampler( + max_way=1000, query=10, ylst=test_dataset.ylst) + self.test = MetaDataLoader(dataset=test_dataset, + sampler=sampler, + batch_size=test_batch_size, + shuffle=False, + num_workers=4) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'imagenet' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 1000 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = self.DEFAULT_PATH + if not os.path.exists(self._save_path): + self._save_path = os.path.expanduser('~/dataset/imagenet') + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + return datasets.ImageFolder(self.train_path, _transforms) + + def test_dataset(self, _transforms): + return datasets.ImageFolder(self.valid_path, _transforms) + + def meta_test_dataset(self, _transforms): + return MetaImageNetDataset('val', max_way=1000, query=10, + dpath='/w14/dataset/ILSVRC2012', transform=_transforms) + + @property + def train_path(self): + return os.path.join(self.save_path, 'train') + + @property + def valid_path(self): + return os.path.join(self.save_path, 'val') + + @property + def normalize(self): + return transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + def build_train_transform(self, image_size=None, print_log=True): + if image_size is None: + image_size = self.image_size + if print_log: + print('Color jitter: %s, resize_scale: %s, img_size: %s' % + (self.distort_color, self.resize_scale, image_size)) + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + else: + resize_transform_class = transforms.RandomResizedCrop + + # random_resize_crop -> random_horizontal_flip + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + + # color augmentation (optional) + color_transform = None + if self.distort_color == 'torch': + color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + elif self.distort_color == 'tf': + color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + if color_transform is not None: + train_transforms.append(color_transform) + + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting BN running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, True, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/imagenet_loader.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/imagenet_loader.py new file mode 100644 index 0000000..225f247 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/imagenet_loader.py @@ -0,0 +1,40 @@ +from .imagenet import ImagenetDataProvider +from ofa_local.imagenet_classification.run_manager import RunConfig + + +__all__ = ['ImagenetRunConfig'] + + +class ImagenetRunConfig(RunConfig): + + def __init__(self, n_epochs=150, init_lr=0.05, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='imagenet', train_batch_size=256, test_batch_size=500, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.1, no_decay_keys=None, + mixup_alpha=None, model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=32, resize_scale=0.08, distort_color='tf', image_size=224, **kwargs): + super(ImagenetRunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == ImagenetDataProvider.name(): + DataProviderClass = ImagenetDataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/metaloader.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/metaloader.py new file mode 100644 index 0000000..4c40c0e --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/metaloader.py @@ -0,0 +1,210 @@ +from torch.utils.data.sampler import Sampler +import os +import random +from PIL import Image +from collections import defaultdict +import torch +from torch.utils.data import Dataset, DataLoader +import glob + + +class RandCycleIter: + ''' + Return data_list per class + Shuffle the returning order after one epoch + ''' + def __init__ (self, data, shuffle=True): + self.data_list = list(data) + self.length = len(self.data_list) + self.i = self.length - 1 + self.shuffle = shuffle + + def __iter__ (self): + return self + + def __next__ (self): + self.i += 1 + + if self.i == self.length: + self.i = 0 + if self.shuffle: + random.shuffle(self.data_list) + + return self.data_list[self.i] + + +class EpisodeSampler(Sampler): + def __init__(self, max_way, query, ylst): + self.max_way = max_way + self.query = query + self.ylst = ylst + # self.n_epi = n_epi + + clswise_xidx = defaultdict(list) + for i, y in enumerate(ylst): + clswise_xidx[y].append(i) + self.cws_xidx_iter = [RandCycleIter(cxidx, shuffle=True) + for cxidx in clswise_xidx.values()] + self.n_cls = len(clswise_xidx) + + self.create_episode() + + + def __iter__ (self): + return self.get_index() + + def __len__ (self): + return self.get_len() + + def create_episode(self): + self.way = torch.randperm(int(self.max_way/10.0)-1)[0] * 10 + 10 + cls_lst = torch.sort(torch.randperm(self.max_way)[:self.way])[0] + self.cls_itr = iter(cls_lst) + self.cls_lst = cls_lst + + def get_len(self): + return self.way * self.query + + def get_index(self): + x_itr = self.cws_xidx_iter + + i, j = 0, 0 + while i < self.query * self.way: + if j >= self.query: + j = 0 + if j == 0: + cls_idx = next(self.cls_itr).item() + bb = [x_itr[cls_idx]] * self.query + didx = next(zip(*bb)) + yield didx[j] + # yield (didx[j], self.way) + + i += 1; j += 1 + + +class MetaImageNetDataset(Dataset): + def __init__(self, mode='val', + max_way=1000, query=10, + dpath='/w14/dataset/ILSVRC2012', transform=None): + self.dpath = dpath + self.transform = transform + self.mode = mode + + self.max_way = max_way + self.query = query + classes, class_to_idx = self._find_classes(dpath+'/'+mode) + self.classes, self.class_to_idx = classes, class_to_idx + # self.class_folder_lst = \ + # glob.glob(dpath+'/'+mode+'/*') + # ## sorting alphabetically + # self.class_folder_lst = sorted(self.class_folder_lst) + self.file_path_lst, self.ylst = [], [] + for cls in classes: + xlst = glob.glob(dpath+'/'+mode+'/'+cls+'/*') + self.file_path_lst += xlst[:self.query] + y = class_to_idx[cls] + self.ylst += [y] * len(xlst[:self.query]) + + # for y, cls in enumerate(self.class_folder_lst): + # xlst = glob.glob(cls+'/*') + # self.file_path_lst += xlst[:self.query] + # self.ylst += [y] * len(xlst[:self.query]) + # # self.file_path_lst += [xlst[_] for _ in + # # torch.randperm(len(xlst))[:self.query]] + # # self.ylst += [cls.split('/')[-1]] * len(xlst) + + self.way_idx = 0 + self.x_idx = 0 + self.way = 2 + self.cls_lst = None + + + def __len__(self): + return self.way * self.query + + def __getitem__(self, index): + # if self.way != index[1]: + # self.way = index[1] + # index = index[0] + + x = Image.open( + self.file_path_lst[index]).convert('RGB') + + if self.transform is not None: + x = self.transform(x) + cls_name = self.ylst[index] + y = self.cls_lst.index(cls_name) + # y = self.way_idx + # self.x_idx += 1 + # if self.x_idx == self.query: + # self.way_idx += 1 + # self.x_idx = 0 + # if self.way_idx == self.way: + # self.way_idx = 0 + # self.x_idx = 0 + return x, y #, cls_name # y # cls_name #y + + def _find_classes(self, dir: str): + """ + Finds the class folders in a dataset. + + Args: + dir (string): Root directory path. + + Returns: + tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary. + + Ensures: + No class is a subdirectory of another. + """ + classes = [d.name for d in os.scandir(dir) if d.is_dir()] + classes.sort() + class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)} + return classes, class_to_idx + + +class MetaDataLoader(DataLoader): + def __init__(self, + dataset, sampler, batch_size, shuffle, num_workers): + super(MetaDataLoader, self).__init__( + dataset=dataset, + sampler=sampler, + batch_size=batch_size, + shuffle=shuffle, + num_workers=num_workers) + + + def create_episode(self): + self.sampler.create_episode() + self.dataset.way = self.sampler.way + self.dataset.cls_lst = self.sampler.cls_lst.tolist() + + + def get_cls_idx(self): + return self.sampler.cls_lst + + +def get_loader(mode='val', way=10, query=10, + n_epi=100, dpath='/w14/dataset/ILSVRC2012', + transform=None): + trans = get_transforms(mode) + dataset = MetaImageNetDataset(mode, way, query, dpath, trans) + sampler = EpisodeSampler( + way, query, n_epi, dataset.ylst) + dataset.way = sampler.way + dataset.cls_lst = sampler.cls_lst + loader = MetaDataLoader(dataset=dataset, + sampler=sampler, + batch_size=10, + shuffle=False, + num_workers=4) + return loader + +# trloader = get_loader() + +# trloader.create_episode() +# print(len(trloader)) +# print(trloader.dataset.way) +# print(trloader.sampler.way) +# for i, episode in enumerate(trloader, start=1): +# print(episode[2]) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/run_manager.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/run_manager.py new file mode 100644 index 0000000..249423f --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/database/run_manager.py @@ -0,0 +1,302 @@ +###################################################################################### +# Copyright (c) Han Cai, Once for All, ICLR 2020 [GitHub OFA] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import os +import json +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.optim +from tqdm import tqdm +from utils import decode_ofa_mbv3_to_igraph +from ofa_local.utils import get_net_info, cross_entropy_loss_with_soft_target, cross_entropy_with_label_smoothing +from ofa_local.utils import AverageMeter, accuracy, write_log, mix_images, mix_labels, init_models + +__all__ = ['RunManager'] +import torchvision.models as models + + +class RunManager: + + def __init__(self, path, args, net, run_config, init=True, measure_latency=None, + no_gpu=False, data_loader=None, pp=None): + self.path = path + self.mode = args.model_name + self.net = net + self.run_config = run_config + + self.best_acc = 0 + self.start_epoch = 0 + + os.makedirs(self.path, exist_ok=True) + # dataloader + if data_loader is not None: + self.data_loader = data_loader + cls_lst = self.data_loader.get_cls_idx() + self.cls_lst = cls_lst + else: + self.data_loader = self.run_config.valid_loader + self.data_loader.create_episode() + cls_lst = self.data_loader.get_cls_idx() + self.cls_lst = cls_lst + + state_dict = self.net.classifier.state_dict() + new_state_dict = {'weight': state_dict['linear.weight'][cls_lst], + 'bias': state_dict['linear.bias'][cls_lst]} + + self.net.classifier = nn.Linear(1280, len(cls_lst), bias=True) + self.net.classifier.load_state_dict(new_state_dict) + + # move network to GPU if available + if torch.cuda.is_available() and (not no_gpu): + self.device = torch.device('cuda:0') + self.net = self.net.to(self.device) + cudnn.benchmark = True + else: + self.device = torch.device('cpu') + + # net info + net_info = get_net_info( + self.net, self.run_config.data_provider.data_shape, measure_latency, False) + self.net_info = net_info + self.test_transform = self.run_config.data_provider.test.dataset.transform + + # criterion + if isinstance(self.run_config.mixup_alpha, float): + self.train_criterion = cross_entropy_loss_with_soft_target + elif self.run_config.label_smoothing > 0: + self.train_criterion = \ + lambda pred, target: cross_entropy_with_label_smoothing(pred, target, self.run_config.label_smoothing) + else: + self.train_criterion = nn.CrossEntropyLoss() + self.test_criterion = nn.CrossEntropyLoss() + + # optimizer + if self.run_config.no_decay_keys: + keys = self.run_config.no_decay_keys.split('#') + net_params = [ + self.network.get_parameters(keys, mode='exclude'), # parameters with weight decay + self.network.get_parameters(keys, mode='include'), # parameters without weight decay + ] + else: + # noinspection PyBroadException + try: + net_params = self.network.weight_parameters() + except Exception: + net_params = [] + for param in self.network.parameters(): + if param.requires_grad: + net_params.append(param) + self.optimizer = self.run_config.build_optimizer(net_params) + + self.net = torch.nn.DataParallel(self.net) + + if self.mode == 'generator': + # PP + save_dir = f'{args.save_path}/predictor/model/ckpt_max_corr.pt' + + self.acc_predictor = pp.to('cuda') + self.acc_predictor.load_state_dict(torch.load(save_dir)) + self.acc_predictor = torch.nn.DataParallel(self.acc_predictor) + model = models.resnet18(pretrained=True).eval() + feature_extractor = torch.nn.Sequential(*list(model.children())[:-1]).to(self.device) + self.feature_extractor = torch.nn.DataParallel(feature_extractor) + + """ save path and log path """ + + @property + def save_path(self): + if self.__dict__.get('_save_path', None) is None: + save_path = os.path.join(self.path, 'checkpoint') + os.makedirs(save_path, exist_ok=True) + self.__dict__['_save_path'] = save_path + return self.__dict__['_save_path'] + + @property + def logs_path(self): + if self.__dict__.get('_logs_path', None) is None: + logs_path = os.path.join(self.path, 'logs') + os.makedirs(logs_path, exist_ok=True) + self.__dict__['_logs_path'] = logs_path + return self.__dict__['_logs_path'] + + @property + def network(self): + return self.net.module if isinstance(self.net, nn.DataParallel) else self.net + + def write_log(self, log_str, prefix='valid', should_print=True, mode='a'): + write_log(self.logs_path, log_str, prefix, should_print, mode) + + """ save and load models """ + + def save_model(self, checkpoint=None, is_best=False, model_name=None): + if checkpoint is None: + checkpoint = {'state_dict': self.network.state_dict()} + + if model_name is None: + model_name = 'checkpoint.pth.tar' + + checkpoint['dataset'] = self.run_config.dataset # add `dataset` info to the checkpoint + latest_fname = os.path.join(self.save_path, 'latest.txt') + model_path = os.path.join(self.save_path, model_name) + with open(latest_fname, 'w') as fout: + fout.write(model_path + '\n') + torch.save(checkpoint, model_path) + + if is_best: + best_path = os.path.join(self.save_path, 'model_best.pth.tar') + torch.save({'state_dict': checkpoint['state_dict']}, best_path) + + def load_model(self, model_fname=None): + latest_fname = os.path.join(self.save_path, 'latest.txt') + if model_fname is None and os.path.exists(latest_fname): + with open(latest_fname, 'r') as fin: + model_fname = fin.readline() + if model_fname[-1] == '\n': + model_fname = model_fname[:-1] + # noinspection PyBroadException + try: + if model_fname is None or not os.path.exists(model_fname): + model_fname = '%s/checkpoint.pth.tar' % self.save_path + with open(latest_fname, 'w') as fout: + fout.write(model_fname + '\n') + print("=> loading checkpoint '{}'".format(model_fname)) + checkpoint = torch.load(model_fname, map_location='cpu') + except Exception: + print('fail to load checkpoint from %s' % self.save_path) + return {} + + self.network.load_state_dict(checkpoint['state_dict']) + if 'epoch' in checkpoint: + self.start_epoch = checkpoint['epoch'] + 1 + if 'best_acc' in checkpoint: + self.best_acc = checkpoint['best_acc'] + if 'optimizer' in checkpoint: + self.optimizer.load_state_dict(checkpoint['optimizer']) + + print("=> loaded checkpoint '{}'".format(model_fname)) + return checkpoint + + def save_config(self, extra_run_config=None, extra_net_config=None): + """ dump run_config and net_config to the model_folder """ + run_save_path = os.path.join(self.path, 'run.config') + if not os.path.isfile(run_save_path): + run_config = self.run_config.config + if extra_run_config is not None: + run_config.update(extra_run_config) + json.dump(run_config, open(run_save_path, 'w'), indent=4) + print('Run configs dump to %s' % run_save_path) + + try: + net_save_path = os.path.join(self.path, 'net.config') + net_config = self.network.config + if extra_net_config is not None: + net_config.update(extra_net_config) + json.dump(net_config, open(net_save_path, 'w'), indent=4) + print('Network configs dump to %s' % net_save_path) + except Exception: + print('%s do not support net config' % type(self.network)) + + """ metric related """ + + def get_metric_dict(self): + return { + 'top1': AverageMeter(), + 'top5': AverageMeter(), + } + + def update_metric(self, metric_dict, output, labels): + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + metric_dict['top1'].update(acc1[0].item(), output.size(0)) + metric_dict['top5'].update(acc5[0].item(), output.size(0)) + + def get_metric_vals(self, metric_dict, return_dict=False): + if return_dict: + return { + key: metric_dict[key].avg for key in metric_dict + } + else: + return [metric_dict[key].avg for key in metric_dict] + + def get_metric_names(self): + return 'top1', 'top5' + + """ train and test """ + def validate(self, epoch=0, is_test=False, run_str='', net=None, + data_loader=None, no_logs=False, train_mode=False, net_setting=None): + if net is None: + net = self.net + if not isinstance(net, nn.DataParallel): + net = nn.DataParallel(net) + + if data_loader is not None: + self.data_loader = data_loader + + if train_mode: + net.train() + else: + net.eval() + + losses = AverageMeter() + metric_dict = self.get_metric_dict() + + features_stack = [] + with torch.no_grad(): + with tqdm(total=len(self.data_loader), + desc='Validate Epoch #{} {}'.format(epoch + 1, run_str), disable=no_logs) as t: + for i, (images, labels) in enumerate(self.data_loader): + images, labels = images.to(self.device), labels.to(self.device) + if self.mode == 'generator': + features = self.feature_extractor(images).squeeze() + features_stack.append(features) + # compute output + output = net(images) + loss = self.test_criterion(output, labels) + # measure accuracy and record loss + self.update_metric(metric_dict, output, labels) + + losses.update(loss.item(), images.size(0)) + t.set_postfix({ + 'loss': losses.avg, + **self.get_metric_vals(metric_dict, return_dict=True), + 'img_size': images.size(2), + }) + t.update(1) + + if self.mode == 'generator': + features_stack = torch.cat(features_stack) + igraph_g = decode_ofa_mbv3_to_igraph(net_setting)[0] + D_mu = self.acc_predictor.module.set_encode(features_stack.unsqueeze(0).to('cuda')) + G_mu = self.acc_predictor.module.graph_encode(igraph_g) + pred_acc = self.acc_predictor.module.predict(D_mu.unsqueeze(0), G_mu).item() + + return losses.avg, self.get_metric_vals(metric_dict), \ + pred_acc if self.mode == 'generator' else None + + + def validate_all_resolution(self, epoch=0, is_test=False, net=None): + if net is None: + net = self.network + if isinstance(self.run_config.data_provider.image_size, list): + img_size_list, loss_list, top1_list, top5_list = [], [], [], [] + for img_size in self.run_config.data_provider.image_size: + img_size_list.append(img_size) + self.run_config.data_provider.assign_active_img_size(img_size) + self.reset_running_statistics(net=net) + loss, (top1, top5) = self.validate(epoch, is_test, net=net) + loss_list.append(loss) + top1_list.append(top1) + top5_list.append(top5) + return img_size_list, loss_list, top1_list, top5_list + else: + loss, (top1, top5) = self.validate(epoch, is_test, net=net) + return [self.run_config.data_provider.active_img_size], [loss], [top1], [top5] + + def reset_running_statistics(self, net=None, subset_size=2000, subset_batch_size=200, data_loader=None): + from ofa_local.imagenet_classification.elastic_nn.utils import set_running_statistics + if net is None: + net = self.network + if data_loader is None: + data_loader = self.run_config.random_sub_train_loader(subset_size, subset_batch_size) + set_running_statistics(net, data_loader) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/__init__.py new file mode 100644 index 0000000..0c66d4c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/__init__.py @@ -0,0 +1,4 @@ +###################################################################################### +# Copyright (c) Han Cai, Once for All, ICLR 2020 [GitHub OFA] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/aircraft.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/aircraft.py new file mode 100644 index 0000000..6c0dd89 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/aircraft.py @@ -0,0 +1,401 @@ +from __future__ import print_function + +import os +import math +import warnings +import numpy as np + +# from timm.data.transforms import _pil_interp +from timm.data.auto_augment import rand_augment_transform + +import torch.utils.data +import torchvision.transforms as transforms +from torchvision.datasets.folder import default_loader + +from ofa.imagenet_codebase.data_providers.base_provider import DataProvider, MyRandomResizedCrop, MyDistributedSampler + + +def make_dataset(dir, image_ids, targets): + assert(len(image_ids) == len(targets)) + images = [] + dir = os.path.expanduser(dir) + for i in range(len(image_ids)): + item = (os.path.join(dir, 'data', 'images', + '%s.jpg' % image_ids[i]), targets[i]) + images.append(item) + return images + + +def find_classes(classes_file): + # read classes file, separating out image IDs and class names + image_ids = [] + targets = [] + f = open(classes_file, 'r') + for line in f: + split_line = line.split(' ') + image_ids.append(split_line[0]) + targets.append(' '.join(split_line[1:])) + f.close() + + # index class names + classes = np.unique(targets) + class_to_idx = {classes[i]: i for i in range(len(classes))} + targets = [class_to_idx[c] for c in targets] + + return (image_ids, targets, classes, class_to_idx) + + +class FGVCAircraft(torch.utils.data.Dataset): + """`FGVC-Aircraft `_ Dataset. + Args: + root (string): Root directory path to dataset. + class_type (string, optional): The level of FGVC-Aircraft fine-grain classification + to label data with (i.e., ``variant``, ``family``, or ``manufacturer``). + transform (callable, optional): A function/transform that takes in a PIL image + and returns a transformed version. E.g. ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + loader (callable, optional): A function to load an image given its path. + download (bool, optional): If true, downloads the dataset from the internet and + puts it in the root directory. If dataset is already downloaded, it is not + downloaded again. + """ + url = 'http://www.robots.ox.ac.uk/~vgg/data/fgvc-aircraft/archives/fgvc-aircraft-2013b.tar.gz' + class_types = ('variant', 'family', 'manufacturer') + splits = ('train', 'val', 'trainval', 'test') + + def __init__(self, root, class_type='variant', split='train', transform=None, + target_transform=None, loader=default_loader, download=False): + if split not in self.splits: + raise ValueError('Split "{}" not found. Valid splits are: {}'.format( + split, ', '.join(self.splits), + )) + if class_type not in self.class_types: + raise ValueError('Class type "{}" not found. Valid class types are: {}'.format( + class_type, ', '.join(self.class_types), + )) + self.root = os.path.expanduser(root) + self.class_type = class_type + self.split = split + self.classes_file = os.path.join(self.root, 'data', + 'images_%s_%s.txt' % (self.class_type, self.split)) + + if download: + self.download() + + (image_ids, targets, classes, class_to_idx) = find_classes(self.classes_file) + samples = make_dataset(self.root, image_ids, targets) + + self.transform = transform + self.target_transform = target_transform + self.loader = loader + + self.samples = samples + self.classes = classes + self.class_to_idx = class_to_idx + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + + path, target = self.samples[index] + sample = self.loader(path) + if self.transform is not None: + sample = self.transform(sample) + if self.target_transform is not None: + target = self.target_transform(target) + + return sample, target + + def __len__(self): + return len(self.samples) + + def __repr__(self): + fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Root Location: {}\n'.format(self.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + def _check_exists(self): + return os.path.exists(os.path.join(self.root, 'data', 'images')) and \ + os.path.exists(self.classes_file) + + def download(self): + """Download the FGVC-Aircraft data if it doesn't exist already.""" + from six.moves import urllib + import tarfile + + if self._check_exists(): + return + + # prepare to download data to PARENT_DIR/fgvc-aircraft-2013.tar.gz + print('Downloading %s ... (may take a few minutes)' % self.url) + + parent_dir = os.path.abspath(os.path.join(self.root, os.pardir)) + tar_name = self.url.rpartition('/')[-1] + tar_path = os.path.join(parent_dir, tar_name) + data = urllib.request.urlopen(self.url) + + # download .tar.gz file + with open(tar_path, 'wb') as f: + f.write(data.read()) + + # extract .tar.gz to PARENT_DIR/fgvc-aircraft-2013b + data_folder = tar_path.strip('.tar.gz') + print('Extracting %s to %s ... (may take a few minutes)' % (tar_path, data_folder)) + tar = tarfile.open(tar_path) + tar.extractall(parent_dir) + + # if necessary, rename data folder to self.root + if not os.path.samefile(data_folder, self.root): + print('Renaming %s to %s ...' % (data_folder, self.root)) + os.rename(data_folder, self.root) + + # delete .tar.gz file + print('Deleting %s ...' % tar_path) + os.remove(tar_path) + + print('Done!') + + +class FGVCAircraftDataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=32, test_batch_size=200, valid_size=None, n_worker=32, + resize_scale=0.08, distort_color=None, image_size=224, + num_replicas=None, rank=None): + + warnings.filterwarnings('ignore') + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset.samples) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset.samples), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'aircraft' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 100 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = '/mnt/datastore/Aircraft' # home server + + if not os.path.exists(self._save_path): + self._save_path = '/mnt/datastore/Aircraft' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.train_path, _transforms) + dataset = FGVCAircraft( + root=self.train_path, split='trainval', download=True, transform=_transforms) + return dataset + + def test_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.valid_path, _transforms) + dataset = FGVCAircraft( + root=self.valid_path, split='test', download=True, transform=_transforms) + return dataset + + @property + def train_path(self): + return self.save_path + + @property + def valid_path(self): + return self.save_path + + @property + def normalize(self): + return transforms.Normalize( + mean=[0.48933587508932375, 0.5183537408957618, 0.5387914411673883], + std=[0.22388883112804625, 0.21641635409388751, 0.24615605842636115]) + + def build_train_transform(self, image_size=None, print_log=True, auto_augment='rand-m9-mstd0.5'): + if image_size is None: + image_size = self.image_size + # if print_log: + # print('Color jitter: %s, resize_scale: %s, img_size: %s' % + # (self.distort_color, self.resize_scale, image_size)) + + # if self.distort_color == 'torch': + # color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + # elif self.distort_color == 'tf': + # color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + # else: + # color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + img_size_min = min(image_size) + else: + resize_transform_class = transforms.RandomResizedCrop + img_size_min = image_size + + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + + aa_params = dict( + translate_const=int(img_size_min * 0.45), + img_mean=tuple([min(255, round(255 * x)) for x in [0.48933587508932375, 0.5183537408957618, + 0.5387914411673883]]), + ) + aa_params['interpolation'] = transforms.Resize(image_size) # _pil_interp('bicubic') + train_transforms += [rand_augment_transform(auto_augment, aa_params)] + + # if color_transform is not None: + # train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.samples) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] + + +if __name__ == '__main__': + data = FGVCAircraft(root='/mnt/datastore/Aircraft', + split='trainval', download=True) + print(len(data.classes)) + print(len(data.samples)) \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/autoaugment.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/autoaugment.py new file mode 100644 index 0000000..5729792 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/autoaugment.py @@ -0,0 +1,238 @@ +""" +Taken from https://github.com/DeepVoltaire/AutoAugment/blob/master/autoaugment.py +""" + +from PIL import Image, ImageEnhance, ImageOps +import numpy as np +import random + + +class ImageNetPolicy(object): + """ Randomly choose one of the best 24 Sub-policies on ImageNet. + + Example: + >>> policy = ImageNetPolicy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> ImageNetPolicy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.4, "posterize", 8, 0.6, "rotate", 9, fillcolor), + SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), + SubPolicy(0.8, "equalize", 8, 0.6, "equalize", 3, fillcolor), + SubPolicy(0.6, "posterize", 7, 0.6, "posterize", 6, fillcolor), + SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), + + SubPolicy(0.4, "equalize", 4, 0.8, "rotate", 8, fillcolor), + SubPolicy(0.6, "solarize", 3, 0.6, "equalize", 7, fillcolor), + SubPolicy(0.8, "posterize", 5, 1.0, "equalize", 2, fillcolor), + SubPolicy(0.2, "rotate", 3, 0.6, "solarize", 8, fillcolor), + SubPolicy(0.6, "equalize", 8, 0.4, "posterize", 6, fillcolor), + + SubPolicy(0.8, "rotate", 8, 0.4, "color", 0, fillcolor), + SubPolicy(0.4, "rotate", 9, 0.6, "equalize", 2, fillcolor), + SubPolicy(0.0, "equalize", 7, 0.8, "equalize", 8, fillcolor), + SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), + SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor), + + SubPolicy(0.8, "rotate", 8, 1.0, "color", 2, fillcolor), + SubPolicy(0.8, "color", 8, 0.8, "solarize", 7, fillcolor), + SubPolicy(0.4, "sharpness", 7, 0.6, "invert", 8, fillcolor), + SubPolicy(0.6, "shearX", 5, 1.0, "equalize", 9, fillcolor), + SubPolicy(0.4, "color", 0, 0.6, "equalize", 3, fillcolor), + + SubPolicy(0.4, "equalize", 7, 0.2, "solarize", 4, fillcolor), + SubPolicy(0.6, "solarize", 5, 0.6, "autocontrast", 5, fillcolor), + SubPolicy(0.6, "invert", 4, 1.0, "equalize", 8, fillcolor), + SubPolicy(0.6, "color", 4, 1.0, "contrast", 8, fillcolor), + SubPolicy(0.8, "equalize", 8, 0.6, "equalize", 3, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment ImageNet Policy" + + +class CIFAR10Policy(object): + """ Randomly choose one of the best 25 Sub-policies on CIFAR10. + + Example: + >>> policy = CIFAR10Policy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> CIFAR10Policy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.1, "invert", 7, 0.2, "contrast", 6, fillcolor), + SubPolicy(0.7, "rotate", 2, 0.3, "translateX", 9, fillcolor), + SubPolicy(0.8, "sharpness", 1, 0.9, "sharpness", 3, fillcolor), + SubPolicy(0.5, "shearY", 8, 0.7, "translateY", 9, fillcolor), + SubPolicy(0.5, "autocontrast", 8, 0.9, "equalize", 2, fillcolor), + + SubPolicy(0.2, "shearY", 7, 0.3, "posterize", 7, fillcolor), + SubPolicy(0.4, "color", 3, 0.6, "brightness", 7, fillcolor), + SubPolicy(0.3, "sharpness", 9, 0.7, "brightness", 9, fillcolor), + SubPolicy(0.6, "equalize", 5, 0.5, "equalize", 1, fillcolor), + SubPolicy(0.6, "contrast", 7, 0.6, "sharpness", 5, fillcolor), + + SubPolicy(0.7, "color", 7, 0.5, "translateX", 8, fillcolor), + SubPolicy(0.3, "equalize", 7, 0.4, "autocontrast", 8, fillcolor), + SubPolicy(0.4, "translateY", 3, 0.2, "sharpness", 6, fillcolor), + SubPolicy(0.9, "brightness", 6, 0.2, "color", 8, fillcolor), + SubPolicy(0.5, "solarize", 2, 0.0, "invert", 3, fillcolor), + + SubPolicy(0.2, "equalize", 0, 0.6, "autocontrast", 0, fillcolor), + SubPolicy(0.2, "equalize", 8, 0.6, "equalize", 4, fillcolor), + SubPolicy(0.9, "color", 9, 0.6, "equalize", 6, fillcolor), + SubPolicy(0.8, "autocontrast", 4, 0.2, "solarize", 8, fillcolor), + SubPolicy(0.1, "brightness", 3, 0.7, "color", 0, fillcolor), + + SubPolicy(0.4, "solarize", 5, 0.9, "autocontrast", 3, fillcolor), + SubPolicy(0.9, "translateY", 9, 0.7, "translateY", 9, fillcolor), + SubPolicy(0.9, "autocontrast", 2, 0.8, "solarize", 3, fillcolor), + SubPolicy(0.8, "equalize", 8, 0.1, "invert", 3, fillcolor), + SubPolicy(0.7, "translateY", 9, 0.9, "autocontrast", 1, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment CIFAR10 Policy" + + +class SVHNPolicy(object): + """ Randomly choose one of the best 25 Sub-policies on SVHN. + + Example: + >>> policy = SVHNPolicy() + >>> transformed = policy(image) + + Example as a PyTorch Transform: + >>> transform=transforms.Compose([ + >>> transforms.Resize(256), + >>> SVHNPolicy(), + >>> transforms.ToTensor()]) + """ + def __init__(self, fillcolor=(128, 128, 128)): + self.policies = [ + SubPolicy(0.9, "shearX", 4, 0.2, "invert", 3, fillcolor), + SubPolicy(0.9, "shearY", 8, 0.7, "invert", 5, fillcolor), + SubPolicy(0.6, "equalize", 5, 0.6, "solarize", 6, fillcolor), + SubPolicy(0.9, "invert", 3, 0.6, "equalize", 3, fillcolor), + SubPolicy(0.6, "equalize", 1, 0.9, "rotate", 3, fillcolor), + + SubPolicy(0.9, "shearX", 4, 0.8, "autocontrast", 3, fillcolor), + SubPolicy(0.9, "shearY", 8, 0.4, "invert", 5, fillcolor), + SubPolicy(0.9, "shearY", 5, 0.2, "solarize", 6, fillcolor), + SubPolicy(0.9, "invert", 6, 0.8, "autocontrast", 1, fillcolor), + SubPolicy(0.6, "equalize", 3, 0.9, "rotate", 3, fillcolor), + + SubPolicy(0.9, "shearX", 4, 0.3, "solarize", 3, fillcolor), + SubPolicy(0.8, "shearY", 8, 0.7, "invert", 4, fillcolor), + SubPolicy(0.9, "equalize", 5, 0.6, "translateY", 6, fillcolor), + SubPolicy(0.9, "invert", 4, 0.6, "equalize", 7, fillcolor), + SubPolicy(0.3, "contrast", 3, 0.8, "rotate", 4, fillcolor), + + SubPolicy(0.8, "invert", 5, 0.0, "translateY", 2, fillcolor), + SubPolicy(0.7, "shearY", 6, 0.4, "solarize", 8, fillcolor), + SubPolicy(0.6, "invert", 4, 0.8, "rotate", 4, fillcolor), + SubPolicy(0.3, "shearY", 7, 0.9, "translateX", 3, fillcolor), + SubPolicy(0.1, "shearX", 6, 0.6, "invert", 5, fillcolor), + + SubPolicy(0.7, "solarize", 2, 0.6, "translateY", 7, fillcolor), + SubPolicy(0.8, "shearY", 4, 0.8, "invert", 8, fillcolor), + SubPolicy(0.7, "shearX", 9, 0.8, "translateY", 3, fillcolor), + SubPolicy(0.8, "shearY", 5, 0.7, "autocontrast", 3, fillcolor), + SubPolicy(0.7, "shearX", 2, 0.1, "invert", 5, fillcolor) + ] + + + def __call__(self, img): + policy_idx = random.randint(0, len(self.policies) - 1) + return self.policies[policy_idx](img) + + def __repr__(self): + return "AutoAugment SVHN Policy" + + +class SubPolicy(object): + def __init__(self, p1, operation1, magnitude_idx1, p2, operation2, magnitude_idx2, fillcolor=(128, 128, 128)): + ranges = { + "shearX": np.linspace(0, 0.3, 10), + "shearY": np.linspace(0, 0.3, 10), + "translateX": np.linspace(0, 150 / 331, 10), + "translateY": np.linspace(0, 150 / 331, 10), + "rotate": np.linspace(0, 30, 10), + "color": np.linspace(0.0, 0.9, 10), + "posterize": np.round(np.linspace(8, 4, 10), 0).astype(np.int), + "solarize": np.linspace(256, 0, 10), + "contrast": np.linspace(0.0, 0.9, 10), + "sharpness": np.linspace(0.0, 0.9, 10), + "brightness": np.linspace(0.0, 0.9, 10), + "autocontrast": [0] * 10, + "equalize": [0] * 10, + "invert": [0] * 10 + } + + # from https://stackoverflow.com/questions/5252170/specify-image-filling-color-when-rotating-in-python-with-pil-and-setting-expand + def rotate_with_fill(img, magnitude): + rot = img.convert("RGBA").rotate(magnitude) + return Image.composite(rot, Image.new("RGBA", rot.size, (128,) * 4), rot).convert(img.mode) + + func = { + "shearX": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0), + Image.BICUBIC, fillcolor=fillcolor), + "shearY": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0), + Image.BICUBIC, fillcolor=fillcolor), + "translateX": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, magnitude * img.size[0] * random.choice([-1, 1]), 0, 1, 0), + fillcolor=fillcolor), + "translateY": lambda img, magnitude: img.transform( + img.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * img.size[1] * random.choice([-1, 1])), + fillcolor=fillcolor), + "rotate": lambda img, magnitude: rotate_with_fill(img, magnitude), + "color": lambda img, magnitude: ImageEnhance.Color(img).enhance(1 + magnitude * random.choice([-1, 1])), + "posterize": lambda img, magnitude: ImageOps.posterize(img, magnitude), + "solarize": lambda img, magnitude: ImageOps.solarize(img, magnitude), + "contrast": lambda img, magnitude: ImageEnhance.Contrast(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "sharpness": lambda img, magnitude: ImageEnhance.Sharpness(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "brightness": lambda img, magnitude: ImageEnhance.Brightness(img).enhance( + 1 + magnitude * random.choice([-1, 1])), + "autocontrast": lambda img, magnitude: ImageOps.autocontrast(img), + "equalize": lambda img, magnitude: ImageOps.equalize(img), + "invert": lambda img, magnitude: ImageOps.invert(img) + } + + self.p1 = p1 + self.operation1 = func[operation1] + self.magnitude1 = ranges[operation1][magnitude_idx1] + self.p2 = p2 + self.operation2 = func[operation2] + self.magnitude2 = ranges[operation2][magnitude_idx2] + + + def __call__(self, img): + if random.random() < self.p1: img = self.operation1(img, self.magnitude1) + if random.random() < self.p2: img = self.operation2(img, self.magnitude2) + return img \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/cifar.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/cifar.py new file mode 100644 index 0000000..e879935 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/cifar.py @@ -0,0 +1,657 @@ +import os +import math +import numpy as np + +import torchvision +import torch.utils.data +import torchvision.transforms as transforms + +from ofa.imagenet_codebase.data_providers.base_provider import DataProvider, MyRandomResizedCrop, MyDistributedSampler + + +class CIFAR10DataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=96, test_batch_size=256, valid_size=None, + n_worker=2, resize_scale=0.08, distort_color=None, image_size=224, num_replicas=None, rank=None): + + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset.data) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset.data), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'cifar10' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 10 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = '/mnt/datastore/CIFAR' # home server + + if not os.path.exists(self._save_path): + self._save_path = '/mnt/datastore/CIFAR' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.train_path, _transforms) + dataset = torchvision.datasets.CIFAR10( + root=self.valid_path, train=True, download=False, transform=_transforms) + return dataset + + def test_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.valid_path, _transforms) + dataset = torchvision.datasets.CIFAR10( + root=self.valid_path, train=False, download=False, transform=_transforms) + return dataset + + @property + def train_path(self): + # return os.path.join(self.save_path, 'train') + return self.save_path + + @property + def valid_path(self): + # return os.path.join(self.save_path, 'val') + return self.save_path + + @property + def normalize(self): + return transforms.Normalize( + mean=[0.49139968, 0.48215827, 0.44653124], std=[0.24703233, 0.24348505, 0.26158768]) + + def build_train_transform(self, image_size=None, print_log=True): + if image_size is None: + image_size = self.image_size + if print_log: + print('Color jitter: %s, resize_scale: %s, img_size: %s' % + (self.distort_color, self.resize_scale, image_size)) + + if self.distort_color == 'torch': + color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + elif self.distort_color == 'tf': + color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + else: + color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + else: + resize_transform_class = transforms.RandomResizedCrop + + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + if color_transform is not None: + train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.data) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] + + +class CIFAR100DataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=96, test_batch_size=256, valid_size=None, + n_worker=2, resize_scale=0.08, distort_color=None, image_size=224, num_replicas=None, rank=None): + + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset.data) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset.data), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'cifar100' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 100 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = '/mnt/datastore/CIFAR' # home server + + if not os.path.exists(self._save_path): + self._save_path = '/mnt/datastore/CIFAR' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.train_path, _transforms) + dataset = torchvision.datasets.CIFAR100( + root=self.valid_path, train=True, download=False, transform=_transforms) + return dataset + + def test_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.valid_path, _transforms) + dataset = torchvision.datasets.CIFAR100( + root=self.valid_path, train=False, download=False, transform=_transforms) + return dataset + + @property + def train_path(self): + # return os.path.join(self.save_path, 'train') + return self.save_path + + @property + def valid_path(self): + # return os.path.join(self.save_path, 'val') + return self.save_path + + @property + def normalize(self): + return transforms.Normalize( + mean=[0.49139968, 0.48215827, 0.44653124], std=[0.24703233, 0.24348505, 0.26158768]) + + def build_train_transform(self, image_size=None, print_log=True): + if image_size is None: + image_size = self.image_size + if print_log: + print('Color jitter: %s, resize_scale: %s, img_size: %s' % + (self.distort_color, self.resize_scale, image_size)) + + if self.distort_color == 'torch': + color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + elif self.distort_color == 'tf': + color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + else: + color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + else: + resize_transform_class = transforms.RandomResizedCrop + + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + if color_transform is not None: + train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.data) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] + + +class CINIC10DataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=96, test_batch_size=256, valid_size=None, + n_worker=2, resize_scale=0.08, distort_color=None, image_size=224, num_replicas=None, rank=None): + + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset.data) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset.data), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'cinic10' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 10 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = '/mnt/datastore/CINIC10' # home server + + if not os.path.exists(self._save_path): + self._save_path = '/mnt/datastore/CINIC10' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + dataset = torchvision.datasets.ImageFolder(self.train_path, transform=_transforms) + # dataset = torchvision.datasets.CIFAR10( + # root=self.valid_path, train=True, download=False, transform=_transforms) + return dataset + + def test_dataset(self, _transforms): + dataset = torchvision.datasets.ImageFolder(self.valid_path, transform=_transforms) + # dataset = torchvision.datasets.CIFAR10( + # root=self.valid_path, train=False, download=False, transform=_transforms) + return dataset + + @property + def train_path(self): + return os.path.join(self.save_path, 'train_and_valid') + # return self.save_path + + @property + def valid_path(self): + return os.path.join(self.save_path, 'test') + # return self.save_path + + @property + def normalize(self): + return transforms.Normalize( + mean=[0.47889522, 0.47227842, 0.43047404], std=[0.24205776, 0.23828046, 0.25874835]) + + def build_train_transform(self, image_size=None, print_log=True): + if image_size is None: + image_size = self.image_size + if print_log: + print('Color jitter: %s, resize_scale: %s, img_size: %s' % + (self.distort_color, self.resize_scale, image_size)) + + if self.distort_color == 'torch': + color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + elif self.distort_color == 'tf': + color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + else: + color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + else: + resize_transform_class = transforms.RandomResizedCrop + + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + if color_transform is not None: + train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.samples) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/dtd.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/dtd.py new file mode 100644 index 0000000..1d2a06b --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/dtd.py @@ -0,0 +1,237 @@ +import os +import warnings +import numpy as np + +from timm.data.transforms import _pil_interp +from timm.data.auto_augment import rand_augment_transform + +import torch.utils.data +import torchvision.transforms as transforms +import torchvision.datasets as datasets + +from ofa.imagenet_codebase.data_providers.base_provider import DataProvider, MyRandomResizedCrop, MyDistributedSampler + + +class DTDDataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=32, test_batch_size=200, valid_size=None, n_worker=32, + resize_scale=0.08, distort_color=None, image_size=224, + num_replicas=None, rank=None): + + warnings.filterwarnings('ignore') + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset.samples) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset.samples), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'dtd' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 47 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = '/mnt/datastore/dtd' # home server + + if not os.path.exists(self._save_path): + self._save_path = '/mnt/datastore/dtd' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + dataset = datasets.ImageFolder(self.train_path, _transforms) + return dataset + + def test_dataset(self, _transforms): + dataset = datasets.ImageFolder(self.valid_path, _transforms) + return dataset + + @property + def train_path(self): + return os.path.join(self.save_path, 'train') + + @property + def valid_path(self): + return os.path.join(self.save_path, 'valid') + + @property + def normalize(self): + return transforms.Normalize( + mean=[0.5329876098715876, 0.474260843249454, 0.42627281899380676], + std=[0.26549755708788914, 0.25473554309855373, 0.2631728035662832]) + + def build_train_transform(self, image_size=None, print_log=True, auto_augment='rand-m9-mstd0.5'): + if image_size is None: + image_size = self.image_size + # if print_log: + # print('Color jitter: %s, resize_scale: %s, img_size: %s' % + # (self.distort_color, self.resize_scale, image_size)) + + # if self.distort_color == 'torch': + # color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + # elif self.distort_color == 'tf': + # color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + # else: + # color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + img_size_min = min(image_size) + else: + resize_transform_class = transforms.RandomResizedCrop + img_size_min = image_size + + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + + aa_params = dict( + translate_const=int(img_size_min * 0.45), + img_mean=tuple([min(255, round(255 * x)) for x in [0.5329876098715876, 0.474260843249454, + 0.42627281899380676]]), + ) + aa_params['interpolation'] = _pil_interp('bicubic') + train_transforms += [rand_augment_transform(auto_augment, aa_params)] + + # if color_transform is not None: + # train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + # transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.Resize((image_size, image_size), interpolation=3), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.samples) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/flowers102.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/flowers102.py new file mode 100644 index 0000000..bf5ab50 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/flowers102.py @@ -0,0 +1,241 @@ +import warnings +import os +import math +import numpy as np + +import PIL + +import torch.utils.data +import torchvision.transforms as transforms +import torchvision.datasets as datasets + +from ofa.imagenet_codebase.data_providers.base_provider import DataProvider, MyRandomResizedCrop, MyDistributedSampler + + +class Flowers102DataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=32, test_batch_size=512, valid_size=None, n_worker=32, + resize_scale=0.08, distort_color=None, image_size=224, + num_replicas=None, rank=None): + + # warnings.filterwarnings('ignore') + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + weights = self.make_weights_for_balanced_classes( + train_dataset.imgs, self.n_classes) + weights = torch.DoubleTensor(weights) + train_sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, len(weights)) + + if valid_size is not None: + raise NotImplementedError("validation dataset not yet implemented") + # valid_dataset = self.valid_dataset(valid_transforms) + + # self.train = train_loader_class( + # train_dataset, batch_size=train_batch_size, sampler=train_sampler, + # num_workers=n_worker, pin_memory=True) + # self.valid = torch.utils.data.DataLoader( + # valid_dataset, batch_size=test_batch_size, + # num_workers=n_worker, pin_memory=True) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'flowers102' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 102 + + @property + def save_path(self): + if self._save_path is None: + # self._save_path = '/mnt/datastore/Oxford102Flowers' # home server + self._save_path = '/mnt/datastore/Flowers102' # home server + + if not os.path.exists(self._save_path): + # self._save_path = '/mnt/datastore/Oxford102Flowers' # home server + self._save_path = '/mnt/datastore/Flowers102' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + dataset = datasets.ImageFolder(self.train_path, _transforms) + return dataset + + # def valid_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.valid_path, _transforms) + # return dataset + + def test_dataset(self, _transforms): + dataset = datasets.ImageFolder(self.test_path, _transforms) + return dataset + + @property + def train_path(self): + return os.path.join(self.save_path, 'train') + + # @property + # def valid_path(self): + # return os.path.join(self.save_path, 'train') + + @property + def test_path(self): + return os.path.join(self.save_path, 'test') + + @property + def normalize(self): + return transforms.Normalize( + mean=[0.5178361839861569, 0.4106749456881299, 0.32864167836880803], + std=[0.2972239085211309, 0.24976049135203868, 0.28533308036347665]) + + @staticmethod + def make_weights_for_balanced_classes(images, nclasses): + count = [0] * nclasses + + # Counts per label + for item in images: + count[item[1]] += 1 + + weight_per_class = [0.] * nclasses + + # Total number of images. + N = float(sum(count)) + + # super-sample the smaller classes. + for i in range(nclasses): + weight_per_class[i] = N / float(count[i]) + + weight = [0] * len(images) + + # Calculate a weight per image. + for idx, val in enumerate(images): + weight[idx] = weight_per_class[val[1]] + + return weight + + def build_train_transform(self, image_size=None, print_log=True): + if image_size is None: + image_size = self.image_size + if print_log: + print('Color jitter: %s, resize_scale: %s, img_size: %s' % + (self.distort_color, self.resize_scale, image_size)) + + if self.distort_color == 'torch': + color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + elif self.distort_color == 'tf': + color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + else: + color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + else: + resize_transform_class = transforms.RandomResizedCrop + + train_transforms = [ + transforms.RandomAffine( + 45, translate=(0.4, 0.4), scale=(0.75, 1.5), shear=None, resample=PIL.Image.BILINEAR, fillcolor=0), + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + # transforms.RandomHorizontalFlip(), + ] + if color_transform is not None: + train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.samples) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/imagenet.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/imagenet.py new file mode 100644 index 0000000..9dd7a5a --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/imagenet.py @@ -0,0 +1,225 @@ +import warnings +import os +import math +import numpy as np + +import torch.utils.data +import torchvision.transforms as transforms +import torchvision.datasets as datasets + +from ofa.imagenet_codebase.data_providers.base_provider import DataProvider, MyRandomResizedCrop, MyDistributedSampler + + +class ImagenetDataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=256, test_batch_size=512, valid_size=None, n_worker=32, + resize_scale=0.08, distort_color=None, image_size=224, + num_replicas=None, rank=None): + + warnings.filterwarnings('ignore') + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset.samples) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset.samples), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'imagenet' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 1000 + + @property + def save_path(self): + if self._save_path is None: + # self._save_path = '/dataset/imagenet' + # self._save_path = '/usr/local/soft/temp-datastore/ILSVRC2012' # servers + self._save_path = '/mnt/datastore/ILSVRC2012' # home server + + if not os.path.exists(self._save_path): + # self._save_path = os.path.expanduser('~/dataset/imagenet') + # self._save_path = os.path.expanduser('/usr/local/soft/temp-datastore/ILSVRC2012') + self._save_path = '/mnt/datastore/ILSVRC2012' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + dataset = datasets.ImageFolder(self.train_path, _transforms) + return dataset + + def test_dataset(self, _transforms): + dataset = datasets.ImageFolder(self.valid_path, _transforms) + return dataset + + @property + def train_path(self): + return os.path.join(self.save_path, 'train') + + @property + def valid_path(self): + return os.path.join(self.save_path, 'val') + + @property + def normalize(self): + return transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + def build_train_transform(self, image_size=None, print_log=True): + if image_size is None: + image_size = self.image_size + if print_log: + print('Color jitter: %s, resize_scale: %s, img_size: %s' % + (self.distort_color, self.resize_scale, image_size)) + + if self.distort_color == 'torch': + color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + elif self.distort_color == 'tf': + color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + else: + color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + else: + resize_transform_class = transforms.RandomResizedCrop + + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + if color_transform is not None: + train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.samples) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/pets.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/pets.py new file mode 100644 index 0000000..d908b36 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/pets.py @@ -0,0 +1,237 @@ +import os +import math +import warnings +import numpy as np + +# from timm.data.transforms import _pil_interp +from timm.data.auto_augment import rand_augment_transform + +import torch.utils.data +import torchvision.transforms as transforms +import torchvision.datasets as datasets + +from ofa.imagenet_codebase.data_providers.base_provider import DataProvider, MyRandomResizedCrop, MyDistributedSampler + + +class OxfordIIITPetsDataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=32, test_batch_size=200, valid_size=None, n_worker=32, + resize_scale=0.08, distort_color=None, image_size=224, + num_replicas=None, rank=None): + + warnings.filterwarnings('ignore') + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset.samples) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset.samples), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'pets' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 37 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = '/mnt/datastore/Oxford-IIITPets' # home server + + if not os.path.exists(self._save_path): + self._save_path = '/mnt/datastore/Oxford-IIITPets' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + dataset = datasets.ImageFolder(self.train_path, _transforms) + return dataset + + def test_dataset(self, _transforms): + dataset = datasets.ImageFolder(self.valid_path, _transforms) + return dataset + + @property + def train_path(self): + return os.path.join(self.save_path, 'train') + + @property + def valid_path(self): + return os.path.join(self.save_path, 'valid') + + @property + def normalize(self): + return transforms.Normalize( + mean=[0.4828895122298728, 0.4448394893850807, 0.39566558230789783], + std=[0.25925664613996574, 0.2532760018681693, 0.25981017205097917]) + + def build_train_transform(self, image_size=None, print_log=True, auto_augment='rand-m9-mstd0.5'): + if image_size is None: + image_size = self.image_size + # if print_log: + # print('Color jitter: %s, resize_scale: %s, img_size: %s' % + # (self.distort_color, self.resize_scale, image_size)) + + # if self.distort_color == 'torch': + # color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + # elif self.distort_color == 'tf': + # color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + # else: + # color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + img_size_min = min(image_size) + else: + resize_transform_class = transforms.RandomResizedCrop + img_size_min = image_size + + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + + aa_params = dict( + translate_const=int(img_size_min * 0.45), + img_mean=tuple([min(255, round(255 * x)) for x in [0.4828895122298728, 0.4448394893850807, + 0.39566558230789783]]), + ) + aa_params['interpolation'] = transforms.Resize(image_size) # _pil_interp('bicubic') + train_transforms += [rand_augment_transform(auto_augment, aa_params)] + + # if color_transform is not None: + # train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.samples) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/pets2.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/pets2.py new file mode 100644 index 0000000..8ba8cc3 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/pets2.py @@ -0,0 +1,69 @@ +import torch +from glob import glob +from torch.utils.data.dataset import Dataset +import os +from PIL import Image + + +def load_image(filename): + img = Image.open(filename) + img = img.convert('RGB') + return img + + +class PetDataset(Dataset): + def __init__(self, root, train=True, num_cl=37, val_split=0.15, transforms=None): + pt_name = os.path.join(root, '{}{}.pth'.format('train' if train else 'test', + int(100 * (1 - val_split)) if train else int( + 100 * val_split))) + if not os.path.exists(pt_name): + filenames = glob(os.path.join(root, 'images') + '/*.jpg') + classes = set() + + data = [] + labels = [] + + for image in filenames: + class_name = image.rsplit("/", 1)[1].rsplit('_', 1)[0] + classes.add(class_name) + img = load_image(image) + + data.append(img) + labels.append(class_name) + + # convert classnames to indices + class2idx = {cl: idx for idx, cl in enumerate(classes)} + labels = torch.Tensor(list(map(lambda x: class2idx[x], labels))).long() + data = list(zip(data, labels)) + + class_values = [[] for x in range(num_cl)] + + # create arrays for each class type + for d in data: + class_values[d[1].item()].append(d) + + train_data = [] + val_data = [] + + for class_dp in class_values: + split_idx = int(len(class_dp) * (1 - val_split)) + train_data += class_dp[:split_idx] + val_data += class_dp[split_idx:] + torch.save(train_data, os.path.join(root, 'train{}.pth'.format(int(100 * (1 - val_split))))) + torch.save(val_data, os.path.join(root, 'test{}.pth'.format(int(100 * val_split)))) + + self.data = torch.load(pt_name) + self.len = len(self.data) + self.transform = transforms + + def __getitem__(self, index): + img, label = self.data[index] + + if self.transform: + img = self.transform(img) + + return img, label + + def __len__(self): + return self.len + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/stl10.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/stl10.py new file mode 100644 index 0000000..7ad6189 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/data_providers/stl10.py @@ -0,0 +1,226 @@ +import os +import math +import numpy as np + +import torchvision +import torch.utils.data +import torchvision.transforms as transforms + +from ofa.imagenet_codebase.data_providers.base_provider import DataProvider, MyRandomResizedCrop, MyDistributedSampler + + +class STL10DataProvider(DataProvider): + + def __init__(self, save_path=None, train_batch_size=96, test_batch_size=256, valid_size=None, + n_worker=2, resize_scale=0.08, distort_color=None, image_size=224, num_replicas=None, rank=None): + + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + assert isinstance(self.image_size, list) + from ofa.imagenet_codebase.data_providers.my_data_loader import MyDataLoader + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_transforms = self.build_train_transform() + train_dataset = self.train_dataset(train_transforms) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset.data) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset.data), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'stl10' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 10 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = '/mnt/datastore/STL10' # home server + + if not os.path.exists(self._save_path): + self._save_path = '/mnt/datastore/STL10' # home server + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.train_path, _transforms) + dataset = torchvision.datasets.STL10( + root=self.valid_path, split='train', download=False, transform=_transforms) + return dataset + + def test_dataset(self, _transforms): + # dataset = datasets.ImageFolder(self.valid_path, _transforms) + dataset = torchvision.datasets.STL10( + root=self.valid_path, split='test', download=False, transform=_transforms) + return dataset + + @property + def train_path(self): + # return os.path.join(self.save_path, 'train') + return self.save_path + + @property + def valid_path(self): + # return os.path.join(self.save_path, 'val') + return self.save_path + + @property + def normalize(self): + return transforms.Normalize( + mean=[0.44671097, 0.4398105, 0.4066468], + std=[0.2603405, 0.25657743, 0.27126738]) + + def build_train_transform(self, image_size=None, print_log=True): + if image_size is None: + image_size = self.image_size + if print_log: + print('Color jitter: %s, resize_scale: %s, img_size: %s' % + (self.distort_color, self.resize_scale, image_size)) + + if self.distort_color == 'torch': + color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + elif self.distort_color == 'tf': + color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + else: + color_transform = None + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + else: + resize_transform_class = transforms.RandomResizedCrop + + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + if color_transform is not None: + train_transforms.append(color_transform) + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset.data) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/networks/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/networks/__init__.py new file mode 100644 index 0000000..863dcb5 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/networks/__init__.py @@ -0,0 +1,4 @@ +from ofa.imagenet_codebase.networks.proxyless_nets import ProxylessNASNets, proxyless_base, MobileNetV2 +from ofa.imagenet_codebase.networks.mobilenet_v3 import MobileNetV3, MobileNetV3Large +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.networks.nsganetv2 import NSGANetV2 + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/networks/nsganetv2.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/networks/nsganetv2.py new file mode 100644 index 0000000..4f75faa --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/networks/nsganetv2.py @@ -0,0 +1,126 @@ +from timm.models.layers import drop_path +from ofa.imagenet_codebase.modules.layers import * +from ofa.imagenet_codebase.networks import MobileNetV3 + + +class MobileInvertedResidualBlock(MyModule): + """ + Modified from https://github.com/mit-han-lab/once-for-all/blob/master/ofa/ + imagenet_codebase/networks/proxyless_nets.py to include drop path in training + + """ + def __init__(self, mobile_inverted_conv, shortcut, drop_connect_rate=0.0): + super(MobileInvertedResidualBlock, self).__init__() + + self.mobile_inverted_conv = mobile_inverted_conv + self.shortcut = shortcut + self.drop_connect_rate = drop_connect_rate + + def forward(self, x): + if self.mobile_inverted_conv is None or isinstance(self.mobile_inverted_conv, ZeroLayer): + res = x + elif self.shortcut is None or isinstance(self.shortcut, ZeroLayer): + res = self.mobile_inverted_conv(x) + else: + # res = self.mobile_inverted_conv(x) + self.shortcut(x) + res = self.mobile_inverted_conv(x) + + if self.drop_connect_rate > 0.: + res = drop_path(res, drop_prob=self.drop_connect_rate, training=self.training) + + res += self.shortcut(x) + + return res + + @property + def module_str(self): + return '(%s, %s)' % ( + self.mobile_inverted_conv.module_str if self.mobile_inverted_conv is not None else None, + self.shortcut.module_str if self.shortcut is not None else None + ) + + @property + def config(self): + return { + 'name': MobileInvertedResidualBlock.__name__, + 'mobile_inverted_conv': self.mobile_inverted_conv.config if self.mobile_inverted_conv is not None else None, + 'shortcut': self.shortcut.config if self.shortcut is not None else None, + } + + @staticmethod + def build_from_config(config): + mobile_inverted_conv = set_layer_from_config(config['mobile_inverted_conv']) + shortcut = set_layer_from_config(config['shortcut']) + return MobileInvertedResidualBlock( + mobile_inverted_conv, shortcut, drop_connect_rate=config['drop_connect_rate']) + + +class NSGANetV2(MobileNetV3): + """ + Modified from https://github.com/mit-han-lab/once-for-all/blob/master/ofa/ + imagenet_codebase/networks/mobilenet_v3.py to include drop path in training + and option to reset classification layer + """ + @staticmethod + def build_from_config(config, drop_connect_rate=0.0): + first_conv = set_layer_from_config(config['first_conv']) + final_expand_layer = set_layer_from_config(config['final_expand_layer']) + feature_mix_layer = set_layer_from_config(config['feature_mix_layer']) + classifier = set_layer_from_config(config['classifier']) + + blocks = [] + for block_idx, block_config in enumerate(config['blocks']): + block_config['drop_connect_rate'] = drop_connect_rate * block_idx / len(config['blocks']) + blocks.append(MobileInvertedResidualBlock.build_from_config(block_config)) + + net = MobileNetV3(first_conv, blocks, final_expand_layer, feature_mix_layer, classifier) + if 'bn' in config: + net.set_bn_param(**config['bn']) + else: + net.set_bn_param(momentum=0.1, eps=1e-3) + + return net + + def zero_last_gamma(self): + for m in self.modules(): + if isinstance(m, MobileInvertedResidualBlock): + if isinstance(m.mobile_inverted_conv, MBInvertedConvLayer) and isinstance(m.shortcut, IdentityLayer): + m.mobile_inverted_conv.point_linear.bn.weight.data.zero_() + + @staticmethod + def build_net_via_cfg(cfg, input_channel, last_channel, n_classes, dropout_rate): + # first conv layer + first_conv = ConvLayer( + 3, input_channel, kernel_size=3, stride=2, use_bn=True, act_func='h_swish', ops_order='weight_bn_act' + ) + # build mobile blocks + feature_dim = input_channel + blocks = [] + for stage_id, block_config_list in cfg.items(): + for k, mid_channel, out_channel, use_se, act_func, stride, expand_ratio in block_config_list: + mb_conv = MBInvertedConvLayer( + feature_dim, out_channel, k, stride, expand_ratio, mid_channel, act_func, use_se + ) + if stride == 1 and out_channel == feature_dim: + shortcut = IdentityLayer(out_channel, out_channel) + else: + shortcut = None + blocks.append(MobileInvertedResidualBlock(mb_conv, shortcut)) + feature_dim = out_channel + # final expand layer + final_expand_layer = ConvLayer( + feature_dim, feature_dim * 6, kernel_size=1, use_bn=True, act_func='h_swish', ops_order='weight_bn_act', + ) + feature_dim = feature_dim * 6 + # feature mix layer + feature_mix_layer = ConvLayer( + feature_dim, last_channel, kernel_size=1, bias=False, use_bn=False, act_func='h_swish', + ) + # classifier + classifier = LinearLayer(last_channel, n_classes, dropout_rate=dropout_rate) + + return first_conv, blocks, final_expand_layer, feature_mix_layer, classifier + + @staticmethod + def reset_classifier(model, last_channel, n_classes, dropout_rate=0.0): + model.classifier = LinearLayer(last_channel, n_classes, dropout_rate=dropout_rate) \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/run_manager/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/run_manager/__init__.py new file mode 100644 index 0000000..80a18a0 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/codebase/run_manager/__init__.py @@ -0,0 +1,309 @@ +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.data_providers.imagenet import * +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.data_providers.cifar import * +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.data_providers.pets import * +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.data_providers.aircraft import * + +from ofa.imagenet_codebase.run_manager.run_manager import * + + +class ImagenetRunConfig(RunConfig): + + def __init__(self, n_epochs=1, init_lr=1e-4, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='imagenet', train_batch_size=128, test_batch_size=512, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.0, no_decay_keys=None, + mixup_alpha=None, + model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=32, resize_scale=0.08, distort_color='tf', image_size=224, + data_path='/mnt/datastore/ILSVRC2012', + **kwargs): + super(ImagenetRunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + self.imagenet_data_path = data_path + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == ImagenetDataProvider.name(): + DataProviderClass = ImagenetDataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + save_path=self.imagenet_data_path, + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] + + +class CIFARRunConfig(RunConfig): + def __init__(self, n_epochs=5, init_lr=0.01, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='cifar10', train_batch_size=96, test_batch_size=256, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.0, no_decay_keys=None, + mixup_alpha=None, + model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=2, resize_scale=0.08, distort_color=None, image_size=224, + data_path='/mnt/datastore/CIFAR', + **kwargs): + super(CIFARRunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + self.cifar_data_path = data_path + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == CIFAR10DataProvider.name(): + DataProviderClass = CIFAR10DataProvider + elif self.dataset == CIFAR100DataProvider.name(): + DataProviderClass = CIFAR100DataProvider + elif self.dataset == CINIC10DataProvider.name(): + DataProviderClass = CINIC10DataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + save_path=self.cifar_data_path, + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] + + +class Flowers102RunConfig(RunConfig): + + def __init__(self, n_epochs=3, init_lr=1e-2, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='flowers102', train_batch_size=32, test_batch_size=250, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.0, no_decay_keys=None, + mixup_alpha=None, + model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=4, resize_scale=0.08, distort_color=None, image_size=224, + data_path='/mnt/datastore/Flowers102', + **kwargs): + super(Flowers102RunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + self.flowers102_data_path = data_path + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == Flowers102DataProvider.name(): + DataProviderClass = Flowers102DataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + save_path=self.flowers102_data_path, + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] + + +class STL10RunConfig(RunConfig): + + def __init__(self, n_epochs=5, init_lr=1e-2, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='stl10', train_batch_size=96, test_batch_size=256, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.0, no_decay_keys=None, + mixup_alpha=None, + model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=4, resize_scale=0.08, distort_color=None, image_size=224, + data_path='/mnt/datastore/STL10', + **kwargs): + super(STL10RunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + self.stl10_data_path = data_path + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == STL10DataProvider.name(): + DataProviderClass = STL10DataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + save_path=self.stl10_data_path, + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] + + +class DTDRunConfig(RunConfig): + + def __init__(self, n_epochs=1, init_lr=0.05, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='dtd', train_batch_size=32, test_batch_size=250, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.0, no_decay_keys=None, + mixup_alpha=None, model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=32, resize_scale=0.08, distort_color='tf', image_size=224, + data_path='/mnt/datastore/dtd', + **kwargs): + super(DTDRunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + self.data_path = data_path + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == DTDDataProvider.name(): + DataProviderClass = DTDDataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + save_path=self.data_path, + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] + + +class PetsRunConfig(RunConfig): + + def __init__(self, n_epochs=1, init_lr=0.05, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='pets', train_batch_size=32, test_batch_size=250, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.0, no_decay_keys=None, + mixup_alpha=None, + model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=32, resize_scale=0.08, distort_color='tf', image_size=224, + data_path='/mnt/datastore/Oxford-IIITPets', + **kwargs): + super(PetsRunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + self.imagenet_data_path = data_path + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == OxfordIIITPetsDataProvider.name(): + DataProviderClass = OxfordIIITPetsDataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + save_path=self.imagenet_data_path, + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] + + +class AircraftRunConfig(RunConfig): + + def __init__(self, n_epochs=1, init_lr=0.05, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='aircraft', train_batch_size=32, test_batch_size=250, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.0, no_decay_keys=None, + mixup_alpha=None, + model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=32, resize_scale=0.08, distort_color='tf', image_size=224, + data_path='/mnt/datastore/Aircraft', + **kwargs): + super(AircraftRunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + self.data_path = data_path + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == FGVCAircraftDataProvider.name(): + DataProviderClass = FGVCAircraftDataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + save_path=self.data_path, + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] + + +def get_run_config(**kwargs): + if kwargs['dataset'] == 'imagenet': + run_config = ImagenetRunConfig(**kwargs) + elif kwargs['dataset'].startswith('cifar') or kwargs['dataset'].startswith('cinic'): + run_config = CIFARRunConfig(**kwargs) + elif kwargs['dataset'] == 'flowers102': + run_config = Flowers102RunConfig(**kwargs) + elif kwargs['dataset'] == 'stl10': + run_config = STL10RunConfig(**kwargs) + elif kwargs['dataset'] == 'dtd': + run_config = DTDRunConfig(**kwargs) + elif kwargs['dataset'] == 'pets': + run_config = PetsRunConfig(**kwargs) + elif kwargs['dataset'] == 'aircraft': + run_config = AircraftRunConfig(**kwargs) + elif kwargs['dataset'] == 'aircraft100': + run_config = AircraftRunConfig(**kwargs) + else: + raise NotImplementedError + + return run_config + + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/eval_utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/eval_utils.py new file mode 100644 index 0000000..aecc2ea --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/eval_utils.py @@ -0,0 +1,122 @@ +import numpy as np +import torch +import torchvision.transforms as transforms +from PIL import Image +import torchvision.utils +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.data_providers.aircraft import FGVCAircraft +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.data_providers.pets2 import PetDataset +import torch.utils.data as Data +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.data_providers.autoaugment import CIFAR10Policy + + +def get_dataset(data_name, batch_size, data_path, num_workers, + img_size, autoaugment, cutout, cutout_length): + num_class_dict = { + 'cifar100': 100, + 'cifar10': 10, + 'mnist': 10, + 'aircraft': 100, + 'svhn': 10, + 'pets': 37 + } + # 'aircraft30': 30, + # 'aircraft100': 100, + + train_transform, valid_transform = _data_transforms( + data_name, img_size, autoaugment, cutout, cutout_length) + if data_name == 'cifar100': + train_data = torchvision.datasets.CIFAR100( + root=data_path, train=True, download=True, transform=train_transform) + valid_data = torchvision.datasets.CIFAR100( + root=data_path, train=False, download=True, transform=valid_transform) + elif data_name == 'cifar10': + train_data = torchvision.datasets.CIFAR10( + root=data_path, train=True, download=True, transform=train_transform) + valid_data = torchvision.datasets.CIFAR10( + root=data_path, train=False, download=True, transform=valid_transform) + elif data_name.startswith('aircraft'): + print(data_path) + if 'aircraft100' in data_path: + data_path = data_path.replace('aircraft100', 'aircraft/fgvc-aircraft-2013b') + else: + data_path = data_path.replace('aircraft', 'aircraft/fgvc-aircraft-2013b') + train_data = FGVCAircraft(data_path, class_type='variant', split='trainval', + transform=train_transform, download=True) + valid_data = FGVCAircraft(data_path, class_type='variant', split='test', + transform=valid_transform, download=True) + elif data_name.startswith('pets'): + train_data = PetDataset(data_path, train=True, num_cl=37, + val_split=0.15, transforms=train_transform) + valid_data = PetDataset(data_path, train=False, num_cl=37, + val_split=0.15, transforms=valid_transform) + else: + raise KeyError + + train_queue = torch.utils.data.DataLoader( + train_data, batch_size=batch_size, shuffle=True, pin_memory=True, + num_workers=num_workers) + + valid_queue = torch.utils.data.DataLoader( + valid_data, batch_size=200, shuffle=False, pin_memory=True, + num_workers=num_workers) + + return train_queue, valid_queue, num_class_dict[data_name] + + + +class Cutout(object): + def __init__(self, length): + self.length = length + + def __call__(self, img): + h, w = img.size(1), img.size(2) + mask = np.ones((h, w), np.float32) + y = np.random.randint(h) + x = np.random.randint(w) + + y1 = np.clip(y - self.length // 2, 0, h) + y2 = np.clip(y + self.length // 2, 0, h) + x1 = np.clip(x - self.length // 2, 0, w) + x2 = np.clip(x + self.length // 2, 0, w) + + mask[y1: y2, x1: x2] = 0. + mask = torch.from_numpy(mask) + mask = mask.expand_as(img) + img *= mask + return img + + +def _data_transforms(data_name, img_size, autoaugment, cutout, cutout_length): + if 'cifar' in data_name: + norm_mean = [0.49139968, 0.48215827, 0.44653124] + norm_std = [0.24703233, 0.24348505, 0.26158768] + elif 'aircraft' in data_name: + norm_mean = [0.48933587508932375, 0.5183537408957618, 0.5387914411673883] + norm_std = [0.22388883112804625, 0.21641635409388751, 0.24615605842636115] + elif 'pets' in data_name: + norm_mean = [0.4828895122298728, 0.4448394893850807, 0.39566558230789783] + norm_std = [0.25925664613996574, 0.2532760018681693, 0.25981017205097917] + else: + raise KeyError + + train_transform = transforms.Compose([ + transforms.Resize((img_size, img_size), interpolation=Image.BICUBIC), # BICUBIC interpolation + transforms.RandomHorizontalFlip(), + ]) + + if autoaugment: + train_transform.transforms.append(CIFAR10Policy()) + + train_transform.transforms.append(transforms.ToTensor()) + + if cutout: + train_transform.transforms.append(Cutout(cutout_length)) + + train_transform.transforms.append(transforms.Normalize(norm_mean, norm_std)) + + valid_transform = transforms.Compose([ + transforms.Resize((img_size, img_size), interpolation=Image.BICUBIC), # BICUBIC interpolation + transforms.ToTensor(), + transforms.Normalize(norm_mean, norm_std), + ]) + return train_transform, valid_transform diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/evaluator.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/evaluator.py new file mode 100644 index 0000000..4fe7718 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/evaluator.py @@ -0,0 +1,233 @@ +import os +import torch +import numpy as np +import random +import sys +import transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.eval_utils +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.networks import NSGANetV2 +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.run_manager import get_run_config +from ofa.elastic_nn.networks import OFAMobileNetV3 +from ofa.imagenet_codebase.run_manager import RunManager +from ofa.elastic_nn.modules.dynamic_op import DynamicSeparableConv2d +from torchprofile import profile_macs +import copy +import json +import warnings + +warnings.simplefilter("ignore") + +DynamicSeparableConv2d.KERNEL_TRANSFORM_MODE = 1 + + +class ArchManager: + def __init__(self): + self.num_blocks = 20 + self.num_stages = 5 + self.kernel_sizes = [3, 5, 7] + self.expand_ratios = [3, 4, 6] + self.depths = [2, 3, 4] + self.resolutions = [160, 176, 192, 208, 224] + + def random_sample(self): + sample = {} + d = [] + e = [] + ks = [] + for i in range(self.num_stages): + d.append(random.choice(self.depths)) + + for i in range(self.num_blocks): + e.append(random.choice(self.expand_ratios)) + ks.append(random.choice(self.kernel_sizes)) + + sample = { + 'wid': None, + 'ks': ks, + 'e': e, + 'd': d, + 'r': [random.choice(self.resolutions)] + } + + return sample + + def random_resample(self, sample, i): + assert i >= 0 and i < self.num_blocks + sample['ks'][i] = random.choice(self.kernel_sizes) + sample['e'][i] = random.choice(self.expand_ratios) + + def random_resample_depth(self, sample, i): + assert i >= 0 and i < self.num_stages + sample['d'][i] = random.choice(self.depths) + + def random_resample_resolution(self, sample): + sample['r'][0] = random.choice(self.resolutions) + + +def parse_string_list(string): + if isinstance(string, str): + # convert '[5 5 5 7 7 7 3 3 7 7 7 3 3]' to [5, 5, 5, 7, 7, 7, 3, 3, 7, 7, 7, 3, 3] + return list(map(int, string[1:-1].split())) + else: + return string + + +def pad_none(x, depth, max_depth): + new_x, counter = [], 0 + for d in depth: + for _ in range(d): + new_x.append(x[counter]) + counter += 1 + if d < max_depth: + new_x += [None] * (max_depth - d) + return new_x + + +def get_net_info(net, data_shape, measure_latency=None, print_info=True, clean=False, lut=None): + net_info = eval_utils.get_net_info( + net, data_shape, measure_latency, print_info=print_info, clean=clean, lut=lut) + + gpu_latency, cpu_latency = None, None + for k in net_info.keys(): + if 'gpu' in k: + gpu_latency = np.round(net_info[k]['val'], 2) + if 'cpu' in k: + cpu_latency = np.round(net_info[k]['val'], 2) + + return { + 'params': np.round(net_info['params'] / 1e6, 2), + 'flops': np.round(net_info['flops'] / 1e6, 2), + 'gpu': gpu_latency, 'cpu': cpu_latency + } + + +def validate_config(config, max_depth=4): + kernel_size, exp_ratio, depth = config['ks'], config['e'], config['d'] + + if isinstance(kernel_size, str): kernel_size = parse_string_list(kernel_size) + if isinstance(exp_ratio, str): exp_ratio = parse_string_list(exp_ratio) + if isinstance(depth, str): depth = parse_string_list(depth) + + assert (isinstance(kernel_size, list) or isinstance(kernel_size, int)) + assert (isinstance(exp_ratio, list) or isinstance(exp_ratio, int)) + assert isinstance(depth, list) + + if len(kernel_size) < len(depth) * max_depth: + kernel_size = pad_none(kernel_size, depth, max_depth) + if len(exp_ratio) < len(depth) * max_depth: + exp_ratio = pad_none(exp_ratio, depth, max_depth) + + # return {'ks': kernel_size, 'e': exp_ratio, 'd': depth, 'w': config['w']} + return {'ks': kernel_size, 'e': exp_ratio, 'd': depth} + + +def set_nas_test_dataset(path, test_data_name, max_img): + if not test_data_name in ['mnist', 'svhn', 'cifar10', + 'cifar100', 'aircraft', 'pets']: raise ValueError(test_data_name) + + dpath = path + num_cls = 10 # mnist, svhn, cifar10 + if test_data_name in ['cifar100', 'aircraft']: + num_cls = 100 + elif test_data_name == 'pets': + num_cls = 37 + + x = torch.load(dpath + f'/{test_data_name}bylabel') + img_per_cls = min(int(max_img / num_cls), 20) + return x, img_per_cls, num_cls + + +class OFAEvaluator: + """ based on OnceForAll supernet taken from https://github.com/mit-han-lab/once-for-all """ + + def __init__(self, num_gen_arch, img_size, drop_path, + n_classes=1000, + model_path=None, + kernel_size=None, exp_ratio=None, depth=None): + # default configurations + self.kernel_size = [3, 5, 7] if kernel_size is None else kernel_size # depth-wise conv kernel size + self.exp_ratio = [3, 4, 6] if exp_ratio is None else exp_ratio # expansion rate + self.depth = [2, 3, 4] if depth is None else depth # number of MB block repetition + + if 'w1.0' in model_path: + self.width_mult = 1.0 + elif 'w1.2' in model_path: + self.width_mult = 1.2 + else: + raise ValueError + + self.engine = OFAMobileNetV3( + n_classes=n_classes, + dropout_rate=0, width_mult_list=self.width_mult, ks_list=self.kernel_size, + expand_ratio_list=self.exp_ratio, depth_list=self.depth) + + + init = torch.load(model_path, map_location='cpu')['state_dict'] + self.engine.load_weights_from_net(init) + print(f'load {model_path}...') + + ## metad2a + self.arch_manager = ArchManager() + self.num_gen_arch = num_gen_arch + + + def sample_random_architecture(self): + sampled_architecture = self.arch_manager.random_sample() + return sampled_architecture + + def get_architecture(self, bound=None): + g_lst, pred_acc_lst, x_lst = [], [], [] + searched_g, max_pred_acc = None, 0 + + with torch.no_grad(): + for n in range(self.num_gen_arch): + file_acc = self.lines[n].split()[0] + g_dict = ' '.join(self.lines[n].split()) + g = json.loads(g_dict.replace("'", "\"")) + + if bound is not None: + subnet, config = self.sample(config=g) + net = NSGANetV2.build_from_config(subnet.config, + drop_connect_rate=self.drop_path) + inputs = torch.randn(1, 3, self.img_size, self.img_size) + flops = profile_macs(copy.deepcopy(net), inputs) / 1e6 + if flops <= bound: + searched_g = g + break + else: + searched_g = g + pred_acc_lst.append(file_acc) + break + + if searched_g is None: + raise ValueError(searched_g) + return searched_g, pred_acc_lst + + + def sample(self, config=None): + """ randomly sample a sub-network """ + if config is not None: + config = validate_config(config) + self.engine.set_active_subnet(ks=config['ks'], e=config['e'], d=config['d']) + else: + config = self.engine.sample_active_subnet() + + subnet = self.engine.get_active_subnet(preserve_weight=True) + return subnet, config + + @staticmethod + def save_net_config(path, net, config_name='net.config'): + """ dump run_config and net_config to the model_folder """ + net_save_path = os.path.join(path, config_name) + json.dump(net.config, open(net_save_path, 'w'), indent=4) + print('Network configs dump to %s' % net_save_path) + + @staticmethod + def save_net(path, net, model_name): + """ dump net weight as checkpoint """ + if isinstance(net, torch.nn.DataParallel): + checkpoint = {'state_dict': net.module.state_dict()} + else: + checkpoint = {'state_dict': net.state_dict()} + model_path = os.path.join(path, model_name) + torch.save(checkpoint, model_path) + print('Network model dump to %s' % model_path) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/main.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/main.py new file mode 100644 index 0000000..d287a35 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/main.py @@ -0,0 +1,169 @@ +import os +import sys +import json +import logging +import numpy as np +import copy +import torch +import torch.nn as nn +import random +import torch.optim as optim +from evaluator import OFAEvaluator +from torchprofile import profile_macs +from codebase.networks import NSGANetV2 +from parser import get_parse +from eval_utils import get_dataset + + +args = get_parse() +os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu +device_list = [int(_) for _ in args.gpu.split(',')] +args.n_gpus = len(device_list) +args.device = torch.device("cuda:0") + +if args.seed is None or args.seed < 0: args.seed = random.randint(1, 100000) +torch.cuda.manual_seed(args.seed) +torch.manual_seed(args.seed) +np.random.seed(args.seed) +random.seed(args.seed) + + +evaluator = OFAEvaluator(args, + model_path='../.torch/ofa_nets/ofa_mbv3_d234_e346_k357_w1.0') + +args.save_path = os.path.join(args.save_path, f'evaluation/{args.data_name}') +if args.model_config.startswith('flops@'): + args.save_path += f'-nsganetV2-{args.model_config}-{args.seed}' +else: + args.save_path += f'-metaD2A-{args.bound}-{args.seed}' +if not os.path.exists(args.save_path): + os.makedirs(args.save_path) + +args.data_path = os.path.join(args.data_path, args.data_name) + +log_format = '%(asctime)s %(message)s' +logging.basicConfig(stream=sys.stdout, level=logging.INFO, + format=log_format, datefmt='%m/%d %I:%M:%S %p') +fh = logging.FileHandler(os.path.join(args.save_path, 'log.txt')) +fh.setFormatter(logging.Formatter(log_format)) +logging.getLogger().addHandler(fh) +if not torch.cuda.is_available(): + logging.info('no gpu self.args.device available') + sys.exit(1) +logging.info("args = %s", args) + + + +def set_architecture(n_cls): + if args.model_config.startswith('flops@'): + names = {'cifar10': 'CIFAR-10', 'cifar100': 'CIFAR-100', + 'aircraft100': 'Aircraft', 'pets': 'Pets'} + p = os.path.join('./searched-architectures/{}/net-{}/net.subnet'. + format(names[args.data_name], args.model_config)) + g = json.load(open(p)) + else: + g, acc = evaluator.get_architecture(args) + + subnet, config = evaluator.sample(g) + net = NSGANetV2.build_from_config(subnet.config, drop_connect_rate=args.drop_path) + net.load_state_dict(subnet.state_dict()) + + NSGANetV2.reset_classifier( + net, last_channel=net.classifier.in_features, + n_classes=n_cls, dropout_rate=args.drop) + # calculate #Paramaters and #FLOPS + inputs = torch.randn(1, 3, args.img_size, args.img_size) + flops = profile_macs(copy.deepcopy(net), inputs) / 1e6 + params = sum(p.numel() for p in net.parameters() if p.requires_grad) / 1e6 + net_name = "net_flops@{:.0f}".format(flops) + logging.info('#params {:.2f}M, #flops {:.0f}M'.format(params, flops)) + OFAEvaluator.save_net_config(args.save_path, net, net_name + '.config') + if args.n_gpus > 1: + net = nn.DataParallel(net) # data parallel in case more than 1 gpu available + net = net.to(args.device) + + return net, net_name + + +def train(train_queue, net, criterion, optimizer): + net.train() + train_loss, correct, total = 0, 0, 0 + for step, (inputs, targets) in enumerate(train_queue): + # upsample by bicubic to match imagenet training size + inputs, targets = inputs.to(args.device), targets.to(args.device) + optimizer.zero_grad() + outputs = net(inputs) + loss = criterion(outputs, targets) + loss.backward() + nn.utils.clip_grad_norm_(net.parameters(), args.grad_clip) + optimizer.step() + train_loss += loss.item() + _, predicted = outputs.max(1) + total += targets.size(0) + correct += predicted.eq(targets).sum().item() + if step % args.report_freq == 0: + logging.info('train %03d %e %f', step, train_loss / total, 100. * correct / total) + logging.info('train acc %f', 100. * correct / total) + return train_loss / total, 100. * correct / total + + +def infer(valid_queue, net, criterion, early_stop=False): + net.eval() + test_loss, correct, total = 0, 0, 0 + with torch.no_grad(): + for step, (inputs, targets) in enumerate(valid_queue): + inputs, targets = inputs.to(args.device), targets.to(args.device) + outputs = net(inputs) + loss = criterion(outputs, targets) + test_loss += loss.item() + _, predicted = outputs.max(1) + total += targets.size(0) + correct += predicted.eq(targets).sum().item() + if step % args.report_freq == 0: + logging.info('valid %03d %e %f', step, test_loss / total, 100. * correct / total) + if early_stop and step == 10: + break + acc = 100. * correct / total + logging.info('valid acc %f', 100. * correct / total) + + return test_loss / total, acc + + +def main(): + best_acc, top_checkpoints = 0, [] + + train_queue, valid_queue, n_cls = get_dataset(args) + net, net_name = set_architecture(n_cls) + parameters = filter(lambda p: p.requires_grad, net.parameters()) + optimizer = optim.SGD(parameters, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) + criterion = nn.CrossEntropyLoss().to(args.device) + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, args.epochs) + + for epoch in range(args.epochs): + logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0]) + + train(train_queue, net, criterion, optimizer) + _, valid_acc = infer(valid_queue, net, criterion) + # checkpoint saving + + if len(top_checkpoints) < args.topk: + OFAEvaluator.save_net(args.save_path, net, net_name + '.ckpt{}'.format(epoch)) + top_checkpoints.append((os.path.join(args.save_path, net_name + '.ckpt{}'.format(epoch)), valid_acc)) + else: + idx = np.argmin([x[1] for x in top_checkpoints]) + if valid_acc > top_checkpoints[idx][1]: + OFAEvaluator.save_net(args.save_path, net, net_name + '.ckpt{}'.format(epoch)) + top_checkpoints.append((os.path.join(args.save_path, net_name + '.ckpt{}'.format(epoch)), valid_acc)) + # remove the idx + os.remove(top_checkpoints[idx][0]) + top_checkpoints.pop(idx) + print(top_checkpoints) + if valid_acc > best_acc: + OFAEvaluator.save_net(args.save_path, net, net_name + '.best') + best_acc = valid_acc + scheduler.step() + + + +if __name__ == '__main__': + main() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/parser.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/parser.py new file mode 100644 index 0000000..c6c4fd7 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/parser.py @@ -0,0 +1,43 @@ +import argparse + +def get_parse(): + parser = argparse.ArgumentParser(description='MetaD2A vs NSGANETv2') + parser.add_argument('--save-path', type=str, default='../results', help='the path of save directory') + parser.add_argument('--data-path', type=str, default='../data', help='the path of save directory') + parser.add_argument('--data-name', type=str, default=None, help='meta-test dataset name') + parser.add_argument('--num-gen-arch', type=int, default=200, + help='the number of candidate architectures generated by the generator') + parser.add_argument('--bound', type=int, default=None) + + # original setting + parser.add_argument('--seed', type=int, default=-1, help='random seed') + parser.add_argument('--batch-size', type=int, default=96, help='batch size') + parser.add_argument('--num_workers', type=int, default=2, help='number of workers for data loading') + parser.add_argument('--gpu', type=str, default='0', help='set visible gpus') + parser.add_argument('--lr', type=float, default=0.01, help='init learning rate') + parser.add_argument('--momentum', type=float, default=0.9, help='momentum') + parser.add_argument('--weight_decay', type=float, default=4e-5, help='weight decay') + parser.add_argument('--report_freq', type=float, default=50, help='report frequency') + parser.add_argument('--epochs', type=int, default=150, help='num of training epochs') + parser.add_argument('--grad_clip', type=float, default=5, help='gradient clipping') + parser.add_argument('--cutout', action='store_true', default=True, help='use cutout') + parser.add_argument('--cutout_length', type=int, default=16, help='cutout length') + parser.add_argument('--autoaugment', action='store_true', default=True, help='use auto augmentation') + + parser.add_argument('--topk', type=int, default=10, help='top k checkpoints to save') + parser.add_argument('--evaluate', action='store_true', default=False, help='evaluate a pretrained model') + # model related + parser.add_argument('--model', default='resnet101', type=str, metavar='MODEL', + help='Name of model to train (default: "countception"') + parser.add_argument('--model-config', type=str, default='search', + help='location of a json file of specific model declaration') + parser.add_argument('--initial-checkpoint', default='', type=str, metavar='PATH', + help='Initialize model from this checkpoint (default: none)') + parser.add_argument('--drop', type=float, default=0.2, + help='dropout rate') + parser.add_argument('--drop-path', type=float, default=0.2, metavar='PCT', + help='Drop path rate (default: None)') + parser.add_argument('--img-size', type=int, default=224, + help='input resolution (192 -> 256)') + args = parser.parse_args() + return args \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/train.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/train.py new file mode 100644 index 0000000..b0a3812 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/evaluation/train.py @@ -0,0 +1,261 @@ +import os +import sys +import json +import logging +import numpy as np +import copy +import torch +import torch.nn as nn +import random +import torch.optim as optim + +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.evaluator import OFAEvaluator +from torchprofile import profile_macs +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.codebase.networks import NSGANetV2 +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.parser import get_parse +from transfer_nag_lib.MetaD2A_mobilenetV3.evaluation.eval_utils import get_dataset +from transfer_nag_lib.MetaD2A_nas_bench_201.metad2a_utils import reset_seed +from transfer_nag_lib.ofa_net import OFASubNet + + +# os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu +# device_list = [int(_) for _ in args.gpu.split(',')] +# args.n_gpus = len(device_list) +# args.device = torch.device("cuda:0") + +# if args.seed is None or args.seed < 0: args.seed = random.randint(1, 100000) +# torch.cuda.manual_seed(args.seed) +# torch.manual_seed(args.seed) +# np.random.seed(args.seed) +# random.seed(args.seed) + + + +# args.save_path = os.path.join(args.save_path, f'evaluation/{args.data_name}') +# if args.model_config.startswith('flops@'): +# args.save_path += f'-nsganetV2-{args.model_config}-{args.seed}' +# else: +# args.save_path += f'-metaD2A-{args.bound}-{args.seed}' +# if not os.path.exists(args.save_path): +# os.makedirs(args.save_path) + +# args.data_path = os.path.join(args.data_path, args.data_name) + +# log_format = '%(asctime)s %(message)s' +# logging.basicConfig(stream=sys.stdout, level=print, +# format=log_format, datefmt='%m/%d %I:%M:%S %p') +# fh = logging.FileHandler(os.path.join(args.save_path, 'log.txt')) +# fh.setFormatter(logging.Formatter(log_format)) +# logging.getLogger().addHandler(fh) +# if not torch.cuda.is_available(): +# print('no gpu self.args.device available') +# sys.exit(1) +# print("args = %s", args) + + + +def set_architecture(n_cls, evaluator, drop_path, drop, img_size, n_gpus, device, save_path, model_str): + # g, acc = evaluator.get_architecture(model_str) + g = OFASubNet(model_str).get_op_dict() + subnet, config = evaluator.sample(g) + net = NSGANetV2.build_from_config(subnet.config, drop_connect_rate=drop_path) + net.load_state_dict(subnet.state_dict()) + + NSGANetV2.reset_classifier( + net, last_channel=net.classifier.in_features, + n_classes=n_cls, dropout_rate=drop) + # calculate #Paramaters and #FLOPS + inputs = torch.randn(1, 3, img_size, img_size) + flops = profile_macs(copy.deepcopy(net), inputs) / 1e6 + params = sum(p.numel() for p in net.parameters() if p.requires_grad) / 1e6 + net_name = "net_flops@{:.0f}".format(flops) + print('#params {:.2f}M, #flops {:.0f}M'.format(params, flops)) + # OFAEvaluator.save_net_config(save_path, net, net_name + '.config') + if torch.cuda.device_count() > 1: + print("Let's use", torch.cuda.device_count(), "GPUs!") + net = nn.DataParallel(net) + net = net.to(device) + + return net, net_name, params, flops + + +def train(train_queue, net, criterion, optimizer, grad_clip, device, report_freq): + net.train() + train_loss, correct, total = 0, 0, 0 + for step, (inputs, targets) in enumerate(train_queue): + # upsample by bicubic to match imagenet training size + inputs, targets = inputs.to(device), targets.to(device) + optimizer.zero_grad() + outputs = net(inputs) + loss = criterion(outputs, targets) + loss.backward() + nn.utils.clip_grad_norm_(net.parameters(), grad_clip) + optimizer.step() + train_loss += loss.item() + _, predicted = outputs.max(1) + total += targets.size(0) + correct += predicted.eq(targets).sum().item() + if step % report_freq == 0: + print(f'train step {step:03d} loss {train_loss / total:.4f} train acc {100. * correct / total:.4f}') + print(f'train acc {100. * correct / total:.4f}') + return train_loss / total, 100. * correct / total + + +def infer(valid_queue, net, criterion, device, report_freq, early_stop=False): + net.eval() + test_loss, correct, total = 0, 0, 0 + with torch.no_grad(): + for step, (inputs, targets) in enumerate(valid_queue): + inputs, targets = inputs.to(device), targets.to(device) + outputs = net(inputs) + loss = criterion(outputs, targets) + test_loss += loss.item() + _, predicted = outputs.max(1) + total += targets.size(0) + correct += predicted.eq(targets).sum().item() + if step % report_freq == 0: + print(f'valid {step:03d} {test_loss / total:.4f} {100. * correct / total:.4f}') + if early_stop and step == 10: + break + acc = 100. * correct / total + print('valid acc {:.4f}'.format(100. * correct / total)) + + return test_loss / total, acc + + +def train_single_model(save_path, workers, datasets, xpaths, splits, use_less, + seed, model_str, device, + lr=0.01, + momentum=0.9, + weight_decay=4e-5, + report_freq=50, + epochs=150, + grad_clip=5, + cutout=True, + cutout_length=16, + autoaugment=True, + drop=0.2, + drop_path=0.2, + img_size=224, + batch_size=96, + ): + assert torch.cuda.is_available(), 'CUDA is not available.' + torch.backends.cudnn.enabled = True + torch.backends.cudnn.deterministic = True + reset_seed(seed) + # save_dir = Path(save_dir) + # logger = Logger(str(save_dir), 0, False) + os.makedirs(save_path, exist_ok=True) + to_save_name = save_path + '/seed-{:04d}.pth'.format(seed) + print(to_save_name) + # args = get_parse() + num_gen_arch = None + evaluator = OFAEvaluator(num_gen_arch, img_size, drop_path, + model_path='/home/data/GTAD/checkpoints/ofa/ofa_net/ofa_mbv3_d234_e346_k357_w1.0') + + train_queue, valid_queue, n_cls = get_dataset(datasets, batch_size, + xpaths, workers, img_size, autoaugment, cutout, cutout_length) + net, net_name, params, flops = set_architecture(n_cls, evaluator, + drop_path, drop, img_size, n_gpus=1, device=device, save_path=save_path, model_str=model_str) + + + # net.to(device) + + parameters = filter(lambda p: p.requires_grad, net.parameters()) + optimizer = optim.SGD(parameters, lr=lr, momentum=momentum, weight_decay=weight_decay) + criterion = nn.CrossEntropyLoss().to(device) + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, epochs) + + # assert epochs == 1 + max_valid_acc = 0 + max_epoch = 0 + for epoch in range(epochs): + print('epoch {:d} lr {:.4f}'.format(epoch, scheduler.get_lr()[0])) + + train(train_queue, net, criterion, optimizer, grad_clip, device, report_freq) + _, valid_acc = infer(valid_queue, net, criterion, device, report_freq) + torch.save(valid_acc, to_save_name) + print(f'seed {seed:04d} last acc {valid_acc:.4f} max acc {max_valid_acc:.4f}') + if max_valid_acc < valid_acc: + max_valid_acc = valid_acc + max_epoch = epoch + # parent_path = os.path.abspath(os.path.join(save_path, os.pardir)) + # with open(parent_path + '/accuracy.txt', 'a+') as f: + # f.write(f'{model_str} seed {seed:04d} {valid_acc:.4f}\n') + + return valid_acc, max_valid_acc, params, flops + + +################ NAS BENCH 201 ##################### +# def train_single_model(save_dir, workers, datasets, xpaths, splits, use_less, +# seeds, model_str, arch_config): +# assert torch.cuda.is_available(), 'CUDA is not available.' +# torch.backends.cudnn.enabled = True +# torch.backends.cudnn.deterministic = True +# torch.set_num_threads(workers) + +# save_dir = Path(save_dir) +# logger = Logger(str(save_dir), 0, False) + +# if model_str in CellArchitectures: +# arch = CellArchitectures[model_str] +# logger.log( +# 'The model string is found in pre-defined architecture dict : {:}'.format(model_str)) +# else: +# try: +# arch = CellStructure.str2structure(model_str) +# except: +# raise ValueError( +# 'Invalid model string : {:}. It can not be found or parsed.'.format(model_str)) + +# assert arch.check_valid_op(get_search_spaces( +# 'cell', 'nas-bench-201')), '{:} has the invalid op.'.format(arch) +# # assert arch.check_valid_op(get_search_spaces('cell', 'full')), '{:} has the invalid op.'.format(arch) +# logger.log('Start train-evaluate {:}'.format(arch.tostr())) +# logger.log('arch_config : {:}'.format(arch_config)) + +# start_time, seed_time = time.time(), AverageMeter() +# for _is, seed in enumerate(seeds): +# logger.log( +# '\nThe {:02d}/{:02d}-th seed is {:} ----------------------<.>----------------------'.format(_is, len(seeds), +# seed)) +# to_save_name = save_dir / 'seed-{:04d}.pth'.format(seed) +# if to_save_name.exists(): +# logger.log( +# 'Find the existing file {:}, directly load!'.format(to_save_name)) +# checkpoint = torch.load(to_save_name) +# else: +# logger.log( +# 'Does not find the existing file {:}, train and evaluate!'.format(to_save_name)) +# checkpoint = evaluate_all_datasets(arch, datasets, xpaths, splits, use_less, +# seed, arch_config, workers, logger) +# torch.save(checkpoint, to_save_name) +# # log information +# logger.log('{:}'.format(checkpoint['info'])) +# all_dataset_keys = checkpoint['all_dataset_keys'] +# for dataset_key in all_dataset_keys: +# logger.log('\n{:} dataset : {:} {:}'.format( +# '-' * 15, dataset_key, '-' * 15)) +# dataset_info = checkpoint[dataset_key] +# # logger.log('Network ==>\n{:}'.format( dataset_info['net_string'] )) +# logger.log('Flops = {:} MB, Params = {:} MB'.format( +# dataset_info['flop'], dataset_info['param'])) +# logger.log('config : {:}'.format(dataset_info['config'])) +# logger.log('Training State (finish) = {:}'.format( +# dataset_info['finish-train'])) +# last_epoch = dataset_info['total_epoch'] - 1 +# train_acc1es, train_acc5es = dataset_info['train_acc1es'], dataset_info['train_acc5es'] +# valid_acc1es, valid_acc5es = dataset_info['valid_acc1es'], dataset_info['valid_acc5es'] +# # measure elapsed time +# seed_time.update(time.time() - start_time) +# start_time = time.time() +# need_time = 'Time Left: {:}'.format(convert_secs2time( +# seed_time.avg * (len(seeds) - _is - 1), True)) +# logger.log( +# '\n<<<***>>> The {:02d}/{:02d}-th seed is {:} other procedures need {:}'.format(_is, len(seeds), seed, +# need_time)) +# logger.close() +# ################### + +if __name__ == '__main__': + train_single_model() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/__init__.py new file mode 100644 index 0000000..b6c6ce7 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/__init__.py @@ -0,0 +1,5 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from .generator import Generator diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/generator.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/generator.py new file mode 100644 index 0000000..a9b3316 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/generator.py @@ -0,0 +1,204 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import os +import random +from tqdm import tqdm +import numpy as np +import time + +import torch +from torch import optim +from torch.optim.lr_scheduler import ReduceLROnPlateau + +from utils import load_graph_config, decode_ofa_mbv3_to_igraph, decode_igraph_to_ofa_mbv3 +from utils import Accumulator, Log +from utils import load_model, save_model +from loader import get_meta_train_loader, get_meta_test_loader + +from .generator_model import GeneratorModel + + +class Generator: + def __init__(self, args): + self.args = args + self.batch_size = args.batch_size + self.data_path = args.data_path + self.num_sample = args.num_sample + self.max_epoch = args.max_epoch + self.save_epoch = args.save_epoch + self.model_path = args.model_path + self.save_path = args.save_path + self.model_name = args.model_name + self.test = args.test + self.device = args.device + + graph_config = load_graph_config( + args.graph_data_name, args.nvt, args.data_path) + self.model = GeneratorModel(args, graph_config) + self.model.to(self.device) + + if self.test: + self.data_name = args.data_name + self.num_class = args.num_class + self.load_epoch = args.load_epoch + self.num_gen_arch = args.num_gen_arch + load_model(self.model, self.model_path, self.load_epoch) + + else: + self.optimizer = optim.Adam(self.model.parameters(), lr=1e-4) + self.scheduler = ReduceLROnPlateau(self.optimizer, 'min', + factor=0.1, patience=10, verbose=True) + self.mtrloader = get_meta_train_loader( + self.batch_size, self.data_path, self.num_sample) + self.mtrlog = Log(self.args, open(os.path.join( + self.save_path, self.model_name, 'meta_train_generator.log'), 'w')) + self.mtrlog.print_args() + self.mtrlogger = Accumulator('loss', 'recon_loss', 'kld') + self.mvallogger = Accumulator('loss', 'recon_loss', 'kld') + + def meta_train(self): + sttime = time.time() + for epoch in range(1, self.max_epoch + 1): + self.mtrlog.ep_sttime = time.time() + loss = self.meta_train_epoch(epoch) + self.scheduler.step(loss) + self.mtrlog.print(self.mtrlogger, epoch, tag='train') + + self.meta_validation() + self.mtrlog.print(self.mvallogger, epoch, tag='valid') + + if epoch % self.save_epoch == 0: + save_model(epoch, self.model, self.model_path) + + self.mtrlog.save_time_log() + + def meta_train_epoch(self, epoch): + self.model.to(self.device) + self.model.train() + + self.mtrloader.dataset.set_mode('train') + pbar = tqdm(self.mtrloader) + + for batch in pbar: + for x, g, acc in batch: + self.optimizer.zero_grad() + g = decode_ofa_mbv3_to_igraph(g)[0] + x_ = x.unsqueeze(0).to(self.device) + mu, logvar = self.model.set_encode(x_) + loss, recon, kld = self.model.loss(mu.unsqueeze(0), logvar.unsqueeze(0), [g]) + loss.backward() + self.optimizer.step() + cnt = len(x) + self.mtrlogger.accum([loss.item() / cnt, + recon.item() / cnt, + kld.item() / cnt]) + + return self.mtrlogger.get('loss') + + + def meta_validation(self): + self.model.to(self.device) + self.model.eval() + + self.mtrloader.dataset.set_mode('valid') + pbar = tqdm(self.mtrloader) + + for batch in pbar: + for x, g, acc in batch: + with torch.no_grad(): + g = decode_ofa_mbv3_to_igraph(g)[0] + x_ = x.unsqueeze(0).to(self.device) + mu, logvar = self.model.set_encode(x_) + loss, recon, kld = self.model.loss(mu.unsqueeze(0), logvar.unsqueeze(0), [g]) + + cnt = len(x) + self.mvallogger.accum([loss.item() / cnt, + recon.item() / cnt, + kld.item() / cnt]) + + return self.mvallogger.get('loss') + + + def meta_test(self, predictor): + if self.data_name == 'all': + for data_name in ['cifar100', 'cifar10', 'mnist', 'svhn', 'aircraft30', 'aircraft100', 'pets']: + self.meta_test_per_dataset(data_name, predictor) + else: + self.meta_test_per_dataset(self.data_name, predictor) + + def meta_test_per_dataset(self, data_name, predictor): + # meta_test_path = os.path.join( + # self.save_path, 'meta_test', data_name, 'generated_arch') + meta_test_path = os.path.join( + self.save_path, 'meta_test', data_name, f'{self.num_gen_arch}', 'generated_arch') + if not os.path.exists(meta_test_path): + os.makedirs(meta_test_path) + + meta_test_loader = get_meta_test_loader( + self.data_path, data_name, self.num_sample, self.num_class) + + print(f'==> generate architectures for {data_name}') + runs = 10 if data_name in ['cifar10', 'cifar100'] else 1 + # num_gen_arch = 500 if data_name in ['cifar100'] else self.num_gen_arch + elasped_time = [] + for run in range(1, runs + 1): + print(f'==> run {run}/{runs}') + elasped_time.append(self.generate_architectures( + meta_test_loader, data_name, + meta_test_path, run, self.num_gen_arch, predictor)) + print(f'==> done\n') + + # time_path = os.path.join(self.save_path, 'meta_test', data_name, 'time.txt') + time_path = os.path.join(self.save_path, 'meta_test', data_name, f'{self.num_gen_arch}', 'time.txt') + with open(time_path, 'w') as f_time: + msg = f'generator elasped time {np.mean(elasped_time):.2f}s' + print(f'==> save time in {time_path}') + f_time.write(msg + '\n'); + print(msg) + + def generate_architectures(self, meta_test_loader, data_name, + meta_test_path, run, num_gen_arch, predictor): + self.model.eval() + self.model.to(self.device) + + architecture_string_lst, pred_acc_lst = [], [] + total_cnt, valid_cnt = 0, 0 + flag = False + + start = time.time() + with torch.no_grad(): + for x in meta_test_loader: + x_ = x.unsqueeze(0).to(self.device) + mu, logvar = self.model.set_encode(x_) + z = self.model.reparameterize(mu.unsqueeze(0), logvar.unsqueeze(0)) + g_recon = self.model.graph_decode(z) + pred_acc = predictor.forward(x_, g_recon) + architecture_string = decode_igraph_to_ofa_mbv3(g_recon[0]) + total_cnt += 1 + if architecture_string is not None: + if not architecture_string in architecture_string_lst: + valid_cnt += 1 + architecture_string_lst.append(architecture_string) + pred_acc_lst.append(pred_acc.item()) + if valid_cnt == num_gen_arch: + flag = True + break + if flag: + break + elapsed = time.time() - start + pred_acc_lst, architecture_string_lst = zip(*sorted(zip(pred_acc_lst, + architecture_string_lst), + key=lambda x: x[0], reverse=True)) + + spath = os.path.join(meta_test_path, f"run_{run}.txt") + with open(spath, 'w') as f: + print(f'==> save generated architectures in {spath}') + msg = f'elapsed time: {elapsed:6.2f}s ' + print(msg); + f.write(msg + '\n') + for i, architecture_string in enumerate(architecture_string_lst): + f.write(f"{architecture_string}\n") + return elapsed diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/generator_model.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/generator_model.py new file mode 100644 index 0000000..c58c431 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/generator/generator_model.py @@ -0,0 +1,396 @@ +###################################################################################### +# Copyright (c) muhanzhang, D-VAE, NeurIPS 2019 [GitHub D-VAE] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +from torch import nn +from torch.nn import functional as F +import numpy as np +import igraph +from set_encoder.setenc_models import SetPool + + +class GeneratorModel(nn.Module): + def __init__(self, args, graph_config): + super(GeneratorModel, self).__init__() + self.max_n = graph_config['max_n'] # maximum number of vertices + self.nvt = graph_config['num_vertex_type'] # number of vertex types + self.START_TYPE = graph_config['START_TYPE'] + self.END_TYPE = graph_config['END_TYPE'] + self.hs = args.hs # hidden state size of each vertex + self.nz = args.nz # size of latent representation z + self.gs = args.hs # size of graph state + self.bidir = True # whether to use bidirectional encoding + self.vid = True + self.device = None + self.num_sample = args.num_sample + + if self.vid: + self.vs = self.hs + self.max_n # vertex state size = hidden state + vid + else: + self.vs = self.hs + + # 0. encoding-related + self.grue_forward = nn.GRUCell(self.nvt, self.hs) # encoder GRU + self.grue_backward = nn.GRUCell(self.nvt, self.hs) # backward encoder GRU + self.enc_g_mu = nn.Linear(self.gs, self.nz) # latent mean + self.enc_g_var = nn.Linear(self.gs, self.nz) # latent var + self.fc1 = nn.Linear(self.gs, self.nz) # latent mean + self.fc2 = nn.Linear(self.gs, self.nz) # latent logvar + + # 1. decoding-related + self.grud = nn.GRUCell(self.nvt, self.hs) # decoder GRU + self.fc3 = nn.Linear(self.nz, self.hs) # from latent z to initial hidden state h0 + self.add_vertex = nn.Sequential( + nn.Linear(self.hs, self.hs * 2), + nn.ReLU(), + nn.Linear(self.hs * 2, self.nvt) + ) # which type of new vertex to add f(h0, hg) + self.add_edge = nn.Sequential( + nn.Linear(self.hs * 2, self.hs * 4), + nn.ReLU(), + nn.Linear(self.hs * 4, 1) + ) # whether to add edge between v_i and v_new, f(hvi, hnew) + self.decoding_gate = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.decoding_mapper = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) # disable bias to ensure padded zeros also mapped to zeros + + # 2. gate-related + self.gate_forward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.gate_backward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.mapper_forward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) # disable bias to ensure padded zeros also mapped to zeros + self.mapper_backward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) + + # 3. bidir-related, to unify sizes + if self.bidir: + self.hv_unify = nn.Sequential( + nn.Linear(self.hs * 2, self.hs), + ) + self.hg_unify = nn.Sequential( + nn.Linear(self.gs * 2, self.gs), + ) + + # 4. other + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + self.tanh = nn.Tanh() + self.logsoftmax1 = nn.LogSoftmax(1) + + # 6. predictor + np = self.gs + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + def get_device(self): + if self.device is None: + self.device = next(self.parameters()).device + return self.device + + def _get_zeros(self, n, length): + return torch.zeros(n, length).to(self.get_device()) # get a zero hidden state + + def _get_zero_hidden(self, n=1): + return self._get_zeros(n, self.hs) # get a zero hidden state + + def _one_hot(self, idx, length): + if type(idx) in [list, range]: + if idx == []: + return None + idx = torch.LongTensor(idx).unsqueeze(0).t() + x = torch.zeros((len(idx), length) + ).scatter_(1, idx, 1).to(self.get_device()) + else: + idx = torch.LongTensor([idx]).unsqueeze(0) + x = torch.zeros((1, length) + ).scatter_(1, idx, 1).to(self.get_device()) + return x + + def _gated(self, h, gate, mapper): + return gate(h) * mapper(h) + + def _collate_fn(self, G): + return [g.copy() for g in G] + + def _propagate_to(self, G, v, propagator, + H=None, reverse=False, gate=None, mapper=None): + # propagate messages to vertex index v for all graphs in G + # return the new messages (states) at v + G = [g for g in G if g.vcount() > v] + if len(G) == 0: + return + if H is not None: + idx = [i for i, g in enumerate(G) if g.vcount() > v] + H = H[idx] + v_types = [g.vs[v]['type'] for g in G] + X = self._one_hot(v_types, self.nvt) + H_name = 'H_forward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.predecessors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.predecessors(v), self.max_n) for g in G] + if reverse: + H_name = 'H_backward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.successors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.successors(v), self.max_n) for g in G] + gate, mapper = self.gate_backward, self.mapper_backward + else: + H_name = 'H_forward' # name of the hidden states attribute + H_pred = [ + [g.vs[x][H_name] for x in g.predecessors(v)] for g in G] + if self.vid: + vids = [ + self._one_hot(g.predecessors(v), self.max_n) for g in G] + if gate is None: + gate, mapper = self.gate_forward, self.mapper_forward + if self.vid: + H_pred = [[torch.cat( + [x[i], y[i:i + 1]], 1) for i in range(len(x)) + ] for x, y in zip(H_pred, vids)] + # if h is not provided, use gated sum of v's predecessors' states as the input hidden state + if H is None: + max_n_pred = max([len(x) for x in H_pred]) # maximum number of predecessors + if max_n_pred == 0: + H = self._get_zero_hidden(len(G)) + else: + H_pred = [torch.cat(h_pred + + [self._get_zeros(max_n_pred - len(h_pred), + self.vs)], 0).unsqueeze(0) + for h_pred in H_pred] # pad all to same length + H_pred = torch.cat(H_pred, 0) # batch * max_n_pred * vs + H = self._gated(H_pred, gate, mapper).sum(1) # batch * hs + Hv = propagator(X, H) + for i, g in enumerate(G): + g.vs[v][H_name] = Hv[i:i + 1] + return Hv + + def _propagate_from(self, G, v, propagator, H0=None, reverse=False): + # perform a series of propagation_to steps starting from v following a topo order + # assume the original vertex indices are in a topological order + if reverse: + prop_order = range(v, -1, -1) + else: + prop_order = range(v, self.max_n) + Hv = self._propagate_to(G, v, propagator, H0, reverse=reverse) # the initial vertex + for v_ in prop_order[1:]: + self._propagate_to(G, v_, propagator, reverse=reverse) + return Hv + + def _update_v(self, G, v, H0=None): + # perform a forward propagation step at v when decoding to update v's state + # self._propagate_to(G, v, self.grud, H0, reverse=False) + self._propagate_to(G, v, self.grud, H0, + reverse=False, gate=self.decoding_gate, + mapper=self.decoding_mapper) + return + + def _get_vertex_state(self, G, v): + # get the vertex states at v + Hv = [] + for g in G: + if v >= g.vcount(): + hv = self._get_zero_hidden() + else: + hv = g.vs[v]['H_forward'] + Hv.append(hv) + Hv = torch.cat(Hv, 0) + return Hv + + def _get_graph_state(self, G, decode=False): + # get the graph states + # when decoding, use the last generated vertex's state as the graph state + # when encoding, use the ending vertex state or unify the starting and ending vertex states + Hg = [] + for g in G: + hg = g.vs[g.vcount() - 1]['H_forward'] + if self.bidir and not decode: # decoding never uses backward propagation + hg_b = g.vs[0]['H_backward'] + hg = torch.cat([hg, hg_b], 1) + Hg.append(hg) + Hg = torch.cat(Hg, 0) + if self.bidir and not decode: + Hg = self.hg_unify(Hg) + return Hg + + def graph_encode(self, G): + # encode graphs G into latent vectors + if type(G) != list: + G = [G] + self._propagate_from(G, 0, self.grue_forward, + H0=self._get_zero_hidden(len(G)), reverse=False) + if self.bidir: + self._propagate_from(G, self.max_n - 1, self.grue_backward, + H0=self._get_zero_hidden(len(G)), reverse=True) + Hg = self._get_graph_state(G) + mu, logvar = self.enc_g_mu(Hg), self.enc_g_var(Hg) + return mu, logvar + + def set_encode(self, X): + proto_batch = [] + for x in X: # X.shape: [32, 400, 512] + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + mu, logvar = self.fc1(v), self.fc2(v) + return mu, logvar + + def reparameterize(self, mu, logvar, eps_scale=0.01): + # return z ~ N(mu, std) + if self.training: + std = logvar.mul(0.5).exp_() + eps = torch.randn_like(std) * eps_scale + return eps.mul(std).add_(mu) + else: + return mu + + def _get_edge_score(self, Hvi, H, H0): + # compute scores for edges from vi based on Hvi, H (current vertex) and H0 + # in most cases, H0 need not be explicitly included since Hvi and H contain its information + return self.sigmoid(self.add_edge(torch.cat([Hvi, H], -1))) + + def graph_decode(self, z, stochastic=True): + # decode latent vectors z back to graphs + # if stochastic=True, stochastically sample each action from the predicted distribution; + # otherwise, select argmax action deterministically. + H0 = self.tanh(self.fc3(z)) # or relu activation, similar performance + G = [igraph.Graph(directed=True) for _ in range(len(z))] + for g in G: + g.add_vertex(type=self.START_TYPE) + self._update_v(G, 0, H0) + finished = [False] * len(G) + for idx in range(1, self.max_n): + # decide the type of the next added vertex + if idx == self.max_n - 1: # force the last node to be end_type + new_types = [self.END_TYPE] * len(G) + else: + Hg = self._get_graph_state(G, decode=True) + type_scores = self.add_vertex(Hg) + if stochastic: + type_probs = F.softmax(type_scores, 1 + ).cpu().detach().numpy() + new_types = [np.random.choice(range(self.nvt), + p=type_probs[i]) for i in range(len(G))] + else: + new_types = torch.argmax(type_scores, 1) + new_types = new_types.flatten().tolist() + for i, g in enumerate(G): + if not finished[i]: + g.add_vertex(type=new_types[i]) + self._update_v(G, idx) + + # decide connections + edge_scores = [] + for vi in range(idx - 1, -1, -1): + Hvi = self._get_vertex_state(G, vi) + H = self._get_vertex_state(G, idx) + ei_score = self._get_edge_score(Hvi, H, H0) + if stochastic: + random_score = torch.rand_like(ei_score) + decisions = random_score < ei_score + else: + decisions = ei_score > 0.5 + for i, g in enumerate(G): + if finished[i]: + continue + if new_types[i] == self.END_TYPE: + # if new node is end_type, connect it to all loose-end vertices (out_degree==0) + end_vertices = set([ + v.index for v in g.vs.select(_outdegree_eq=0) + if v.index != g.vcount() - 1]) + for v in end_vertices: + g.add_edge(v, g.vcount() - 1) + finished[i] = True + continue + if decisions[i, 0]: + g.add_edge(vi, g.vcount() - 1) + self._update_v(G, idx) + + for g in G: + del g.vs['H_forward'] # delete hidden states to save GPU memory + return G + + def loss(self, mu, logvar, G_true, beta=0.005): + # compute the loss of decoding mu and logvar to true graphs using teacher forcing + # ensure when computing the loss of step i, steps 0 to i-1 are correct + z = self.reparameterize(mu, logvar) + H0 = self.tanh(self.fc3(z)) # or relu activation, similar performance + G = [igraph.Graph(directed=True) for _ in range(len(z))] + for g in G: + g.add_vertex(type=self.START_TYPE) + self._update_v(G, 0, H0) + res = 0 # log likelihood + for v_true in range(1, self.max_n): + # calculate the likelihood of adding true types of nodes + # use start type to denote padding vertices since start type only appears for vertex 0 + # and will never be a true type for later vertices, thus it's free to use + true_types = [g_true.vs[v_true]['type'] + if v_true < g_true.vcount() + else self.START_TYPE for g_true in G_true] + Hg = self._get_graph_state(G, decode=True) + type_scores = self.add_vertex(Hg) + # vertex log likelihood + vll = self.logsoftmax1(type_scores)[ + np.arange(len(G)), true_types].sum() + res = res + vll + for i, g in enumerate(G): + if true_types[i] != self.START_TYPE: + g.add_vertex(type=true_types[i]) + self._update_v(G, v_true) + + # calculate the likelihood of adding true edges + true_edges = [] + for i, g_true in enumerate(G_true): + true_edges.append(g_true.get_adjlist(igraph.IN)[v_true] + if v_true < g_true.vcount() else []) + edge_scores = [] + for vi in range(v_true - 1, -1, -1): + Hvi = self._get_vertex_state(G, vi) + H = self._get_vertex_state(G, v_true) + ei_score = self._get_edge_score(Hvi, H, H0) + edge_scores.append(ei_score) + for i, g in enumerate(G): + if vi in true_edges[i]: + g.add_edge(vi, v_true) + self._update_v(G, v_true) + edge_scores = torch.cat(edge_scores[::-1], 1) + + ground_truth = torch.zeros_like(edge_scores) + idx1 = [i for i, x in enumerate(true_edges) + for _ in range(len(x))] + idx2 = [xx for x in true_edges for xx in x] + ground_truth[idx1, idx2] = 1.0 + + # edges log-likelihood + ell = - F.binary_cross_entropy( + edge_scores, ground_truth, reduction='sum') + res = res + ell + + res = -res # convert likelihood to loss + kld = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) + return res + beta * kld, res, kld \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_generator_checkpoint.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_generator_checkpoint.py new file mode 100644 index 0000000..10715a5 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_generator_checkpoint.py @@ -0,0 +1,37 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +file_name = 'ckpt_120.pt' +dir_path = 'results/generator/model' +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, file_name) +if not os.path.exists(file_name): + print(f"Downloading {file_name}\n") + download_file('https://www.dropbox.com/s/zss9yt034hen45h/ckpt_120.pt?dl=1', file_name) + print("Downloading done.\n") +else: + print(f"{file_name} has already been downloaded. Did not download twice.\n") + + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_generator_database.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_generator_database.py new file mode 100644 index 0000000..2c104ed --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_generator_database.py @@ -0,0 +1,38 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + + +file_name = 'collected_database.pt' +dir_path = 'data/generator/processed' +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, file_name) +if not os.path.exists(file_name): + print(f"Downloading generator {file_name}\n") + download_file('https://www.dropbox.com/s/zgip4aq0w2pkj49/generator_collected_database.pt?dl=1', file_name) + print("Downloading done.\n") +else: + print(f"{file_name} has already been downloaded. Did not download twice.\n") + + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_pets.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_pets.py new file mode 100644 index 0000000..7a78ed2 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_pets.py @@ -0,0 +1,43 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +dir_path = 'data/pets' +if not os.path.exists(dir_path): + os.makedirs(dir_path) + +full_name = os.path.join(dir_path, 'test15.pth') +if not os.path.exists(full_name): + print(f"Downloading {full_name}\n") + download_file('https://www.dropbox.com/s/kzmrwyyk5iaugv0/test15.pth?dl=1', full_name) + print("Downloading done.\n") +else: + print(f"{full_name} has already been downloaded. Did not download twice.\n") + +full_name = os.path.join(dir_path, 'train85.pth') +if not os.path.exists(full_name): + print(f"Downloading {full_name}\n") + download_file('https://www.dropbox.com/s/w7mikpztkamnw9s/train85.pth?dl=1', full_name) + print("Downloading done.\n") +else: + print(f"{full_name} has already been downloaded. Did not download twice.\n") diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_predictor_checkpoint.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_predictor_checkpoint.py new file mode 100644 index 0000000..e11bc97 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_predictor_checkpoint.py @@ -0,0 +1,35 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +file_name = 'ckpt_max_corr.pt' +dir_path = 'results/predictor/model' +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, file_name) +if not os.path.exists(file_name): + print(f"Downloading {file_name}\n") + download_file('https://www.dropbox.com/s/ycm4jaojgswp0zm/ckpt_max_corr.pt?dl=1', file_name) + print("Downloading done.\n") +else: + print(f"{file_name} has already been downloaded. Did not download twice.\n") diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_predictor_database.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_predictor_database.py new file mode 100644 index 0000000..d30383d --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_predictor_database.py @@ -0,0 +1,38 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + + +file_name = 'collected_database.pt' +dir_path = 'data/predictor/processed' +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, file_name) +if not os.path.exists(file_name): + print(f"Downloading predictor {file_name}\n") + download_file('https://www.dropbox.com/s/ycm4jaojgswp0zm/ckpt_max_corr.pt?dl=1', file_name) + print("Downloading done.\n") +else: + print(f"{file_name} has already been downloaded. Did not download twice.\n") + + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_preprocessed_data.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_preprocessed_data.py new file mode 100644 index 0000000..ebbfcec --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/get_files/get_preprocessed_data.py @@ -0,0 +1,47 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +dir_path = 'data' +if not os.path.exists(dir_path): + os.makedirs(dir_path) + +def get_preprocessed_data(file_name, url): + print(f"Downloading {file_name} datasets\n") + full_name = os.path.join(dir_path, file_name) + download_file(url, full_name) + print("Downloading done.\n") + + +for file_name, url in [ + ('imgnet32bylabel.pt', 'https://www.dropbox.com/s/7r3hpugql8qgi9d/imgnet32bylabel.pt?dl=1'), + ('aircraft100bylabel.pt', 'https://www.dropbox.com/s/nn6mlrk1jijg108/aircraft100bylabel.pt?dl=1'), + ('cifar100bylabel.pt', 'https://www.dropbox.com/s/y0xahxgzj29kffk/cifar100bylabel.pt?dl=1'), + ('cifar10bylabel.pt', 'https://www.dropbox.com/s/wt1pcwi991xyhwr/cifar10bylabel.pt?dl=1'), + ('imgnet32bylabel.pt', 'https://www.dropbox.com/s/7r3hpugql8qgi9d/imgnet32bylabel.pt?dl=1'), + ('petsbylabel.pt', 'https://www.dropbox.com/s/mxh6qz3grhy7wcn/petsbylabel.pt?dl=1'), + ('mnistbylabel.pt', 'https://www.dropbox.com/s/86rbuic7a7y34e4/mnistbylabel.pt?dl=1'), + ('svhnbylabel.pt', 'https://www.dropbox.com/s/yywaelhrsl6egvd/svhnbylabel.pt?dl=1') + ]: + + get_preprocessed_data(file_name, url) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/loader.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/loader.py new file mode 100644 index 0000000..76723d9 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/loader.py @@ -0,0 +1,149 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import os +import torch +from tqdm import tqdm +from torch.utils.data import Dataset +from torch.utils.data import DataLoader + + +def get_meta_train_loader(batch_size, data_path, num_sample, is_pred=False): + dataset = MetaTrainDatabase(data_path, num_sample, is_pred) + print(f'==> The number of tasks for meta-training: {len(dataset)}') + + loader = DataLoader(dataset=dataset, + batch_size=batch_size, + shuffle=True, + num_workers=1, + collate_fn=collate_fn) + return loader + + +def get_meta_test_loader(data_path, data_name, num_class=None, is_pred=False): + dataset = MetaTestDataset(data_path, data_name, num_class) + print(f'==> Meta-Test dataset {data_name}') + + loader = DataLoader(dataset=dataset, + batch_size=100, + shuffle=False, + num_workers=1) + return loader + + +class MetaTrainDatabase(Dataset): + def __init__(self, data_path, num_sample, is_pred=False): + self.mode = 'train' + self.acc_norm = True + self.num_sample = num_sample + self.x = torch.load(os.path.join(data_path, 'imgnet32bylabel.pt')) + + self.dpath = '{}/{}/processed/'.format(data_path, 'predictor' if is_pred else 'generator') + self.dname = f'database_219152_14.0K' + + if not os.path.exists(self.dpath + f'{self.dname}_train.pt'): + raise ValueError('') + database = torch.load(self.dpath + f'{self.dname}.pt') + + rand_idx = torch.randperm(len(database)) + test_len = int(len(database) * 0.15) + idxlst = {'test': rand_idx[:test_len], + 'valid': rand_idx[test_len:2 * test_len], + 'train': rand_idx[2 * test_len:]} + + for m in ['train', 'valid', 'test']: + acc, graph, cls, net, flops = [], [], [], [], [] + for idx in tqdm(idxlst[m].tolist(), desc=f'data-{m}'): + acc.append(database[idx]['top1']) + net.append(database[idx]['net']) + cls.append(database[idx]['class']) + flops.append(database[idx]['flops']) + if m == 'train': + mean = torch.mean(torch.tensor(acc)).item() + std = torch.std(torch.tensor(acc)).item() + torch.save({'acc': acc, + 'class': cls, + 'net': net, + 'flops': flops, + 'mean': mean, + 'std': std}, + self.dpath + f'{self.dname}_{m}.pt') + + self.set_mode(self.mode) + + def set_mode(self, mode): + self.mode = mode + data = torch.load(self.dpath + f'{self.dname}_{self.mode}.pt') + self.acc = data['acc'] + self.cls = data['class'] + self.net = data['net'] + self.flops = data['flops'] + self.mean = data['mean'] + self.std = data['std'] + + def __len__(self): + return len(self.acc) + + def __getitem__(self, index): + data = [] + classes = self.cls[index] + acc = self.acc[index] + graph = self.net[index] + + for i, cls in enumerate(classes): + cx = self.x[cls.item()][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + x = torch.cat(data) + if self.acc_norm: + acc = ((acc - self.mean) / self.std) / 100.0 + else: + acc = acc / 100.0 + return x, graph, torch.tensor(acc).view(1, 1) + + +class MetaTestDataset(Dataset): + def __init__(self, data_path, data_name, num_sample, num_class=None): + self.num_sample = num_sample + self.data_name = data_name + if data_name == 'aircraft': + data_name = 'aircraft100' + num_class_dict = { + 'cifar100': 100, + 'cifar10': 10, + 'mnist': 10, + 'aircraft100': 30, + 'svhn': 10, + 'pets': 37 + } + # 'aircraft30': 30, + # 'aircraft100': 100, + + if num_class is not None: + self.num_class = num_class + else: + self.num_class = num_class_dict[data_name] + + self.x = torch.load(os.path.join(data_path, f'{data_name}bylabel.pt')) + + def __len__(self): + return 1000000 + + def __getitem__(self, index): + data = [] + classes = list(range(self.num_class)) + for cls in classes: + cx = self.x[cls][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + x = torch.cat(data) + return x + + +def collate_fn(batch): + # x = torch.stack([item[0] for item in batch]) + # graph = [item[1] for item in batch] + # acc = torch.stack([item[2] for item in batch]) + return batch diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/main.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/main.py new file mode 100644 index 0000000..c028ba0 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/main.py @@ -0,0 +1,48 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +import random +import numpy as np +import torch +from parser import get_parser +from generator import Generator +from predictor import Predictor + +def main(): + args = get_parser() + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu + args.device = torch.device("cuda:0") + torch.cuda.manual_seed(args.seed) + torch.manual_seed(args.seed) + np.random.seed(args.seed) + random.seed(args.seed) + + if not os.path.exists(args.save_path): + os.makedirs(args.save_path) + args.model_path = os.path.join(args.save_path, args.model_name, 'model') + if not os.path.exists(args.model_path): + os.makedirs(args.model_path) + + if args.model_name == 'generator': + g = Generator(args) + if args.test: + args.model_path = os.path.join(args.save_path, 'predictor', 'model') + hs = args.hs + args.hs = 512 + p = Predictor(args) + args.model_path = os.path.join(args.save_path, args.model_name, 'model') + args.hs = hs + g.meta_test(p) + else: + g.meta_train() + elif args.model_name == 'predictor': + p = Predictor(args) + p.meta_train() + else: + raise ValueError('You should select generator|predictor|train_arch') + + +if __name__ == '__main__': + main() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/metad2a_utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/metad2a_utils.py new file mode 100644 index 0000000..6078c6c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/metad2a_utils.py @@ -0,0 +1,344 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import os +import time +import igraph +import random +import numpy as np +import scipy.stats +import argparse +import torch + + +def load_graph_config(graph_data_name, nvt, data_path): + max_n=20 + graph_config = {} + graph_config['num_vertex_type'] = nvt + 2 # original types + start/end types + graph_config['max_n'] = max_n + 2 # maximum number of nodes + graph_config['START_TYPE'] = 0 # predefined start vertex type + graph_config['END_TYPE'] = 1 # predefined end vertex type + + return graph_config + + +type_dict = {'2-3-3': 0, '2-3-4': 1, '2-3-6': 2, + '2-5-3': 3, '2-5-4': 4, '2-5-6': 5, + '2-7-3': 6, '2-7-4': 7, '2-7-6': 8, + '3-3-3': 9, '3-3-4': 10, '3-3-6': 11, + '3-5-3': 12, '3-5-4': 13, '3-5-6': 14, + '3-7-3': 15, '3-7-4': 16, '3-7-6': 17, + '4-3-3': 18, '4-3-4': 19, '4-3-6': 20, + '4-5-3': 21, '4-5-4': 22, '4-5-6': 23, + '4-7-3': 24, '4-7-4': 25, '4-7-6': 26} + +edge_dict = {2: (2, 3, 3), 3: (2, 3, 4), 4: (2, 3, 6), + 5: (2, 5, 3), 6: (2, 5, 4), 7: (2, 5, 6), + 8: (2, 7, 3), 9: (2, 7, 4), 10: (2, 7, 6), + 11: (3, 3, 3), 12: (3, 3, 4), 13: (3, 3, 6), + 14: (3, 5, 3), 15: (3, 5, 4), 16: (3, 5, 6), + 17: (3, 7, 3), 18: (3, 7, 4), 19: (3, 7, 6), + 20: (4, 3, 3), 21: (4, 3, 4), 22: (4, 3, 6), + 23: (4, 5, 3), 24: (4, 5, 4), 25: (4, 5, 6), + 26: (4, 7, 3), 27: (4, 7, 4), 28: (4, 7, 6)} + + +def decode_ofa_mbv3_to_igraph(matrix): + # 5 stages, 4 layers for each stage + # d: 2, 3, 4 + # e: 3, 4, 6 + # k: 3, 5, 7 + + # stage_depth to one hot + num_stage = 5 + num_layer = 4 + + node_types = torch.zeros(num_stage * num_layer) + + d = [] + for i in range(num_stage): + for j in range(num_layer): + d.append(matrix['d'][i]) + for i, (ks, e, d) in enumerate(zip( + matrix['ks'], matrix['e'], d)): + node_types[i] = type_dict[f'{d}-{ks}-{e}'] + + n = num_stage * num_layer + g = igraph.Graph(directed=True) + g.add_vertices(n + 2) # + in/out nodes + g.vs[0]['type'] = 0 + for i, v in enumerate(node_types): + g.vs[i + 1]['type'] = v + 2 # in node: 0, out node: 1 + g.add_edge(i, i + 1) + g.vs[n + 1]['type'] = 1 + g.add_edge(n, n + 1) + return g, n + 2 + + +def decode_ofa_mbv3_str_to_igraph(gen_str): + # 5 stages, 4 layers for each stage + # d: 2, 3, 4 + # e: 3, 4, 6 + # k: 3, 5, 7 + + # stage_depth to one hot + num_stage = 5 + num_layer = 4 + + node_types = torch.zeros(num_stage * num_layer) + + d = [] + split_str = gen_str.split('_') + for i, s in enumerate(split_str): + if s == '0-0-0': + node_types[i] = random.randint(0, 26) + else: + node_types[i] = type_dict[s] + + n = num_stage * num_layer + g = igraph.Graph(directed=True) + g.add_vertices(n + 2) # + in/out nodes + g.vs[0]['type'] = 0 + for i, v in enumerate(node_types): + g.vs[i + 1]['type'] = v + 2 # in node: 0, out node: 1 + g.add_edge(i, i + 1) + g.vs[n + 1]['type'] = 1 + g.add_edge(n, n + 1) + return g + + +def is_valid_ofa_mbv3(g, START_TYPE=0, END_TYPE=1): + # first need to be a valid DAG computation graph + msg = '' + res = is_valid_DAG(g, START_TYPE, END_TYPE) + # in addition, node i must connect to node i+1 + res = res and len(g.vs['type']) == 22 + if not res: + return res + msg += '{} ({}) '.format(g.vs['type'][1:-1], len(g.vs['type'])) + + for i in range(5): + if ((g.vs['type'][1:-1][i * 4]) - 2) // 9 == 0: + for j in range(1, 4): + res = res and ((g.vs['type'][1:-1][i * 4 + j]) - 2) // 9 == 0 + + elif ((g.vs['type'][1:-1][i * 4]) - 2) // 9 == 1: + for j in range(1, 4): + res = res and ((g.vs['type'][1:-1][i * 4 + j]) - 2) // 9 == 1 + + elif ((g.vs['type'][1:-1][i * 4]) - 2) // 9 == 2: + for j in range(1, 4): + res = res and ((g.vs['type'][1:-1][i * 4 + j]) - 2) // 9 == 2 + else: + raise ValueError + return res + + +def is_valid_DAG(g, START_TYPE=0, END_TYPE=1): + res = g.is_dag() + n_start, n_end = 0, 0 + for v in g.vs: + if v['type'] == START_TYPE: + n_start += 1 + elif v['type'] == END_TYPE: + n_end += 1 + if v.indegree() == 0 and v['type'] != START_TYPE: + return False + if v.outdegree() == 0 and v['type'] != END_TYPE: + return False + return res and n_start == 1 and n_end == 1 + + +def decode_igraph_to_ofa_mbv3(g): + if not is_valid_ofa_mbv3(g, START_TYPE=0, END_TYPE=1): + return None + + graph = {'ks': [], 'e': [], 'd': [4, 4, 4, 4, 4]} + for i, edge_type in enumerate(g.vs['type'][1:-1]): + edge_type = int(edge_type) + d, ks, e = edge_dict[edge_type] + graph['ks'].append(ks) + graph['e'].append(e) + graph['d'][i // 4] = d + return graph + + +class Accumulator(): + def __init__(self, *args): + self.args = args + self.argdict = {} + for i, arg in enumerate(args): + self.argdict[arg] = i + self.sums = [0] * len(args) + self.cnt = 0 + + def accum(self, val): + val = [val] if type(val) is not list else val + val = [v for v in val if v is not None] + assert (len(val) == len(self.args)) + for i in range(len(val)): + if torch.is_tensor(val[i]): + val[i] = val[i].item() + self.sums[i] += val[i] + self.cnt += 1 + + def clear(self): + self.sums = [0] * len(self.args) + self.cnt = 0 + + def get(self, arg, avg=True): + i = self.argdict.get(arg, -1) + assert (i is not -1) + if avg: + return self.sums[i] / (self.cnt + 1e-8) + else: + return self.sums[i] + + def print_(self, header=None, time=None, + logfile=None, do_not_print=[], as_int=[], + avg=True): + msg = '' if header is None else header + ': ' + if time is not None: + msg += ('(%.3f secs), ' % time) + + args = [arg for arg in self.args if arg not in do_not_print] + arg = [] + for arg in args: + val = self.sums[self.argdict[arg]] + if avg: + val /= (self.cnt + 1e-8) + if arg in as_int: + msg += ('%s %d, ' % (arg, int(val))) + else: + msg += ('%s %.4f, ' % (arg, val)) + print(msg) + + if logfile is not None: + logfile.write(msg + '\n') + logfile.flush() + + def add_scalars(self, summary, header=None, tag_scalar=None, + step=None, avg=True, args=None): + for arg in self.args: + val = self.sums[self.argdict[arg]] + if avg: + val /= (self.cnt + 1e-8) + else: + val = val + tag = f'{header}/{arg}' if header is not None else arg + if tag_scalar is not None: + summary.add_scalars(main_tag=tag, + tag_scalar_dict={tag_scalar: val}, + global_step=step) + else: + summary.add_scalar(tag=tag, + scalar_value=val, + global_step=step) + + +class Log: + def __init__(self, args, logf, summary=None): + self.args = args + self.logf = logf + self.summary = summary + self.stime = time.time() + self.ep_sttime = None + + def print(self, logger, epoch, tag=None, avg=True): + if tag == 'train': + ct = time.time() - self.ep_sttime + tt = time.time() - self.stime + msg = f'[total {tt:6.2f}s (ep {ct:6.2f}s)] epoch {epoch:3d}' + print(msg) + self.logf.write(msg + '\n') + logger.print_(header=tag, logfile=self.logf, avg=avg) + + if self.summary is not None: + logger.add_scalars( + self.summary, header=tag, step=epoch, avg=avg) + logger.clear() + + def print_args(self): + argdict = vars(self.args) + print(argdict) + for k, v in argdict.items(): + self.logf.write(k + ': ' + str(v) + '\n') + self.logf.write('\n') + + def set_time(self): + self.stime = time.time() + + def save_time_log(self): + ct = time.time() - self.stime + msg = f'({ct:6.2f}s) meta-training phase done' + print(msg) + self.logf.write(msg + '\n') + + def print_pred_log(self, loss, corr, tag, epoch=None, max_corr_dict=None): + if tag == 'train': + ct = time.time() - self.ep_sttime + tt = time.time() - self.stime + msg = f'[total {tt:6.2f}s (ep {ct:6.2f}s)] epoch {epoch:3d}' + self.logf.write(msg + '\n'); + print(msg); + self.logf.flush() + # msg = f'ep {epoch:3d} ep time {time.time() - ep_sttime:8.2f} ' + # msg += f'time {time.time() - sttime:6.2f} ' + if max_corr_dict is not None: + max_corr = max_corr_dict['corr'] + max_loss = max_corr_dict['loss'] + msg = f'{tag}: loss {loss:.6f} ({max_loss:.6f}) ' + msg += f'corr {corr:.4f} ({max_corr:.4f})' + else: + msg = f'{tag}: loss {loss:.6f} corr {corr:.4f}' + self.logf.write(msg + '\n'); + print(msg); + self.logf.flush() + + def max_corr_log(self, max_corr_dict): + corr = max_corr_dict['corr'] + loss = max_corr_dict['loss'] + epoch = max_corr_dict['epoch'] + msg = f'[epoch {epoch}] max correlation: {corr:.4f}, loss: {loss:.6f}' + self.logf.write(msg + '\n'); + print(msg); + self.logf.flush() + + +def get_log(epoch, loss, y_pred, y, acc_std, acc_mean, tag='train'): + msg = f'[{tag}] Ep {epoch} loss {loss.item() / len(y):0.4f} ' + msg += f'pacc {y_pred[0]:0.4f}' + msg += f'({y_pred[0] * 100.0 * acc_std + acc_mean:0.4f}) ' + msg += f'acc {y[0]:0.4f}({y[0] * 100 * acc_std + acc_mean:0.4f})' + return msg + + +def load_model(model, model_path, load_epoch=None, load_max_pt=None): + if load_max_pt is not None: + ckpt_path = os.path.join(model_path, load_max_pt) + else: + ckpt_path = os.path.join(model_path, f'ckpt_{load_epoch}.pt') + + print(f"==> load checkpoint for MetaD2A predictor: {ckpt_path} ...") + model.cpu() + model.load_state_dict(torch.load(ckpt_path)) + + +def save_model(epoch, model, model_path, max_corr=None): + print("==> save current model...") + if max_corr is not None: + torch.save(model.cpu().state_dict(), + os.path.join(model_path, 'ckpt_max_corr.pt')) + else: + torch.save(model.cpu().state_dict(), + os.path.join(model_path, f'ckpt_{epoch}.pt')) + + +def mean_confidence_interval(data, confidence=0.95): + a = 1.0 * np.array(data) + n = len(a) + m, se = np.mean(a), scipy.stats.sem(a) + h = se * scipy.stats.t.ppf((1 + confidence) / 2., n - 1) + return m, h \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/__init__.py new file mode 100644 index 0000000..92c720b --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/__init__.py @@ -0,0 +1,5 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .imagenet import * diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/base_provider.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/base_provider.py new file mode 100644 index 0000000..95dc18c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/base_provider.py @@ -0,0 +1,56 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import numpy as np +import torch + +__all__ = ['DataProvider'] + + +class DataProvider: + SUB_SEED = 937162211 # random seed for sampling subset + VALID_SEED = 2147483647 # random seed for the validation set + + @staticmethod + def name(): + """ Return name of the dataset """ + raise NotImplementedError + + @property + def data_shape(self): + """ Return shape as python list of one data entry """ + raise NotImplementedError + + @property + def n_classes(self): + """ Return `int` of num classes """ + raise NotImplementedError + + @property + def save_path(self): + """ local path to save the data """ + raise NotImplementedError + + @property + def data_url(self): + """ link to download the data """ + raise NotImplementedError + + @staticmethod + def random_sample_valid_set(train_size, valid_size): + assert train_size > valid_size + + g = torch.Generator() + g.manual_seed(DataProvider.VALID_SEED) # set random seed before sampling validation set + rand_indexes = torch.randperm(train_size, generator=g).tolist() + + valid_indexes = rand_indexes[:valid_size] + train_indexes = rand_indexes[valid_size:] + return train_indexes, valid_indexes + + @staticmethod + def labels_to_one_hot(n_classes, labels): + new_labels = np.zeros((labels.shape[0], n_classes), dtype=np.float32) + new_labels[range(labels.shape[0]), labels] = np.ones(labels.shape) + return new_labels diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/imagenet.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/imagenet.py new file mode 100644 index 0000000..92d8180 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/data_providers/imagenet.py @@ -0,0 +1,225 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import warnings +import os +import math +import numpy as np +import torch.utils.data +import torchvision.transforms as transforms +import torchvision.datasets as datasets + +from .base_provider import DataProvider +from ofa_local.utils.my_dataloader import MyRandomResizedCrop, MyDistributedSampler + +__all__ = ['ImagenetDataProvider'] + + +class ImagenetDataProvider(DataProvider): + DEFAULT_PATH = '/dataset/imagenet' + + def __init__(self, save_path=None, train_batch_size=256, test_batch_size=512, valid_size=None, n_worker=32, + resize_scale=0.08, distort_color=None, image_size=224, + num_replicas=None, rank=None): + + warnings.filterwarnings('ignore') + self._save_path = save_path + + self.image_size = image_size # int or list of int + self.distort_color = 'None' if distort_color is None else distort_color + self.resize_scale = resize_scale + + self._valid_transform_dict = {} + if not isinstance(self.image_size, int): + from ofa.utils.my_dataloader import MyDataLoader + assert isinstance(self.image_size, list) + self.image_size.sort() # e.g., 160 -> 224 + MyRandomResizedCrop.IMAGE_SIZE_LIST = self.image_size.copy() + MyRandomResizedCrop.ACTIVE_SIZE = max(self.image_size) + + for img_size in self.image_size: + self._valid_transform_dict[img_size] = self.build_valid_transform(img_size) + self.active_img_size = max(self.image_size) # active resolution for test + valid_transforms = self._valid_transform_dict[self.active_img_size] + train_loader_class = MyDataLoader # randomly sample image size for each batch of training image + else: + self.active_img_size = self.image_size + valid_transforms = self.build_valid_transform() + train_loader_class = torch.utils.data.DataLoader + + train_dataset = self.train_dataset(self.build_train_transform()) + + if valid_size is not None: + if not isinstance(valid_size, int): + assert isinstance(valid_size, float) and 0 < valid_size < 1 + valid_size = int(len(train_dataset) * valid_size) + + valid_dataset = self.train_dataset(valid_transforms) + train_indexes, valid_indexes = self.random_sample_valid_set(len(train_dataset), valid_size) + + if num_replicas is not None: + train_sampler = MyDistributedSampler(train_dataset, num_replicas, rank, True, np.array(train_indexes)) + valid_sampler = MyDistributedSampler(valid_dataset, num_replicas, rank, True, np.array(valid_indexes)) + else: + train_sampler = torch.utils.data.sampler.SubsetRandomSampler(train_indexes) + valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(valid_indexes) + + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True, + ) + self.valid = torch.utils.data.DataLoader( + valid_dataset, batch_size=test_batch_size, sampler=valid_sampler, + num_workers=n_worker, pin_memory=True, + ) + else: + if num_replicas is not None: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset, num_replicas, rank) + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True + ) + else: + self.train = train_loader_class( + train_dataset, batch_size=train_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + self.valid = None + + test_dataset = self.test_dataset(valid_transforms) + if num_replicas is not None: + test_sampler = torch.utils.data.distributed.DistributedSampler(test_dataset, num_replicas, rank) + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, sampler=test_sampler, num_workers=n_worker, pin_memory=True, + ) + else: + self.test = torch.utils.data.DataLoader( + test_dataset, batch_size=test_batch_size, shuffle=True, num_workers=n_worker, pin_memory=True, + ) + + if self.valid is None: + self.valid = self.test + + @staticmethod + def name(): + return 'imagenet' + + @property + def data_shape(self): + return 3, self.active_img_size, self.active_img_size # C, H, W + + @property + def n_classes(self): + return 1000 + + @property + def save_path(self): + if self._save_path is None: + self._save_path = self.DEFAULT_PATH + if not os.path.exists(self._save_path): + self._save_path = os.path.expanduser('~/dataset/imagenet') + return self._save_path + + @property + def data_url(self): + raise ValueError('unable to download %s' % self.name()) + + def train_dataset(self, _transforms): + return datasets.ImageFolder(self.train_path, _transforms) + + def test_dataset(self, _transforms): + return datasets.ImageFolder(self.valid_path, _transforms) + + @property + def train_path(self): + return os.path.join(self.save_path, 'train') + + @property + def valid_path(self): + return os.path.join(self.save_path, 'val') + + @property + def normalize(self): + return transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + def build_train_transform(self, image_size=None, print_log=True): + if image_size is None: + image_size = self.image_size + if print_log: + print('Color jitter: %s, resize_scale: %s, img_size: %s' % + (self.distort_color, self.resize_scale, image_size)) + + if isinstance(image_size, list): + resize_transform_class = MyRandomResizedCrop + print('Use MyRandomResizedCrop: %s, \t %s' % MyRandomResizedCrop.get_candidate_image_size(), + 'sync=%s, continuous=%s' % (MyRandomResizedCrop.SYNC_DISTRIBUTED, MyRandomResizedCrop.CONTINUOUS)) + else: + resize_transform_class = transforms.RandomResizedCrop + + # random_resize_crop -> random_horizontal_flip + train_transforms = [ + resize_transform_class(image_size, scale=(self.resize_scale, 1.0)), + transforms.RandomHorizontalFlip(), + ] + + # color augmentation (optional) + color_transform = None + if self.distort_color == 'torch': + color_transform = transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1) + elif self.distort_color == 'tf': + color_transform = transforms.ColorJitter(brightness=32. / 255., saturation=0.5) + if color_transform is not None: + train_transforms.append(color_transform) + + train_transforms += [ + transforms.ToTensor(), + self.normalize, + ] + + train_transforms = transforms.Compose(train_transforms) + return train_transforms + + def build_valid_transform(self, image_size=None): + if image_size is None: + image_size = self.active_img_size + return transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + self.normalize, + ]) + + def assign_active_img_size(self, new_img_size): + self.active_img_size = new_img_size + if self.active_img_size not in self._valid_transform_dict: + self._valid_transform_dict[self.active_img_size] = self.build_valid_transform() + # change the transform of the valid and test set + self.valid.dataset.transform = self._valid_transform_dict[self.active_img_size] + self.test.dataset.transform = self._valid_transform_dict[self.active_img_size] + + def build_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + # used for resetting BN running statistics + if self.__dict__.get('sub_train_%d' % self.active_img_size, None) is None: + if num_worker is None: + num_worker = self.train.num_workers + + n_samples = len(self.train.dataset) + g = torch.Generator() + g.manual_seed(DataProvider.SUB_SEED) + rand_indexes = torch.randperm(n_samples, generator=g).tolist() + + new_train_dataset = self.train_dataset( + self.build_train_transform(image_size=self.active_img_size, print_log=False)) + chosen_indexes = rand_indexes[:n_images] + if num_replicas is not None: + sub_sampler = MyDistributedSampler(new_train_dataset, num_replicas, rank, True, np.array(chosen_indexes)) + else: + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + sub_data_loader = torch.utils.data.DataLoader( + new_train_dataset, batch_size=batch_size, sampler=sub_sampler, + num_workers=num_worker, pin_memory=True, + ) + self.__dict__['sub_train_%d' % self.active_img_size] = [] + for images, labels in sub_data_loader: + self.__dict__['sub_train_%d' % self.active_img_size].append((images, labels)) + return self.__dict__['sub_train_%d' % self.active_img_size] diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/__init__.py new file mode 100644 index 0000000..4173c1e --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/__init__.py @@ -0,0 +1,6 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .dynamic_layers import * +from .dynamic_op import * diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/dynamic_layers.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/dynamic_layers.py new file mode 100644 index 0000000..f1fb0b9 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/dynamic_layers.py @@ -0,0 +1,632 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import copy +import torch +import torch.nn as nn +from collections import OrderedDict + +from ofa_local.utils.layers import MBConvLayer, ConvLayer, IdentityLayer, set_layer_from_config +from ofa_local.utils.layers import ResNetBottleneckBlock, LinearLayer +from ofa_local.utils import MyModule, val2list, get_net_device, build_activation, make_divisible, SEModule, MyNetwork +from .dynamic_op import DynamicSeparableConv2d, DynamicConv2d, DynamicBatchNorm2d, DynamicSE, DynamicGroupNorm +from .dynamic_op import DynamicLinear + +__all__ = [ + 'adjust_bn_according_to_idx', 'copy_bn', + 'DynamicMBConvLayer', 'DynamicConvLayer', 'DynamicLinearLayer', 'DynamicResNetBottleneckBlock' +] + + +def adjust_bn_according_to_idx(bn, idx): + bn.weight.data = torch.index_select(bn.weight.data, 0, idx) + bn.bias.data = torch.index_select(bn.bias.data, 0, idx) + if type(bn) in [nn.BatchNorm1d, nn.BatchNorm2d]: + bn.running_mean.data = torch.index_select(bn.running_mean.data, 0, idx) + bn.running_var.data = torch.index_select(bn.running_var.data, 0, idx) + + +def copy_bn(target_bn, src_bn): + feature_dim = target_bn.num_channels if isinstance(target_bn, nn.GroupNorm) else target_bn.num_features + + target_bn.weight.data.copy_(src_bn.weight.data[:feature_dim]) + target_bn.bias.data.copy_(src_bn.bias.data[:feature_dim]) + if type(src_bn) in [nn.BatchNorm1d, nn.BatchNorm2d]: + target_bn.running_mean.data.copy_(src_bn.running_mean.data[:feature_dim]) + target_bn.running_var.data.copy_(src_bn.running_var.data[:feature_dim]) + + +class DynamicLinearLayer(MyModule): + + def __init__(self, in_features_list, out_features, bias=True, dropout_rate=0): + super(DynamicLinearLayer, self).__init__() + + self.in_features_list = in_features_list + self.out_features = out_features + self.bias = bias + self.dropout_rate = dropout_rate + + if self.dropout_rate > 0: + self.dropout = nn.Dropout(self.dropout_rate, inplace=True) + else: + self.dropout = None + self.linear = DynamicLinear( + max_in_features=max(self.in_features_list), max_out_features=self.out_features, bias=self.bias + ) + + def forward(self, x): + if self.dropout is not None: + x = self.dropout(x) + return self.linear(x) + + @property + def module_str(self): + return 'DyLinear(%d, %d)' % (max(self.in_features_list), self.out_features) + + @property + def config(self): + return { + 'name': DynamicLinear.__name__, + 'in_features_list': self.in_features_list, + 'out_features': self.out_features, + 'bias': self.bias, + 'dropout_rate': self.dropout_rate, + } + + @staticmethod + def build_from_config(config): + return DynamicLinearLayer(**config) + + def get_active_subnet(self, in_features, preserve_weight=True): + sub_layer = LinearLayer(in_features, self.out_features, self.bias, dropout_rate=self.dropout_rate) + sub_layer = sub_layer.to(get_net_device(self)) + if not preserve_weight: + return sub_layer + + sub_layer.linear.weight.data.copy_( + self.linear.get_active_weight(self.out_features, in_features).data + ) + if self.bias: + sub_layer.linear.bias.data.copy_( + self.linear.get_active_bias(self.out_features).data + ) + return sub_layer + + def get_active_subnet_config(self, in_features): + return { + 'name': LinearLayer.__name__, + 'in_features': in_features, + 'out_features': self.out_features, + 'bias': self.bias, + 'dropout_rate': self.dropout_rate, + } + + +class DynamicMBConvLayer(MyModule): + + def __init__(self, in_channel_list, out_channel_list, + kernel_size_list=3, expand_ratio_list=6, stride=1, act_func='relu6', use_se=False): + super(DynamicMBConvLayer, self).__init__() + + self.in_channel_list = in_channel_list + self.out_channel_list = out_channel_list + + self.kernel_size_list = val2list(kernel_size_list) + self.expand_ratio_list = val2list(expand_ratio_list) + + self.stride = stride + self.act_func = act_func + self.use_se = use_se + + # build modules + max_middle_channel = make_divisible( + round(max(self.in_channel_list) * max(self.expand_ratio_list)), MyNetwork.CHANNEL_DIVISIBLE) + if max(self.expand_ratio_list) == 1: + self.inverted_bottleneck = None + else: + self.inverted_bottleneck = nn.Sequential(OrderedDict([ + ('conv', DynamicConv2d(max(self.in_channel_list), max_middle_channel)), + ('bn', DynamicBatchNorm2d(max_middle_channel)), + ('act', build_activation(self.act_func)), + ])) + + self.depth_conv = nn.Sequential(OrderedDict([ + ('conv', DynamicSeparableConv2d(max_middle_channel, self.kernel_size_list, self.stride)), + ('bn', DynamicBatchNorm2d(max_middle_channel)), + ('act', build_activation(self.act_func)) + ])) + if self.use_se: + self.depth_conv.add_module('se', DynamicSE(max_middle_channel)) + + self.point_linear = nn.Sequential(OrderedDict([ + ('conv', DynamicConv2d(max_middle_channel, max(self.out_channel_list))), + ('bn', DynamicBatchNorm2d(max(self.out_channel_list))), + ])) + + self.active_kernel_size = max(self.kernel_size_list) + self.active_expand_ratio = max(self.expand_ratio_list) + self.active_out_channel = max(self.out_channel_list) + + def forward(self, x): + in_channel = x.size(1) + + if self.inverted_bottleneck is not None: + self.inverted_bottleneck.conv.active_out_channel = \ + make_divisible(round(in_channel * self.active_expand_ratio), MyNetwork.CHANNEL_DIVISIBLE) + + self.depth_conv.conv.active_kernel_size = self.active_kernel_size + self.point_linear.conv.active_out_channel = self.active_out_channel + + if self.inverted_bottleneck is not None: + x = self.inverted_bottleneck(x) + x = self.depth_conv(x) + x = self.point_linear(x) + return x + + @property + def module_str(self): + if self.use_se: + return 'SE(O%d, E%.1f, K%d)' % (self.active_out_channel, self.active_expand_ratio, self.active_kernel_size) + else: + return '(O%d, E%.1f, K%d)' % (self.active_out_channel, self.active_expand_ratio, self.active_kernel_size) + + @property + def config(self): + return { + 'name': DynamicMBConvLayer.__name__, + 'in_channel_list': self.in_channel_list, + 'out_channel_list': self.out_channel_list, + 'kernel_size_list': self.kernel_size_list, + 'expand_ratio_list': self.expand_ratio_list, + 'stride': self.stride, + 'act_func': self.act_func, + 'use_se': self.use_se, + } + + @staticmethod + def build_from_config(config): + return DynamicMBConvLayer(**config) + + ############################################################################################ + + @property + def in_channels(self): + return max(self.in_channel_list) + + @property + def out_channels(self): + return max(self.out_channel_list) + + def active_middle_channel(self, in_channel): + return make_divisible(round(in_channel * self.active_expand_ratio), MyNetwork.CHANNEL_DIVISIBLE) + + ############################################################################################ + + def get_active_subnet(self, in_channel, preserve_weight=True): + # build the new layer + sub_layer = set_layer_from_config(self.get_active_subnet_config(in_channel)) + sub_layer = sub_layer.to(get_net_device(self)) + if not preserve_weight: + return sub_layer + + middle_channel = self.active_middle_channel(in_channel) + # copy weight from current layer + if sub_layer.inverted_bottleneck is not None: + sub_layer.inverted_bottleneck.conv.weight.data.copy_( + self.inverted_bottleneck.conv.get_active_filter(middle_channel, in_channel).data, + ) + copy_bn(sub_layer.inverted_bottleneck.bn, self.inverted_bottleneck.bn.bn) + + sub_layer.depth_conv.conv.weight.data.copy_( + self.depth_conv.conv.get_active_filter(middle_channel, self.active_kernel_size).data + ) + copy_bn(sub_layer.depth_conv.bn, self.depth_conv.bn.bn) + + if self.use_se: + se_mid = make_divisible(middle_channel // SEModule.REDUCTION, divisor=MyNetwork.CHANNEL_DIVISIBLE) + sub_layer.depth_conv.se.fc.reduce.weight.data.copy_( + self.depth_conv.se.get_active_reduce_weight(se_mid, middle_channel).data + ) + sub_layer.depth_conv.se.fc.reduce.bias.data.copy_( + self.depth_conv.se.get_active_reduce_bias(se_mid).data + ) + + sub_layer.depth_conv.se.fc.expand.weight.data.copy_( + self.depth_conv.se.get_active_expand_weight(se_mid, middle_channel).data + ) + sub_layer.depth_conv.se.fc.expand.bias.data.copy_( + self.depth_conv.se.get_active_expand_bias(middle_channel).data + ) + + sub_layer.point_linear.conv.weight.data.copy_( + self.point_linear.conv.get_active_filter(self.active_out_channel, middle_channel).data + ) + copy_bn(sub_layer.point_linear.bn, self.point_linear.bn.bn) + + return sub_layer + + def get_active_subnet_config(self, in_channel): + return { + 'name': MBConvLayer.__name__, + 'in_channels': in_channel, + 'out_channels': self.active_out_channel, + 'kernel_size': self.active_kernel_size, + 'stride': self.stride, + 'expand_ratio': self.active_expand_ratio, + 'mid_channels': self.active_middle_channel(in_channel), + 'act_func': self.act_func, + 'use_se': self.use_se, + } + + def re_organize_middle_weights(self, expand_ratio_stage=0): + importance = torch.sum(torch.abs(self.point_linear.conv.conv.weight.data), dim=(0, 2, 3)) + if isinstance(self.depth_conv.bn, DynamicGroupNorm): + channel_per_group = self.depth_conv.bn.channel_per_group + importance_chunks = torch.split(importance, channel_per_group) + for chunk in importance_chunks: + chunk.data.fill_(torch.mean(chunk)) + importance = torch.cat(importance_chunks, dim=0) + if expand_ratio_stage > 0: + sorted_expand_list = copy.deepcopy(self.expand_ratio_list) + sorted_expand_list.sort(reverse=True) + target_width_list = [ + make_divisible(round(max(self.in_channel_list) * expand), MyNetwork.CHANNEL_DIVISIBLE) + for expand in sorted_expand_list + ] + + right = len(importance) + base = - len(target_width_list) * 1e5 + for i in range(expand_ratio_stage + 1): + left = target_width_list[i] + importance[left:right] += base + base += 1e5 + right = left + + sorted_importance, sorted_idx = torch.sort(importance, dim=0, descending=True) + self.point_linear.conv.conv.weight.data = torch.index_select( + self.point_linear.conv.conv.weight.data, 1, sorted_idx + ) + + adjust_bn_according_to_idx(self.depth_conv.bn.bn, sorted_idx) + self.depth_conv.conv.conv.weight.data = torch.index_select( + self.depth_conv.conv.conv.weight.data, 0, sorted_idx + ) + + if self.use_se: + # se expand: output dim 0 reorganize + se_expand = self.depth_conv.se.fc.expand + se_expand.weight.data = torch.index_select(se_expand.weight.data, 0, sorted_idx) + se_expand.bias.data = torch.index_select(se_expand.bias.data, 0, sorted_idx) + # se reduce: input dim 1 reorganize + se_reduce = self.depth_conv.se.fc.reduce + se_reduce.weight.data = torch.index_select(se_reduce.weight.data, 1, sorted_idx) + # middle weight reorganize + se_importance = torch.sum(torch.abs(se_expand.weight.data), dim=(0, 2, 3)) + se_importance, se_idx = torch.sort(se_importance, dim=0, descending=True) + + se_expand.weight.data = torch.index_select(se_expand.weight.data, 1, se_idx) + se_reduce.weight.data = torch.index_select(se_reduce.weight.data, 0, se_idx) + se_reduce.bias.data = torch.index_select(se_reduce.bias.data, 0, se_idx) + + if self.inverted_bottleneck is not None: + adjust_bn_according_to_idx(self.inverted_bottleneck.bn.bn, sorted_idx) + self.inverted_bottleneck.conv.conv.weight.data = torch.index_select( + self.inverted_bottleneck.conv.conv.weight.data, 0, sorted_idx + ) + return None + else: + return sorted_idx + + +class DynamicConvLayer(MyModule): + + def __init__(self, in_channel_list, out_channel_list, kernel_size=3, stride=1, dilation=1, + use_bn=True, act_func='relu6'): + super(DynamicConvLayer, self).__init__() + + self.in_channel_list = in_channel_list + self.out_channel_list = out_channel_list + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + self.use_bn = use_bn + self.act_func = act_func + + self.conv = DynamicConv2d( + max_in_channels=max(self.in_channel_list), max_out_channels=max(self.out_channel_list), + kernel_size=self.kernel_size, stride=self.stride, dilation=self.dilation, + ) + if self.use_bn: + self.bn = DynamicBatchNorm2d(max(self.out_channel_list)) + self.act = build_activation(self.act_func) + + self.active_out_channel = max(self.out_channel_list) + + def forward(self, x): + self.conv.active_out_channel = self.active_out_channel + + x = self.conv(x) + if self.use_bn: + x = self.bn(x) + x = self.act(x) + return x + + @property + def module_str(self): + return 'DyConv(O%d, K%d, S%d)' % (self.active_out_channel, self.kernel_size, self.stride) + + @property + def config(self): + return { + 'name': DynamicConvLayer.__name__, + 'in_channel_list': self.in_channel_list, + 'out_channel_list': self.out_channel_list, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'dilation': self.dilation, + 'use_bn': self.use_bn, + 'act_func': self.act_func, + } + + @staticmethod + def build_from_config(config): + return DynamicConvLayer(**config) + + ############################################################################################ + + @property + def in_channels(self): + return max(self.in_channel_list) + + @property + def out_channels(self): + return max(self.out_channel_list) + + ############################################################################################ + + def get_active_subnet(self, in_channel, preserve_weight=True): + sub_layer = set_layer_from_config(self.get_active_subnet_config(in_channel)) + sub_layer = sub_layer.to(get_net_device(self)) + + if not preserve_weight: + return sub_layer + + sub_layer.conv.weight.data.copy_(self.conv.get_active_filter(self.active_out_channel, in_channel).data) + if self.use_bn: + copy_bn(sub_layer.bn, self.bn.bn) + + return sub_layer + + def get_active_subnet_config(self, in_channel): + return { + 'name': ConvLayer.__name__, + 'in_channels': in_channel, + 'out_channels': self.active_out_channel, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'dilation': self.dilation, + 'use_bn': self.use_bn, + 'act_func': self.act_func, + } + + +class DynamicResNetBottleneckBlock(MyModule): + + def __init__(self, in_channel_list, out_channel_list, expand_ratio_list=0.25, + kernel_size=3, stride=1, act_func='relu', downsample_mode='avgpool_conv'): + super(DynamicResNetBottleneckBlock, self).__init__() + + self.in_channel_list = in_channel_list + self.out_channel_list = out_channel_list + self.expand_ratio_list = val2list(expand_ratio_list) + + self.kernel_size = kernel_size + self.stride = stride + self.act_func = act_func + self.downsample_mode = downsample_mode + + # build modules + max_middle_channel = make_divisible( + round(max(self.out_channel_list) * max(self.expand_ratio_list)), MyNetwork.CHANNEL_DIVISIBLE) + + self.conv1 = nn.Sequential(OrderedDict([ + ('conv', DynamicConv2d(max(self.in_channel_list), max_middle_channel)), + ('bn', DynamicBatchNorm2d(max_middle_channel)), + ('act', build_activation(self.act_func, inplace=True)), + ])) + + self.conv2 = nn.Sequential(OrderedDict([ + ('conv', DynamicConv2d(max_middle_channel, max_middle_channel, kernel_size, stride)), + ('bn', DynamicBatchNorm2d(max_middle_channel)), + ('act', build_activation(self.act_func, inplace=True)) + ])) + + self.conv3 = nn.Sequential(OrderedDict([ + ('conv', DynamicConv2d(max_middle_channel, max(self.out_channel_list))), + ('bn', DynamicBatchNorm2d(max(self.out_channel_list))), + ])) + + if self.stride == 1 and self.in_channel_list == self.out_channel_list: + self.downsample = IdentityLayer(max(self.in_channel_list), max(self.out_channel_list)) + elif self.downsample_mode == 'conv': + self.downsample = nn.Sequential(OrderedDict([ + ('conv', DynamicConv2d(max(self.in_channel_list), max(self.out_channel_list), stride=stride)), + ('bn', DynamicBatchNorm2d(max(self.out_channel_list))), + ])) + elif self.downsample_mode == 'avgpool_conv': + self.downsample = nn.Sequential(OrderedDict([ + ('avg_pool', nn.AvgPool2d(kernel_size=stride, stride=stride, padding=0, ceil_mode=True)), + ('conv', DynamicConv2d(max(self.in_channel_list), max(self.out_channel_list))), + ('bn', DynamicBatchNorm2d(max(self.out_channel_list))), + ])) + else: + raise NotImplementedError + + self.final_act = build_activation(self.act_func, inplace=True) + + self.active_expand_ratio = max(self.expand_ratio_list) + self.active_out_channel = max(self.out_channel_list) + + def forward(self, x): + feature_dim = self.active_middle_channels + + self.conv1.conv.active_out_channel = feature_dim + self.conv2.conv.active_out_channel = feature_dim + self.conv3.conv.active_out_channel = self.active_out_channel + if not isinstance(self.downsample, IdentityLayer): + self.downsample.conv.active_out_channel = self.active_out_channel + + residual = self.downsample(x) + + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + + x = x + residual + x = self.final_act(x) + return x + + @property + def module_str(self): + return '(%s, %s)' % ( + '%dx%d_BottleneckConv_in->%d->%d_S%d' % ( + self.kernel_size, self.kernel_size, self.active_middle_channels, self.active_out_channel, self.stride + ), + 'Identity' if isinstance(self.downsample, IdentityLayer) else self.downsample_mode, + ) + + @property + def config(self): + return { + 'name': DynamicResNetBottleneckBlock.__name__, + 'in_channel_list': self.in_channel_list, + 'out_channel_list': self.out_channel_list, + 'expand_ratio_list': self.expand_ratio_list, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'act_func': self.act_func, + 'downsample_mode': self.downsample_mode, + } + + @staticmethod + def build_from_config(config): + return DynamicResNetBottleneckBlock(**config) + + ############################################################################################ + + @property + def in_channels(self): + return max(self.in_channel_list) + + @property + def out_channels(self): + return max(self.out_channel_list) + + @property + def active_middle_channels(self): + feature_dim = round(self.active_out_channel * self.active_expand_ratio) + feature_dim = make_divisible(feature_dim, MyNetwork.CHANNEL_DIVISIBLE) + return feature_dim + + ############################################################################################ + + def get_active_subnet(self, in_channel, preserve_weight=True): + # build the new layer + sub_layer = set_layer_from_config(self.get_active_subnet_config(in_channel)) + sub_layer = sub_layer.to(get_net_device(self)) + if not preserve_weight: + return sub_layer + + # copy weight from current layer + sub_layer.conv1.conv.weight.data.copy_( + self.conv1.conv.get_active_filter(self.active_middle_channels, in_channel).data) + copy_bn(sub_layer.conv1.bn, self.conv1.bn.bn) + + sub_layer.conv2.conv.weight.data.copy_( + self.conv2.conv.get_active_filter(self.active_middle_channels, self.active_middle_channels).data) + copy_bn(sub_layer.conv2.bn, self.conv2.bn.bn) + + sub_layer.conv3.conv.weight.data.copy_( + self.conv3.conv.get_active_filter(self.active_out_channel, self.active_middle_channels).data) + copy_bn(sub_layer.conv3.bn, self.conv3.bn.bn) + + if not isinstance(self.downsample, IdentityLayer): + sub_layer.downsample.conv.weight.data.copy_( + self.downsample.conv.get_active_filter(self.active_out_channel, in_channel).data) + copy_bn(sub_layer.downsample.bn, self.downsample.bn.bn) + + return sub_layer + + def get_active_subnet_config(self, in_channel): + return { + 'name': ResNetBottleneckBlock.__name__, + 'in_channels': in_channel, + 'out_channels': self.active_out_channel, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'expand_ratio': self.active_expand_ratio, + 'mid_channels': self.active_middle_channels, + 'act_func': self.act_func, + 'groups': 1, + 'downsample_mode': self.downsample_mode, + } + + def re_organize_middle_weights(self, expand_ratio_stage=0): + # conv3 -> conv2 + importance = torch.sum(torch.abs(self.conv3.conv.conv.weight.data), dim=(0, 2, 3)) + if isinstance(self.conv2.bn, DynamicGroupNorm): + channel_per_group = self.conv2.bn.channel_per_group + importance_chunks = torch.split(importance, channel_per_group) + for chunk in importance_chunks: + chunk.data.fill_(torch.mean(chunk)) + importance = torch.cat(importance_chunks, dim=0) + if expand_ratio_stage > 0: + sorted_expand_list = copy.deepcopy(self.expand_ratio_list) + sorted_expand_list.sort(reverse=True) + target_width_list = [ + make_divisible(round(max(self.out_channel_list) * expand), MyNetwork.CHANNEL_DIVISIBLE) + for expand in sorted_expand_list + ] + right = len(importance) + base = - len(target_width_list) * 1e5 + for i in range(expand_ratio_stage + 1): + left = target_width_list[i] + importance[left:right] += base + base += 1e5 + right = left + + sorted_importance, sorted_idx = torch.sort(importance, dim=0, descending=True) + self.conv3.conv.conv.weight.data = torch.index_select(self.conv3.conv.conv.weight.data, 1, sorted_idx) + adjust_bn_according_to_idx(self.conv2.bn.bn, sorted_idx) + self.conv2.conv.conv.weight.data = torch.index_select(self.conv2.conv.conv.weight.data, 0, sorted_idx) + + # conv2 -> conv1 + importance = torch.sum(torch.abs(self.conv2.conv.conv.weight.data), dim=(0, 2, 3)) + if isinstance(self.conv1.bn, DynamicGroupNorm): + channel_per_group = self.conv1.bn.channel_per_group + importance_chunks = torch.split(importance, channel_per_group) + for chunk in importance_chunks: + chunk.data.fill_(torch.mean(chunk)) + importance = torch.cat(importance_chunks, dim=0) + if expand_ratio_stage > 0: + sorted_expand_list = copy.deepcopy(self.expand_ratio_list) + sorted_expand_list.sort(reverse=True) + target_width_list = [ + make_divisible(round(max(self.out_channel_list) * expand), MyNetwork.CHANNEL_DIVISIBLE) + for expand in sorted_expand_list + ] + right = len(importance) + base = - len(target_width_list) * 1e5 + for i in range(expand_ratio_stage + 1): + left = target_width_list[i] + importance[left:right] += base + base += 1e5 + right = left + sorted_importance, sorted_idx = torch.sort(importance, dim=0, descending=True) + + self.conv2.conv.conv.weight.data = torch.index_select(self.conv2.conv.conv.weight.data, 1, sorted_idx) + adjust_bn_according_to_idx(self.conv1.bn.bn, sorted_idx) + self.conv1.conv.conv.weight.data = torch.index_select(self.conv1.conv.conv.weight.data, 0, sorted_idx) + + return None diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/dynamic_op.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/dynamic_op.py new file mode 100644 index 0000000..1d21986 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/modules/dynamic_op.py @@ -0,0 +1,314 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import torch.nn.functional as F +import torch.nn as nn +import torch +from torch.nn.parameter import Parameter + +from ofa_local.utils import get_same_padding, sub_filter_start_end, make_divisible, SEModule, MyNetwork, MyConv2d + +__all__ = ['DynamicSeparableConv2d', 'DynamicConv2d', 'DynamicGroupConv2d', + 'DynamicBatchNorm2d', 'DynamicGroupNorm', 'DynamicSE', 'DynamicLinear'] + + +class DynamicSeparableConv2d(nn.Module): + KERNEL_TRANSFORM_MODE = 1 # None or 1 + + def __init__(self, max_in_channels, kernel_size_list, stride=1, dilation=1): + super(DynamicSeparableConv2d, self).__init__() + + self.max_in_channels = max_in_channels + self.kernel_size_list = kernel_size_list + self.stride = stride + self.dilation = dilation + + self.conv = nn.Conv2d( + self.max_in_channels, self.max_in_channels, max(self.kernel_size_list), self.stride, + groups=self.max_in_channels, bias=False, + ) + + self._ks_set = list(set(self.kernel_size_list)) + self._ks_set.sort() # e.g., [3, 5, 7] + if self.KERNEL_TRANSFORM_MODE is not None: + # register scaling parameters + # 7to5_matrix, 5to3_matrix + scale_params = {} + for i in range(len(self._ks_set) - 1): + ks_small = self._ks_set[i] + ks_larger = self._ks_set[i + 1] + param_name = '%dto%d' % (ks_larger, ks_small) + # noinspection PyArgumentList + scale_params['%s_matrix' % param_name] = Parameter(torch.eye(ks_small ** 2)) + for name, param in scale_params.items(): + self.register_parameter(name, param) + + self.active_kernel_size = max(self.kernel_size_list) + + def get_active_filter(self, in_channel, kernel_size): + out_channel = in_channel + max_kernel_size = max(self.kernel_size_list) + + start, end = sub_filter_start_end(max_kernel_size, kernel_size) + filters = self.conv.weight[:out_channel, :in_channel, start:end, start:end] + if self.KERNEL_TRANSFORM_MODE is not None and kernel_size < max_kernel_size: + start_filter = self.conv.weight[:out_channel, :in_channel, :, :] # start with max kernel + for i in range(len(self._ks_set) - 1, 0, -1): + src_ks = self._ks_set[i] + if src_ks <= kernel_size: + break + target_ks = self._ks_set[i - 1] + start, end = sub_filter_start_end(src_ks, target_ks) + _input_filter = start_filter[:, :, start:end, start:end] + _input_filter = _input_filter.contiguous() + _input_filter = _input_filter.view(_input_filter.size(0), _input_filter.size(1), -1) + _input_filter = _input_filter.view(-1, _input_filter.size(2)) + _input_filter = F.linear( + _input_filter, self.__getattr__('%dto%d_matrix' % (src_ks, target_ks)), + ) + _input_filter = _input_filter.view(filters.size(0), filters.size(1), target_ks ** 2) + _input_filter = _input_filter.view(filters.size(0), filters.size(1), target_ks, target_ks) + start_filter = _input_filter + filters = start_filter + return filters + + def forward(self, x, kernel_size=None): + if kernel_size is None: + kernel_size = self.active_kernel_size + in_channel = x.size(1) + + filters = self.get_active_filter(in_channel, kernel_size).contiguous() + + padding = get_same_padding(kernel_size) + filters = self.conv.weight_standardization(filters) if isinstance(self.conv, MyConv2d) else filters + y = F.conv2d( + x, filters, None, self.stride, padding, self.dilation, in_channel + ) + return y + + +class DynamicConv2d(nn.Module): + + def __init__(self, max_in_channels, max_out_channels, kernel_size=1, stride=1, dilation=1): + super(DynamicConv2d, self).__init__() + + self.max_in_channels = max_in_channels + self.max_out_channels = max_out_channels + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + + self.conv = nn.Conv2d( + self.max_in_channels, self.max_out_channels, self.kernel_size, stride=self.stride, bias=False, + ) + + self.active_out_channel = self.max_out_channels + + def get_active_filter(self, out_channel, in_channel): + return self.conv.weight[:out_channel, :in_channel, :, :] + + def forward(self, x, out_channel=None): + if out_channel is None: + out_channel = self.active_out_channel + in_channel = x.size(1) + filters = self.get_active_filter(out_channel, in_channel).contiguous() + + padding = get_same_padding(self.kernel_size) + filters = self.conv.weight_standardization(filters) if isinstance(self.conv, MyConv2d) else filters + y = F.conv2d(x, filters, None, self.stride, padding, self.dilation, 1) + return y + + +class DynamicGroupConv2d(nn.Module): + + def __init__(self, in_channels, out_channels, kernel_size_list, groups_list, stride=1, dilation=1): + super(DynamicGroupConv2d, self).__init__() + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size_list = kernel_size_list + self.groups_list = groups_list + self.stride = stride + self.dilation = dilation + + self.conv = nn.Conv2d( + self.in_channels, self.out_channels, max(self.kernel_size_list), self.stride, + groups=min(self.groups_list), bias=False, + ) + + self.active_kernel_size = max(self.kernel_size_list) + self.active_groups = min(self.groups_list) + + def get_active_filter(self, kernel_size, groups): + start, end = sub_filter_start_end(max(self.kernel_size_list), kernel_size) + filters = self.conv.weight[:, :, start:end, start:end] + + sub_filters = torch.chunk(filters, groups, dim=0) + sub_in_channels = self.in_channels // groups + sub_ratio = filters.size(1) // sub_in_channels + + filter_crops = [] + for i, sub_filter in enumerate(sub_filters): + part_id = i % sub_ratio + start = part_id * sub_in_channels + filter_crops.append(sub_filter[:, start:start + sub_in_channels, :, :]) + filters = torch.cat(filter_crops, dim=0) + return filters + + def forward(self, x, kernel_size=None, groups=None): + if kernel_size is None: + kernel_size = self.active_kernel_size + if groups is None: + groups = self.active_groups + + filters = self.get_active_filter(kernel_size, groups).contiguous() + padding = get_same_padding(kernel_size) + filters = self.conv.weight_standardization(filters) if isinstance(self.conv, MyConv2d) else filters + y = F.conv2d( + x, filters, None, self.stride, padding, self.dilation, groups, + ) + return y + + +class DynamicBatchNorm2d(nn.Module): + SET_RUNNING_STATISTICS = False + + def __init__(self, max_feature_dim): + super(DynamicBatchNorm2d, self).__init__() + + self.max_feature_dim = max_feature_dim + self.bn = nn.BatchNorm2d(self.max_feature_dim) + + @staticmethod + def bn_forward(x, bn: nn.BatchNorm2d, feature_dim): + if bn.num_features == feature_dim or DynamicBatchNorm2d.SET_RUNNING_STATISTICS: + return bn(x) + else: + exponential_average_factor = 0.0 + + if bn.training and bn.track_running_stats: + if bn.num_batches_tracked is not None: + bn.num_batches_tracked += 1 + if bn.momentum is None: # use cumulative moving average + exponential_average_factor = 1.0 / float(bn.num_batches_tracked) + else: # use exponential moving average + exponential_average_factor = bn.momentum + return F.batch_norm( + x, bn.running_mean[:feature_dim], bn.running_var[:feature_dim], bn.weight[:feature_dim], + bn.bias[:feature_dim], bn.training or not bn.track_running_stats, + exponential_average_factor, bn.eps, + ) + + def forward(self, x): + feature_dim = x.size(1) + y = self.bn_forward(x, self.bn, feature_dim) + return y + + +class DynamicGroupNorm(nn.GroupNorm): + + def __init__(self, num_groups, num_channels, eps=1e-5, affine=True, channel_per_group=None): + super(DynamicGroupNorm, self).__init__(num_groups, num_channels, eps, affine) + self.channel_per_group = channel_per_group + + def forward(self, x): + n_channels = x.size(1) + n_groups = n_channels // self.channel_per_group + return F.group_norm(x, n_groups, self.weight[:n_channels], self.bias[:n_channels], self.eps) + + @property + def bn(self): + return self + + +class DynamicSE(SEModule): + + def __init__(self, max_channel): + super(DynamicSE, self).__init__(max_channel) + + def get_active_reduce_weight(self, num_mid, in_channel, groups=None): + if groups is None or groups == 1: + return self.fc.reduce.weight[:num_mid, :in_channel, :, :] + else: + assert in_channel % groups == 0 + sub_in_channels = in_channel // groups + sub_filters = torch.chunk(self.fc.reduce.weight[:num_mid, :, :, :], groups, dim=1) + return torch.cat([ + sub_filter[:, :sub_in_channels, :, :] for sub_filter in sub_filters + ], dim=1) + + def get_active_reduce_bias(self, num_mid): + return self.fc.reduce.bias[:num_mid] if self.fc.reduce.bias is not None else None + + def get_active_expand_weight(self, num_mid, in_channel, groups=None): + if groups is None or groups == 1: + return self.fc.expand.weight[:in_channel, :num_mid, :, :] + else: + assert in_channel % groups == 0 + sub_in_channels = in_channel // groups + sub_filters = torch.chunk(self.fc.expand.weight[:, :num_mid, :, :], groups, dim=0) + return torch.cat([ + sub_filter[:sub_in_channels, :, :, :] for sub_filter in sub_filters + ], dim=0) + + def get_active_expand_bias(self, in_channel, groups=None): + if groups is None or groups == 1: + return self.fc.expand.bias[:in_channel] if self.fc.expand.bias is not None else None + else: + assert in_channel % groups == 0 + sub_in_channels = in_channel // groups + sub_bias_list = torch.chunk(self.fc.expand.bias, groups, dim=0) + return torch.cat([ + sub_bias[:sub_in_channels] for sub_bias in sub_bias_list + ], dim=0) + + def forward(self, x, groups=None): + in_channel = x.size(1) + num_mid = make_divisible(in_channel // self.reduction, divisor=MyNetwork.CHANNEL_DIVISIBLE) + + y = x.mean(3, keepdim=True).mean(2, keepdim=True) + # reduce + reduce_filter = self.get_active_reduce_weight(num_mid, in_channel, groups=groups).contiguous() + reduce_bias = self.get_active_reduce_bias(num_mid) + y = F.conv2d(y, reduce_filter, reduce_bias, 1, 0, 1, 1) + # relu + y = self.fc.relu(y) + # expand + expand_filter = self.get_active_expand_weight(num_mid, in_channel, groups=groups).contiguous() + expand_bias = self.get_active_expand_bias(in_channel, groups=groups) + y = F.conv2d(y, expand_filter, expand_bias, 1, 0, 1, 1) + # hard sigmoid + y = self.fc.h_sigmoid(y) + + return x * y + + +class DynamicLinear(nn.Module): + + def __init__(self, max_in_features, max_out_features, bias=True): + super(DynamicLinear, self).__init__() + + self.max_in_features = max_in_features + self.max_out_features = max_out_features + self.bias = bias + + self.linear = nn.Linear(self.max_in_features, self.max_out_features, self.bias) + + self.active_out_features = self.max_out_features + + def get_active_weight(self, out_features, in_features): + return self.linear.weight[:out_features, :in_features] + + def get_active_bias(self, out_features): + return self.linear.bias[:out_features] if self.bias else None + + def forward(self, x, out_features=None): + if out_features is None: + out_features = self.active_out_features + + in_features = x.size(1) + weight = self.get_active_weight(out_features, in_features).contiguous() + bias = self.get_active_bias(out_features) + y = F.linear(x, weight, bias) + return y diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/__init__.py new file mode 100644 index 0000000..41d97dc --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/__init__.py @@ -0,0 +1,7 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .ofa_proxyless import OFAProxylessNASNets +from .ofa_mbv3 import OFAMobileNetV3 +from .ofa_resnets import OFAResNets diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_mbv3.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_mbv3.py new file mode 100644 index 0000000..eb55f51 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_mbv3.py @@ -0,0 +1,336 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import copy +import random + +from ofa_local.imagenet_classification.elastic_nn.modules.dynamic_layers import DynamicMBConvLayer +from ofa_local.utils.layers import ConvLayer, IdentityLayer, LinearLayer, MBConvLayer, ResidualBlock +from ofa_local.imagenet_classification.networks import MobileNetV3 +from ofa_local.utils import make_divisible, val2list, MyNetwork +from ofa_local.utils.layers import set_layer_from_config +import gin + +__all__ = ['OFAMobileNetV3'] + +@gin.configurable +class OFAMobileNetV3(MobileNetV3): + + def __init__(self, n_classes=1000, bn_param=(0.1, 1e-5), dropout_rate=0.1, base_stage_width=None, width_mult=1.0, + ks_list=3, expand_ratio_list=6, depth_list=4, dropblock=False, block_size=0): + + self.width_mult = width_mult + self.ks_list = val2list(ks_list, 1) + self.expand_ratio_list = val2list(expand_ratio_list, 1) + self.depth_list = val2list(depth_list, 1) + + self.ks_list.sort() + self.expand_ratio_list.sort() + self.depth_list.sort() + + base_stage_width = [16, 16, 24, 40, 80, 112, 160, 960, 1280] + + final_expand_width = make_divisible(base_stage_width[-2] * self.width_mult, MyNetwork.CHANNEL_DIVISIBLE) + last_channel = make_divisible(base_stage_width[-1] * self.width_mult, MyNetwork.CHANNEL_DIVISIBLE) + + stride_stages = [1, 2, 2, 2, 1, 2] + act_stages = ['relu', 'relu', 'relu', 'h_swish', 'h_swish', 'h_swish'] + se_stages = [False, False, True, False, True, True] + n_block_list = [1] + [max(self.depth_list)] * 5 + width_list = [] + for base_width in base_stage_width[:-2]: + width = make_divisible(base_width * self.width_mult, MyNetwork.CHANNEL_DIVISIBLE) + width_list.append(width) + + input_channel, first_block_dim = width_list[0], width_list[1] + # first conv layer + first_conv = ConvLayer(3, input_channel, kernel_size=3, stride=2, act_func='h_swish') + first_block_conv = MBConvLayer( + in_channels=input_channel, out_channels=first_block_dim, kernel_size=3, stride=stride_stages[0], + expand_ratio=1, act_func=act_stages[0], use_se=se_stages[0], + ) + first_block = ResidualBlock( + first_block_conv, + IdentityLayer(first_block_dim, first_block_dim) if input_channel == first_block_dim else None, + dropout_rate, dropblock, block_size + ) + + # inverted residual blocks + self.block_group_info = [] + blocks = [first_block] + _block_index = 1 + feature_dim = first_block_dim + + for width, n_block, s, act_func, use_se in zip(width_list[2:], n_block_list[1:], + stride_stages[1:], act_stages[1:], se_stages[1:]): + self.block_group_info.append([_block_index + i for i in range(n_block)]) + _block_index += n_block + + output_channel = width + for i in range(n_block): + if i == 0: + stride = s + else: + stride = 1 + mobile_inverted_conv = DynamicMBConvLayer( + in_channel_list=val2list(feature_dim), out_channel_list=val2list(output_channel), + kernel_size_list=ks_list, expand_ratio_list=expand_ratio_list, + stride=stride, act_func=act_func, use_se=use_se, + ) + if stride == 1 and feature_dim == output_channel: + shortcut = IdentityLayer(feature_dim, feature_dim) + else: + shortcut = None + blocks.append(ResidualBlock(mobile_inverted_conv, shortcut, + dropout_rate, dropblock, block_size)) + feature_dim = output_channel + # final expand layer, feature mix layer & classifier + final_expand_layer = ConvLayer(feature_dim, final_expand_width, kernel_size=1, act_func='h_swish') + feature_mix_layer = ConvLayer( + final_expand_width, last_channel, kernel_size=1, bias=False, use_bn=False, act_func='h_swish', + ) + + classifier = LinearLayer(last_channel, n_classes, dropout_rate=dropout_rate) + + super(OFAMobileNetV3, self).__init__(first_conv, blocks, final_expand_layer, feature_mix_layer, classifier) + + # set bn param + self.set_bn_param(momentum=bn_param[0], eps=bn_param[1]) + + # runtime_depth + self.runtime_depth = [len(block_idx) for block_idx in self.block_group_info] + + """ MyNetwork required methods """ + + @staticmethod + def name(): + return 'OFAMobileNetV3' + + def forward(self, x): + # first conv + x = self.first_conv(x) + # first block + x = self.blocks[0](x) + # blocks + for stage_id, block_idx in enumerate(self.block_group_info): + depth = self.runtime_depth[stage_id] + active_idx = block_idx[:depth] + for idx in active_idx: + x = self.blocks[idx](x) + x = self.final_expand_layer(x) + x = x.mean(3, keepdim=True).mean(2, keepdim=True) # global average pooling + x = self.feature_mix_layer(x) + x = x.view(x.size(0), -1) + x = self.classifier(x) + return x + + @property + def module_str(self): + _str = self.first_conv.module_str + '\n' + _str += self.blocks[0].module_str + '\n' + + for stage_id, block_idx in enumerate(self.block_group_info): + depth = self.runtime_depth[stage_id] + active_idx = block_idx[:depth] + for idx in active_idx: + _str += self.blocks[idx].module_str + '\n' + + _str += self.final_expand_layer.module_str + '\n' + _str += self.feature_mix_layer.module_str + '\n' + _str += self.classifier.module_str + '\n' + return _str + + @property + def config(self): + return { + 'name': OFAMobileNetV3.__name__, + 'bn': self.get_bn_param(), + 'first_conv': self.first_conv.config, + 'blocks': [ + block.config for block in self.blocks + ], + 'final_expand_layer': self.final_expand_layer.config, + 'feature_mix_layer': self.feature_mix_layer.config, + 'classifier': self.classifier.config, + } + + @staticmethod + def build_from_config(config): + raise ValueError('do not support this function') + + @property + def grouped_block_index(self): + return self.block_group_info + + def load_state_dict(self, state_dict, **kwargs): + model_dict = self.state_dict() + for key in state_dict: + if '.mobile_inverted_conv.' in key: + new_key = key.replace('.mobile_inverted_conv.', '.conv.') + else: + new_key = key + if new_key in model_dict: + pass + elif '.bn.bn.' in new_key: + new_key = new_key.replace('.bn.bn.', '.bn.') + elif '.conv.conv.weight' in new_key: + new_key = new_key.replace('.conv.conv.weight', '.conv.weight') + elif '.linear.linear.' in new_key: + new_key = new_key.replace('.linear.linear.', '.linear.') + ############################################################################## + elif '.linear.' in new_key: + new_key = new_key.replace('.linear.', '.linear.linear.') + elif 'bn.' in new_key: + new_key = new_key.replace('bn.', 'bn.bn.') + elif 'conv.weight' in new_key: + new_key = new_key.replace('conv.weight', 'conv.conv.weight') + else: + raise ValueError(new_key) + assert new_key in model_dict, '%s' % new_key + model_dict[new_key] = state_dict[key] + super(OFAMobileNetV3, self).load_state_dict(model_dict) + + """ set, sample and get active sub-networks """ + + def set_max_net(self): + self.set_active_subnet(ks=max(self.ks_list), e=max(self.expand_ratio_list), d=max(self.depth_list)) + + def set_active_subnet(self, ks=None, e=None, d=None, **kwargs): + ks = val2list(ks, len(self.blocks) - 1) + expand_ratio = val2list(e, len(self.blocks) - 1) + depth = val2list(d, len(self.block_group_info)) + + for block, k, e in zip(self.blocks[1:], ks, expand_ratio): + if k is not None: + block.conv.active_kernel_size = k + if e is not None: + block.conv.active_expand_ratio = e + + for i, d in enumerate(depth): + if d is not None: + self.runtime_depth[i] = min(len(self.block_group_info[i]), d) + + def set_constraint(self, include_list, constraint_type='depth'): + if constraint_type == 'depth': + self.__dict__['_depth_include_list'] = include_list.copy() + elif constraint_type == 'expand_ratio': + self.__dict__['_expand_include_list'] = include_list.copy() + elif constraint_type == 'kernel_size': + self.__dict__['_ks_include_list'] = include_list.copy() + else: + raise NotImplementedError + + def clear_constraint(self): + self.__dict__['_depth_include_list'] = None + self.__dict__['_expand_include_list'] = None + self.__dict__['_ks_include_list'] = None + + def sample_active_subnet(self): + ks_candidates = self.ks_list if self.__dict__.get('_ks_include_list', None) is None \ + else self.__dict__['_ks_include_list'] + expand_candidates = self.expand_ratio_list if self.__dict__.get('_expand_include_list', None) is None \ + else self.__dict__['_expand_include_list'] + depth_candidates = self.depth_list if self.__dict__.get('_depth_include_list', None) is None else \ + self.__dict__['_depth_include_list'] + + # sample kernel size + ks_setting = [] + if not isinstance(ks_candidates[0], list): + ks_candidates = [ks_candidates for _ in range(len(self.blocks) - 1)] + for k_set in ks_candidates: + k = random.choice(k_set) + ks_setting.append(k) + + # sample expand ratio + expand_setting = [] + if not isinstance(expand_candidates[0], list): + expand_candidates = [expand_candidates for _ in range(len(self.blocks) - 1)] + for e_set in expand_candidates: + e = random.choice(e_set) + expand_setting.append(e) + + # sample depth + depth_setting = [] + if not isinstance(depth_candidates[0], list): + depth_candidates = [depth_candidates for _ in range(len(self.block_group_info))] + for d_set in depth_candidates: + d = random.choice(d_set) + depth_setting.append(d) + + import pdb; pdb.set_trace() + self.set_active_subnet(ks_setting, expand_setting, depth_setting) + + return { + 'ks': ks_setting, + 'e': expand_setting, + 'd': depth_setting, + } + + def get_active_subnet(self, preserve_weight=True): + first_conv = copy.deepcopy(self.first_conv) + blocks = [copy.deepcopy(self.blocks[0])] + + final_expand_layer = copy.deepcopy(self.final_expand_layer) + feature_mix_layer = copy.deepcopy(self.feature_mix_layer) + classifier = copy.deepcopy(self.classifier) + + input_channel = blocks[0].conv.out_channels + # blocks + for stage_id, block_idx in enumerate(self.block_group_info): + depth = self.runtime_depth[stage_id] + active_idx = block_idx[:depth] + stage_blocks = [] + for idx in active_idx: + stage_blocks.append(ResidualBlock( + self.blocks[idx].conv.get_active_subnet(input_channel, preserve_weight), + copy.deepcopy(self.blocks[idx].shortcut), + copy.deepcopy(self.blocks[idx].dropout_rate), + copy.deepcopy(self.blocks[idx].dropblock), + copy.deepcopy(self.blocks[idx].block_size), + )) + input_channel = stage_blocks[-1].conv.out_channels + blocks += stage_blocks + + _subnet = MobileNetV3(first_conv, blocks, final_expand_layer, feature_mix_layer, classifier) + _subnet.set_bn_param(**self.get_bn_param()) + return _subnet + + def get_active_net_config(self): + # first conv + first_conv_config = self.first_conv.config + first_block_config = self.blocks[0].config + final_expand_config = self.final_expand_layer.config + feature_mix_layer_config = self.feature_mix_layer.config + classifier_config = self.classifier.config + + block_config_list = [first_block_config] + input_channel = first_block_config['conv']['out_channels'] + for stage_id, block_idx in enumerate(self.block_group_info): + depth = self.runtime_depth[stage_id] + active_idx = block_idx[:depth] + stage_blocks = [] + for idx in active_idx: + stage_blocks.append({ + 'name': ResidualBlock.__name__, + 'conv': self.blocks[idx].conv.get_active_subnet_config(input_channel), + 'shortcut': self.blocks[idx].shortcut.config if self.blocks[idx].shortcut is not None else None, + }) + input_channel = self.blocks[idx].conv.active_out_channel + block_config_list += stage_blocks + + return { + 'name': MobileNetV3.__name__, + 'bn': self.get_bn_param(), + 'first_conv': first_conv_config, + 'blocks': block_config_list, + 'final_expand_layer': final_expand_config, + 'feature_mix_layer': feature_mix_layer_config, + 'classifier': classifier_config, + } + + """ Width Related Methods """ + + def re_organize_middle_weights(self, expand_ratio_stage=0): + for block in self.blocks[1:]: + block.conv.re_organize_middle_weights(expand_ratio_stage) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_proxyless.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_proxyless.py new file mode 100644 index 0000000..72a8259 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_proxyless.py @@ -0,0 +1,331 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import copy +import random + +from ofa_local.utils import make_divisible, val2list, MyNetwork +from ofa_local.imagenet_classification.elastic_nn.modules import DynamicMBConvLayer +from ofa_local.utils.layers import ConvLayer, IdentityLayer, LinearLayer, MBConvLayer, ResidualBlock +from ofa_local.imagenet_classification.networks.proxyless_nets import ProxylessNASNets + +__all__ = ['OFAProxylessNASNets'] + + +class OFAProxylessNASNets(ProxylessNASNets): + + def __init__(self, n_classes=1000, bn_param=(0.1, 1e-3), dropout_rate=0.1, base_stage_width=None, width_mult=1.0, + ks_list=3, expand_ratio_list=6, depth_list=4): + + self.width_mult = width_mult + self.ks_list = val2list(ks_list, 1) + self.expand_ratio_list = val2list(expand_ratio_list, 1) + self.depth_list = val2list(depth_list, 1) + + self.ks_list.sort() + self.expand_ratio_list.sort() + self.depth_list.sort() + + if base_stage_width == 'google': + # MobileNetV2 Stage Width + base_stage_width = [32, 16, 24, 32, 64, 96, 160, 320, 1280] + else: + # ProxylessNAS Stage Width + base_stage_width = [32, 16, 24, 40, 80, 96, 192, 320, 1280] + + input_channel = make_divisible(base_stage_width[0] * self.width_mult, MyNetwork.CHANNEL_DIVISIBLE) + first_block_width = make_divisible(base_stage_width[1] * self.width_mult, MyNetwork.CHANNEL_DIVISIBLE) + last_channel = make_divisible(base_stage_width[-1] * self.width_mult, MyNetwork.CHANNEL_DIVISIBLE) + + # first conv layer + first_conv = ConvLayer( + 3, input_channel, kernel_size=3, stride=2, use_bn=True, act_func='relu6', ops_order='weight_bn_act' + ) + # first block + first_block_conv = MBConvLayer( + in_channels=input_channel, out_channels=first_block_width, kernel_size=3, stride=1, + expand_ratio=1, act_func='relu6', + ) + first_block = ResidualBlock(first_block_conv, None) + + input_channel = first_block_width + # inverted residual blocks + self.block_group_info = [] + blocks = [first_block] + _block_index = 1 + + stride_stages = [2, 2, 2, 1, 2, 1] + n_block_list = [max(self.depth_list)] * 5 + [1] + + width_list = [] + for base_width in base_stage_width[2:-1]: + width = make_divisible(base_width * self.width_mult, MyNetwork.CHANNEL_DIVISIBLE) + width_list.append(width) + + for width, n_block, s in zip(width_list, n_block_list, stride_stages): + self.block_group_info.append([_block_index + i for i in range(n_block)]) + _block_index += n_block + + output_channel = width + for i in range(n_block): + if i == 0: + stride = s + else: + stride = 1 + + mobile_inverted_conv = DynamicMBConvLayer( + in_channel_list=val2list(input_channel, 1), out_channel_list=val2list(output_channel, 1), + kernel_size_list=ks_list, expand_ratio_list=expand_ratio_list, stride=stride, act_func='relu6', + ) + + if stride == 1 and input_channel == output_channel: + shortcut = IdentityLayer(input_channel, input_channel) + else: + shortcut = None + + mb_inverted_block = ResidualBlock(mobile_inverted_conv, shortcut) + + blocks.append(mb_inverted_block) + input_channel = output_channel + # 1x1_conv before global average pooling + feature_mix_layer = ConvLayer( + input_channel, last_channel, kernel_size=1, use_bn=True, act_func='relu6', + ) + classifier = LinearLayer(last_channel, n_classes, dropout_rate=dropout_rate) + + super(OFAProxylessNASNets, self).__init__(first_conv, blocks, feature_mix_layer, classifier) + + # set bn param + self.set_bn_param(momentum=bn_param[0], eps=bn_param[1]) + + # runtime_depth + self.runtime_depth = [len(block_idx) for block_idx in self.block_group_info] + + """ MyNetwork required methods """ + + @staticmethod + def name(): + return 'OFAProxylessNASNets' + + def forward(self, x): + # first conv + x = self.first_conv(x) + # first block + x = self.blocks[0](x) + + # blocks + for stage_id, block_idx in enumerate(self.block_group_info): + depth = self.runtime_depth[stage_id] + active_idx = block_idx[:depth] + for idx in active_idx: + x = self.blocks[idx](x) + + # feature_mix_layer + x = self.feature_mix_layer(x) + x = x.mean(3).mean(2) + + x = self.classifier(x) + return x + + @property + def module_str(self): + _str = self.first_conv.module_str + '\n' + _str += self.blocks[0].module_str + '\n' + + for stage_id, block_idx in enumerate(self.block_group_info): + depth = self.runtime_depth[stage_id] + active_idx = block_idx[:depth] + for idx in active_idx: + _str += self.blocks[idx].module_str + '\n' + _str += self.feature_mix_layer.module_str + '\n' + _str += self.classifier.module_str + '\n' + return _str + + @property + def config(self): + return { + 'name': OFAProxylessNASNets.__name__, + 'bn': self.get_bn_param(), + 'first_conv': self.first_conv.config, + 'blocks': [ + block.config for block in self.blocks + ], + 'feature_mix_layer': None if self.feature_mix_layer is None else self.feature_mix_layer.config, + 'classifier': self.classifier.config, + } + + @staticmethod + def build_from_config(config): + raise ValueError('do not support this function') + + @property + def grouped_block_index(self): + return self.block_group_info + + def load_state_dict(self, state_dict, **kwargs): + model_dict = self.state_dict() + for key in state_dict: + if '.mobile_inverted_conv.' in key: + new_key = key.replace('.mobile_inverted_conv.', '.conv.') + else: + new_key = key + if new_key in model_dict: + pass + elif '.bn.bn.' in new_key: + new_key = new_key.replace('.bn.bn.', '.bn.') + elif '.conv.conv.weight' in new_key: + new_key = new_key.replace('.conv.conv.weight', '.conv.weight') + elif '.linear.linear.' in new_key: + new_key = new_key.replace('.linear.linear.', '.linear.') + ############################################################################## + elif '.linear.' in new_key: + new_key = new_key.replace('.linear.', '.linear.linear.') + elif 'bn.' in new_key: + new_key = new_key.replace('bn.', 'bn.bn.') + elif 'conv.weight' in new_key: + new_key = new_key.replace('conv.weight', 'conv.conv.weight') + else: + raise ValueError(new_key) + assert new_key in model_dict, '%s' % new_key + model_dict[new_key] = state_dict[key] + super(OFAProxylessNASNets, self).load_state_dict(model_dict) + + """ set, sample and get active sub-networks """ + + def set_max_net(self): + self.set_active_subnet(ks=max(self.ks_list), e=max(self.expand_ratio_list), d=max(self.depth_list)) + + def set_active_subnet(self, ks=None, e=None, d=None, **kwargs): + ks = val2list(ks, len(self.blocks) - 1) + expand_ratio = val2list(e, len(self.blocks) - 1) + depth = val2list(d, len(self.block_group_info)) + + for block, k, e in zip(self.blocks[1:], ks, expand_ratio): + if k is not None: + block.conv.active_kernel_size = k + if e is not None: + block.conv.active_expand_ratio = e + + for i, d in enumerate(depth): + if d is not None: + self.runtime_depth[i] = min(len(self.block_group_info[i]), d) + + def set_constraint(self, include_list, constraint_type='depth'): + if constraint_type == 'depth': + self.__dict__['_depth_include_list'] = include_list.copy() + elif constraint_type == 'expand_ratio': + self.__dict__['_expand_include_list'] = include_list.copy() + elif constraint_type == 'kernel_size': + self.__dict__['_ks_include_list'] = include_list.copy() + else: + raise NotImplementedError + + def clear_constraint(self): + self.__dict__['_depth_include_list'] = None + self.__dict__['_expand_include_list'] = None + self.__dict__['_ks_include_list'] = None + + def sample_active_subnet(self): + ks_candidates = self.ks_list if self.__dict__.get('_ks_include_list', None) is None \ + else self.__dict__['_ks_include_list'] + expand_candidates = self.expand_ratio_list if self.__dict__.get('_expand_include_list', None) is None \ + else self.__dict__['_expand_include_list'] + depth_candidates = self.depth_list if self.__dict__.get('_depth_include_list', None) is None else \ + self.__dict__['_depth_include_list'] + + # sample kernel size + ks_setting = [] + if not isinstance(ks_candidates[0], list): + ks_candidates = [ks_candidates for _ in range(len(self.blocks) - 1)] + for k_set in ks_candidates: + k = random.choice(k_set) + ks_setting.append(k) + + # sample expand ratio + expand_setting = [] + if not isinstance(expand_candidates[0], list): + expand_candidates = [expand_candidates for _ in range(len(self.blocks) - 1)] + for e_set in expand_candidates: + e = random.choice(e_set) + expand_setting.append(e) + + # sample depth + depth_setting = [] + if not isinstance(depth_candidates[0], list): + depth_candidates = [depth_candidates for _ in range(len(self.block_group_info))] + for d_set in depth_candidates: + d = random.choice(d_set) + depth_setting.append(d) + + depth_setting[-1] = 1 + self.set_active_subnet(ks_setting, expand_setting, depth_setting) + + return { + 'ks': ks_setting, + 'e': expand_setting, + 'd': depth_setting, + } + + def get_active_subnet(self, preserve_weight=True): + first_conv = copy.deepcopy(self.first_conv) + blocks = [copy.deepcopy(self.blocks[0])] + feature_mix_layer = copy.deepcopy(self.feature_mix_layer) + classifier = copy.deepcopy(self.classifier) + + input_channel = blocks[0].conv.out_channels + # blocks + for stage_id, block_idx in enumerate(self.block_group_info): + depth = self.runtime_depth[stage_id] + active_idx = block_idx[:depth] + stage_blocks = [] + for idx in active_idx: + stage_blocks.append(ResidualBlock( + self.blocks[idx].conv.get_active_subnet(input_channel, preserve_weight), + copy.deepcopy(self.blocks[idx].shortcut) + )) + input_channel = stage_blocks[-1].conv.out_channels + blocks += stage_blocks + + _subnet = ProxylessNASNets(first_conv, blocks, feature_mix_layer, classifier) + _subnet.set_bn_param(**self.get_bn_param()) + return _subnet + + def get_active_net_config(self): + first_conv_config = self.first_conv.config + first_block_config = self.blocks[0].config + feature_mix_layer_config = self.feature_mix_layer.config + classifier_config = self.classifier.config + + block_config_list = [first_block_config] + input_channel = first_block_config['conv']['out_channels'] + for stage_id, block_idx in enumerate(self.block_group_info): + depth = self.runtime_depth[stage_id] + active_idx = block_idx[:depth] + stage_blocks = [] + for idx in active_idx: + stage_blocks.append({ + 'name': ResidualBlock.__name__, + 'conv': self.blocks[idx].conv.get_active_subnet_config(input_channel), + 'shortcut': self.blocks[idx].shortcut.config if self.blocks[idx].shortcut is not None else None, + }) + try: + input_channel = self.blocks[idx].conv.active_out_channel + except Exception: + input_channel = self.blocks[idx].conv.out_channels + block_config_list += stage_blocks + + return { + 'name': ProxylessNASNets.__name__, + 'bn': self.get_bn_param(), + 'first_conv': first_conv_config, + 'blocks': block_config_list, + 'feature_mix_layer': feature_mix_layer_config, + 'classifier': classifier_config, + } + + """ Width Related Methods """ + + def re_organize_middle_weights(self, expand_ratio_stage=0): + for block in self.blocks[1:]: + block.conv.re_organize_middle_weights(expand_ratio_stage) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_resnets.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_resnets.py new file mode 100644 index 0000000..a020e0e --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/networks/ofa_resnets.py @@ -0,0 +1,267 @@ +import random + +from ofa_local.imagenet_classification.elastic_nn.modules.dynamic_layers import DynamicConvLayer, DynamicLinearLayer +from ofa_local.imagenet_classification.elastic_nn.modules.dynamic_layers import DynamicResNetBottleneckBlock +from ofa_local.utils.layers import IdentityLayer, ResidualBlock +from ofa_local.imagenet_classification.networks import ResNets +from ofa_local.utils import make_divisible, val2list, MyNetwork + +__all__ = ['OFAResNets'] + + +class OFAResNets(ResNets): + + def __init__(self, n_classes=1000, bn_param=(0.1, 1e-5), dropout_rate=0, + depth_list=2, expand_ratio_list=0.25, width_mult_list=1.0): + + self.depth_list = val2list(depth_list) + self.expand_ratio_list = val2list(expand_ratio_list) + self.width_mult_list = val2list(width_mult_list) + # sort + self.depth_list.sort() + self.expand_ratio_list.sort() + self.width_mult_list.sort() + + input_channel = [ + make_divisible(64 * width_mult, MyNetwork.CHANNEL_DIVISIBLE) for width_mult in self.width_mult_list + ] + mid_input_channel = [ + make_divisible(channel // 2, MyNetwork.CHANNEL_DIVISIBLE) for channel in input_channel + ] + + stage_width_list = ResNets.STAGE_WIDTH_LIST.copy() + for i, width in enumerate(stage_width_list): + stage_width_list[i] = [ + make_divisible(width * width_mult, MyNetwork.CHANNEL_DIVISIBLE) for width_mult in self.width_mult_list + ] + + n_block_list = [base_depth + max(self.depth_list) for base_depth in ResNets.BASE_DEPTH_LIST] + stride_list = [1, 2, 2, 2] + + # build input stem + input_stem = [ + DynamicConvLayer(val2list(3), mid_input_channel, 3, stride=2, use_bn=True, act_func='relu'), + ResidualBlock( + DynamicConvLayer(mid_input_channel, mid_input_channel, 3, stride=1, use_bn=True, act_func='relu'), + IdentityLayer(mid_input_channel, mid_input_channel) + ), + DynamicConvLayer(mid_input_channel, input_channel, 3, stride=1, use_bn=True, act_func='relu') + ] + + # blocks + blocks = [] + for d, width, s in zip(n_block_list, stage_width_list, stride_list): + for i in range(d): + stride = s if i == 0 else 1 + bottleneck_block = DynamicResNetBottleneckBlock( + input_channel, width, expand_ratio_list=self.expand_ratio_list, + kernel_size=3, stride=stride, act_func='relu', downsample_mode='avgpool_conv', + ) + blocks.append(bottleneck_block) + input_channel = width + # classifier + classifier = DynamicLinearLayer(input_channel, n_classes, dropout_rate=dropout_rate) + + super(OFAResNets, self).__init__(input_stem, blocks, classifier) + + # set bn param + self.set_bn_param(*bn_param) + + # runtime_depth + self.input_stem_skipping = 0 + self.runtime_depth = [0] * len(n_block_list) + + @property + def ks_list(self): + return [3] + + @staticmethod + def name(): + return 'OFAResNets' + + def forward(self, x): + for layer in self.input_stem: + if self.input_stem_skipping > 0 \ + and isinstance(layer, ResidualBlock) and isinstance(layer.shortcut, IdentityLayer): + pass + else: + x = layer(x) + x = self.max_pooling(x) + for stage_id, block_idx in enumerate(self.grouped_block_index): + depth_param = self.runtime_depth[stage_id] + active_idx = block_idx[:len(block_idx) - depth_param] + for idx in active_idx: + x = self.blocks[idx](x) + x = self.global_avg_pool(x) + x = self.classifier(x) + return x + + @property + def module_str(self): + _str = '' + for layer in self.input_stem: + if self.input_stem_skipping > 0 \ + and isinstance(layer, ResidualBlock) and isinstance(layer.shortcut, IdentityLayer): + pass + else: + _str += layer.module_str + '\n' + _str += 'max_pooling(ks=3, stride=2)\n' + for stage_id, block_idx in enumerate(self.grouped_block_index): + depth_param = self.runtime_depth[stage_id] + active_idx = block_idx[:len(block_idx) - depth_param] + for idx in active_idx: + _str += self.blocks[idx].module_str + '\n' + _str += self.global_avg_pool.__repr__() + '\n' + _str += self.classifier.module_str + return _str + + @property + def config(self): + return { + 'name': OFAResNets.__name__, + 'bn': self.get_bn_param(), + 'input_stem': [ + layer.config for layer in self.input_stem + ], + 'blocks': [ + block.config for block in self.blocks + ], + 'classifier': self.classifier.config, + } + + @staticmethod + def build_from_config(config): + raise ValueError('do not support this function') + + def load_state_dict(self, state_dict, **kwargs): + model_dict = self.state_dict() + for key in state_dict: + new_key = key + if new_key in model_dict: + pass + elif '.linear.' in new_key: + new_key = new_key.replace('.linear.', '.linear.linear.') + elif 'bn.' in new_key: + new_key = new_key.replace('bn.', 'bn.bn.') + elif 'conv.weight' in new_key: + new_key = new_key.replace('conv.weight', 'conv.conv.weight') + else: + raise ValueError(new_key) + assert new_key in model_dict, '%s' % new_key + model_dict[new_key] = state_dict[key] + super(OFAResNets, self).load_state_dict(model_dict) + + """ set, sample and get active sub-networks """ + + def set_max_net(self): + self.set_active_subnet(d=max(self.depth_list), e=max(self.expand_ratio_list), w=len(self.width_mult_list) - 1) + + def set_active_subnet(self, d=None, e=None, w=None, **kwargs): + depth = val2list(d, len(ResNets.BASE_DEPTH_LIST) + 1) + expand_ratio = val2list(e, len(self.blocks)) + width_mult = val2list(w, len(ResNets.BASE_DEPTH_LIST) + 2) + + for block, e in zip(self.blocks, expand_ratio): + if e is not None: + block.active_expand_ratio = e + + if width_mult[0] is not None: + self.input_stem[1].conv.active_out_channel = self.input_stem[0].active_out_channel = \ + self.input_stem[0].out_channel_list[width_mult[0]] + if width_mult[1] is not None: + self.input_stem[2].active_out_channel = self.input_stem[2].out_channel_list[width_mult[1]] + + if depth[0] is not None: + self.input_stem_skipping = (depth[0] != max(self.depth_list)) + for stage_id, (block_idx, d, w) in enumerate(zip(self.grouped_block_index, depth[1:], width_mult[2:])): + if d is not None: + self.runtime_depth[stage_id] = max(self.depth_list) - d + if w is not None: + for idx in block_idx: + self.blocks[idx].active_out_channel = self.blocks[idx].out_channel_list[w] + + def sample_active_subnet(self): + # sample expand ratio + expand_setting = [] + for block in self.blocks: + expand_setting.append(random.choice(block.expand_ratio_list)) + + # sample depth + depth_setting = [random.choice([max(self.depth_list), min(self.depth_list)])] + for stage_id in range(len(ResNets.BASE_DEPTH_LIST)): + depth_setting.append(random.choice(self.depth_list)) + + # sample width_mult + width_mult_setting = [ + random.choice(list(range(len(self.input_stem[0].out_channel_list)))), + random.choice(list(range(len(self.input_stem[2].out_channel_list)))), + ] + for stage_id, block_idx in enumerate(self.grouped_block_index): + stage_first_block = self.blocks[block_idx[0]] + width_mult_setting.append( + random.choice(list(range(len(stage_first_block.out_channel_list)))) + ) + + arch_config = { + 'd': depth_setting, + 'e': expand_setting, + 'w': width_mult_setting + } + self.set_active_subnet(**arch_config) + return arch_config + + def get_active_subnet(self, preserve_weight=True): + input_stem = [self.input_stem[0].get_active_subnet(3, preserve_weight)] + if self.input_stem_skipping <= 0: + input_stem.append(ResidualBlock( + self.input_stem[1].conv.get_active_subnet(self.input_stem[0].active_out_channel, preserve_weight), + IdentityLayer(self.input_stem[0].active_out_channel, self.input_stem[0].active_out_channel) + )) + input_stem.append(self.input_stem[2].get_active_subnet(self.input_stem[0].active_out_channel, preserve_weight)) + input_channel = self.input_stem[2].active_out_channel + + blocks = [] + for stage_id, block_idx in enumerate(self.grouped_block_index): + depth_param = self.runtime_depth[stage_id] + active_idx = block_idx[:len(block_idx) - depth_param] + for idx in active_idx: + blocks.append(self.blocks[idx].get_active_subnet(input_channel, preserve_weight)) + input_channel = self.blocks[idx].active_out_channel + classifier = self.classifier.get_active_subnet(input_channel, preserve_weight) + subnet = ResNets(input_stem, blocks, classifier) + + subnet.set_bn_param(**self.get_bn_param()) + return subnet + + def get_active_net_config(self): + input_stem_config = [self.input_stem[0].get_active_subnet_config(3)] + if self.input_stem_skipping <= 0: + input_stem_config.append({ + 'name': ResidualBlock.__name__, + 'conv': self.input_stem[1].conv.get_active_subnet_config(self.input_stem[0].active_out_channel), + 'shortcut': IdentityLayer(self.input_stem[0].active_out_channel, self.input_stem[0].active_out_channel), + }) + input_stem_config.append(self.input_stem[2].get_active_subnet_config(self.input_stem[0].active_out_channel)) + input_channel = self.input_stem[2].active_out_channel + + blocks_config = [] + for stage_id, block_idx in enumerate(self.grouped_block_index): + depth_param = self.runtime_depth[stage_id] + active_idx = block_idx[:len(block_idx) - depth_param] + for idx in active_idx: + blocks_config.append(self.blocks[idx].get_active_subnet_config(input_channel)) + input_channel = self.blocks[idx].active_out_channel + classifier_config = self.classifier.get_active_subnet_config(input_channel) + return { + 'name': ResNets.__name__, + 'bn': self.get_bn_param(), + 'input_stem': input_stem_config, + 'blocks': blocks_config, + 'classifier': classifier_config, + } + + """ Width Related Methods """ + + def re_organize_middle_weights(self, expand_ratio_stage=0): + for block in self.blocks: + block.re_organize_middle_weights(expand_ratio_stage) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/training/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/training/__init__.py new file mode 100644 index 0000000..219b8fc --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/training/__init__.py @@ -0,0 +1,5 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .progressive_shrinking import * diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/training/progressive_shrinking.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/training/progressive_shrinking.py new file mode 100644 index 0000000..1aad90e --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/training/progressive_shrinking.py @@ -0,0 +1,320 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import torch.nn as nn +import random +import time +import torch +import torch.nn.functional as F +from tqdm import tqdm + +from ofa.utils import AverageMeter, cross_entropy_loss_with_soft_target +from ofa.utils import DistributedMetric, list_mean, subset_mean, val2list, MyRandomResizedCrop +from ofa.imagenet_classification.run_manager import DistributedRunManager + +__all__ = [ + 'validate', 'train_one_epoch', 'train', 'load_models', + 'train_elastic_depth', 'train_elastic_expand', 'train_elastic_width_mult', +] + + +def validate(run_manager, epoch=0, is_test=False, image_size_list=None, + ks_list=None, expand_ratio_list=None, depth_list=None, width_mult_list=None, additional_setting=None): + dynamic_net = run_manager.net + if isinstance(dynamic_net, nn.DataParallel): + dynamic_net = dynamic_net.module + + dynamic_net.eval() + + if image_size_list is None: + image_size_list = val2list(run_manager.run_config.data_provider.image_size, 1) + if ks_list is None: + ks_list = dynamic_net.ks_list + if expand_ratio_list is None: + expand_ratio_list = dynamic_net.expand_ratio_list + if depth_list is None: + depth_list = dynamic_net.depth_list + if width_mult_list is None: + if 'width_mult_list' in dynamic_net.__dict__: + width_mult_list = list(range(len(dynamic_net.width_mult_list))) + else: + width_mult_list = [0] + + subnet_settings = [] + for d in depth_list: + for e in expand_ratio_list: + for k in ks_list: + for w in width_mult_list: + for img_size in image_size_list: + subnet_settings.append([{ + 'image_size': img_size, + 'd': d, + 'e': e, + 'ks': k, + 'w': w, + }, 'R%s-D%s-E%s-K%s-W%s' % (img_size, d, e, k, w)]) + if additional_setting is not None: + subnet_settings += additional_setting + + losses_of_subnets, top1_of_subnets, top5_of_subnets = [], [], [] + + valid_log = '' + for setting, name in subnet_settings: + run_manager.write_log('-' * 30 + ' Validate %s ' % name + '-' * 30, 'train', should_print=False) + run_manager.run_config.data_provider.assign_active_img_size(setting.pop('image_size')) + dynamic_net.set_active_subnet(**setting) + run_manager.write_log(dynamic_net.module_str, 'train', should_print=False) + + run_manager.reset_running_statistics(dynamic_net) + loss, (top1, top5) = run_manager.validate(epoch=epoch, is_test=is_test, run_str=name, net=dynamic_net) + losses_of_subnets.append(loss) + top1_of_subnets.append(top1) + top5_of_subnets.append(top5) + valid_log += '%s (%.3f), ' % (name, top1) + + return list_mean(losses_of_subnets), list_mean(top1_of_subnets), list_mean(top5_of_subnets), valid_log + + +def train_one_epoch(run_manager, args, epoch, warmup_epochs=0, warmup_lr=0): + dynamic_net = run_manager.network + distributed = isinstance(run_manager, DistributedRunManager) + + # switch to train mode + dynamic_net.train() + if distributed: + run_manager.run_config.train_loader.sampler.set_epoch(epoch) + MyRandomResizedCrop.EPOCH = epoch + + nBatch = len(run_manager.run_config.train_loader) + + data_time = AverageMeter() + losses = DistributedMetric('train_loss') if distributed else AverageMeter() + metric_dict = run_manager.get_metric_dict() + + with tqdm(total=nBatch, + desc='Train Epoch #{}'.format(epoch + 1), + disable=distributed and not run_manager.is_root) as t: + end = time.time() + for i, (images, labels) in enumerate(run_manager.run_config.train_loader): + MyRandomResizedCrop.BATCH = i + data_time.update(time.time() - end) + if epoch < warmup_epochs: + new_lr = run_manager.run_config.warmup_adjust_learning_rate( + run_manager.optimizer, warmup_epochs * nBatch, nBatch, epoch, i, warmup_lr, + ) + else: + new_lr = run_manager.run_config.adjust_learning_rate( + run_manager.optimizer, epoch - warmup_epochs, i, nBatch + ) + + images, labels = images.cuda(), labels.cuda() + target = labels + + # soft target + if args.kd_ratio > 0: + args.teacher_model.train() + with torch.no_grad(): + soft_logits = args.teacher_model(images).detach() + soft_label = F.softmax(soft_logits, dim=1) + + # clean gradients + dynamic_net.zero_grad() + + loss_of_subnets = [] + # compute output + subnet_str = '' + for _ in range(args.dynamic_batch_size): + # set random seed before sampling + subnet_seed = int('%d%.3d%.3d' % (epoch * nBatch + i, _, 0)) + random.seed(subnet_seed) + subnet_settings = dynamic_net.sample_active_subnet() + subnet_str += '%d: ' % _ + ','.join(['%s_%s' % ( + key, '%.1f' % subset_mean(val, 0) if isinstance(val, list) else val + ) for key, val in subnet_settings.items()]) + ' || ' + + output = run_manager.net(images) + if args.kd_ratio == 0: + loss = run_manager.train_criterion(output, labels) + loss_type = 'ce' + else: + if args.kd_type == 'ce': + kd_loss = cross_entropy_loss_with_soft_target(output, soft_label) + else: + kd_loss = F.mse_loss(output, soft_logits) + loss = args.kd_ratio * kd_loss + run_manager.train_criterion(output, labels) + loss_type = '%.1fkd-%s & ce' % (args.kd_ratio, args.kd_type) + + # measure accuracy and record loss + loss_of_subnets.append(loss) + run_manager.update_metric(metric_dict, output, target) + + loss.backward() + run_manager.optimizer.step() + + losses.update(list_mean(loss_of_subnets), images.size(0)) + + t.set_postfix({ + 'loss': losses.avg.item(), + **run_manager.get_metric_vals(metric_dict, return_dict=True), + 'R': images.size(2), + 'lr': new_lr, + 'loss_type': loss_type, + 'seed': str(subnet_seed), + 'str': subnet_str, + 'data_time': data_time.avg, + }) + t.update(1) + end = time.time() + return losses.avg.item(), run_manager.get_metric_vals(metric_dict) + + +def train(run_manager, args, validate_func=None): + distributed = isinstance(run_manager, DistributedRunManager) + if validate_func is None: + validate_func = validate + + for epoch in range(run_manager.start_epoch, run_manager.run_config.n_epochs + args.warmup_epochs): + train_loss, (train_top1, train_top5) = train_one_epoch( + run_manager, args, epoch, args.warmup_epochs, args.warmup_lr) + + if (epoch + 1) % args.validation_frequency == 0: + val_loss, val_acc, val_acc5, _val_log = validate_func(run_manager, epoch=epoch, is_test=False) + # best_acc + is_best = val_acc > run_manager.best_acc + run_manager.best_acc = max(run_manager.best_acc, val_acc) + if not distributed or run_manager.is_root: + val_log = 'Valid [{0}/{1}] loss={2:.3f}, top-1={3:.3f} ({4:.3f})'. \ + format(epoch + 1 - args.warmup_epochs, run_manager.run_config.n_epochs, val_loss, val_acc, + run_manager.best_acc) + val_log += ', Train top-1 {top1:.3f}, Train loss {loss:.3f}\t'.format(top1=train_top1, loss=train_loss) + val_log += _val_log + run_manager.write_log(val_log, 'valid', should_print=False) + + run_manager.save_model({ + 'epoch': epoch, + 'best_acc': run_manager.best_acc, + 'optimizer': run_manager.optimizer.state_dict(), + 'state_dict': run_manager.network.state_dict(), + }, is_best=is_best) + + +def load_models(run_manager, dynamic_net, model_path=None): + # specify init path + init = torch.load(model_path, map_location='cpu')['state_dict'] + dynamic_net.load_state_dict(init) + run_manager.write_log('Loaded init from %s' % model_path, 'valid') + + +def train_elastic_depth(train_func, run_manager, args, validate_func_dict): + dynamic_net = run_manager.net + if isinstance(dynamic_net, nn.DataParallel): + dynamic_net = dynamic_net.module + + depth_stage_list = dynamic_net.depth_list.copy() + depth_stage_list.sort(reverse=True) + n_stages = len(depth_stage_list) - 1 + current_stage = n_stages - 1 + + # load pretrained models + if run_manager.start_epoch == 0 and not args.resume: + validate_func_dict['depth_list'] = sorted(dynamic_net.depth_list) + + load_models(run_manager, dynamic_net, model_path=args.ofa_checkpoint_path) + # validate after loading weights + run_manager.write_log('%.3f\t%.3f\t%.3f\t%s' % + validate(run_manager, is_test=True, **validate_func_dict), 'valid') + else: + assert args.resume + + run_manager.write_log( + '-' * 30 + 'Supporting Elastic Depth: %s -> %s' % + (depth_stage_list[:current_stage + 1], depth_stage_list[:current_stage + 2]) + '-' * 30, 'valid' + ) + # add depth list constraints + if len(set(dynamic_net.ks_list)) == 1 and len(set(dynamic_net.expand_ratio_list)) == 1: + validate_func_dict['depth_list'] = depth_stage_list + else: + validate_func_dict['depth_list'] = sorted({min(depth_stage_list), max(depth_stage_list)}) + + # train + train_func( + run_manager, args, + lambda _run_manager, epoch, is_test: validate(_run_manager, epoch, is_test, **validate_func_dict) + ) + + +def train_elastic_expand(train_func, run_manager, args, validate_func_dict): + dynamic_net = run_manager.net + if isinstance(dynamic_net, nn.DataParallel): + dynamic_net = dynamic_net.module + + expand_stage_list = dynamic_net.expand_ratio_list.copy() + expand_stage_list.sort(reverse=True) + n_stages = len(expand_stage_list) - 1 + current_stage = n_stages - 1 + + # load pretrained models + if run_manager.start_epoch == 0 and not args.resume: + validate_func_dict['expand_ratio_list'] = sorted(dynamic_net.expand_ratio_list) + + load_models(run_manager, dynamic_net, model_path=args.ofa_checkpoint_path) + dynamic_net.re_organize_middle_weights(expand_ratio_stage=current_stage) + run_manager.write_log('%.3f\t%.3f\t%.3f\t%s' % + validate(run_manager, is_test=True, **validate_func_dict), 'valid') + else: + assert args.resume + + run_manager.write_log( + '-' * 30 + 'Supporting Elastic Expand Ratio: %s -> %s' % + (expand_stage_list[:current_stage + 1], expand_stage_list[:current_stage + 2]) + '-' * 30, 'valid' + ) + if len(set(dynamic_net.ks_list)) == 1 and len(set(dynamic_net.depth_list)) == 1: + validate_func_dict['expand_ratio_list'] = expand_stage_list + else: + validate_func_dict['expand_ratio_list'] = sorted({min(expand_stage_list), max(expand_stage_list)}) + + # train + train_func( + run_manager, args, + lambda _run_manager, epoch, is_test: validate(_run_manager, epoch, is_test, **validate_func_dict) + ) + + +def train_elastic_width_mult(train_func, run_manager, args, validate_func_dict): + dynamic_net = run_manager.net + if isinstance(dynamic_net, nn.DataParallel): + dynamic_net = dynamic_net.module + + width_stage_list = dynamic_net.width_mult_list.copy() + width_stage_list.sort(reverse=True) + n_stages = len(width_stage_list) - 1 + current_stage = n_stages - 1 + + if run_manager.start_epoch == 0 and not args.resume: + load_models(run_manager, dynamic_net, model_path=args.ofa_checkpoint_path) + if current_stage == 0: + dynamic_net.re_organize_middle_weights(expand_ratio_stage=len(dynamic_net.expand_ratio_list) - 1) + run_manager.write_log('reorganize_middle_weights (expand_ratio_stage=%d)' + % (len(dynamic_net.expand_ratio_list) - 1), 'valid') + try: + dynamic_net.re_organize_outer_weights() + run_manager.write_log('reorganize_outer_weights', 'valid') + except Exception: + pass + run_manager.write_log('%.3f\t%.3f\t%.3f\t%s' % + validate(run_manager, is_test=True, **validate_func_dict), 'valid') + else: + assert args.resume + + run_manager.write_log( + '-' * 30 + 'Supporting Elastic Width Mult: %s -> %s' % + (width_stage_list[:current_stage + 1], width_stage_list[:current_stage + 2]) + '-' * 30, 'valid' + ) + validate_func_dict['width_mult_list'] = sorted({0, len(width_stage_list) - 1}) + + # train + train_func( + run_manager, args, + lambda _run_manager, epoch, is_test: validate(_run_manager, epoch, is_test, **validate_func_dict) + ) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/utils.py new file mode 100644 index 0000000..a7fc834 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/elastic_nn/utils.py @@ -0,0 +1,70 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import copy +import torch.nn.functional as F +import torch.nn as nn +import torch + +from ofa_local.utils import AverageMeter, get_net_device, DistributedTensor +from ofa_local.imagenet_classification.elastic_nn.modules.dynamic_op import DynamicBatchNorm2d + +__all__ = ['set_running_statistics'] + + +def set_running_statistics(model, data_loader, distributed=False): + bn_mean = {} + bn_var = {} + + forward_model = copy.deepcopy(model) + for name, m in forward_model.named_modules(): + if isinstance(m, nn.BatchNorm2d): + if distributed: + bn_mean[name] = DistributedTensor(name + '#mean') + bn_var[name] = DistributedTensor(name + '#var') + else: + bn_mean[name] = AverageMeter() + bn_var[name] = AverageMeter() + + def new_forward(bn, mean_est, var_est): + def lambda_forward(x): + batch_mean = x.mean(0, keepdim=True).mean(2, keepdim=True).mean(3, keepdim=True) # 1, C, 1, 1 + batch_var = (x - batch_mean) * (x - batch_mean) + batch_var = batch_var.mean(0, keepdim=True).mean(2, keepdim=True).mean(3, keepdim=True) + + batch_mean = torch.squeeze(batch_mean) + batch_var = torch.squeeze(batch_var) + + mean_est.update(batch_mean.data, x.size(0)) + var_est.update(batch_var.data, x.size(0)) + + # bn forward using calculated mean & var + _feature_dim = batch_mean.size(0) + return F.batch_norm( + x, batch_mean, batch_var, bn.weight[:_feature_dim], + bn.bias[:_feature_dim], False, + 0.0, bn.eps, + ) + + return lambda_forward + + m.forward = new_forward(m, bn_mean[name], bn_var[name]) + + if len(bn_mean) == 0: + # skip if there is no batch normalization layers in the network + return + + with torch.no_grad(): + DynamicBatchNorm2d.SET_RUNNING_STATISTICS = True + for images, labels in data_loader: + images = images.to(get_net_device(forward_model)) + forward_model(images) + DynamicBatchNorm2d.SET_RUNNING_STATISTICS = False + + for name, m in model.named_modules(): + if name in bn_mean and bn_mean[name].count > 0: + feature_dim = bn_mean[name].avg.size(0) + assert isinstance(m, nn.BatchNorm2d) + m.running_mean.data[:feature_dim].copy_(bn_mean[name].avg) + m.running_var.data[:feature_dim].copy_(bn_var[name].avg) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/__init__.py new file mode 100644 index 0000000..1b1d83f --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/__init__.py @@ -0,0 +1,18 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .proxyless_nets import * +from .mobilenet_v3 import * +from .resnets import * + + +def get_net_by_name(name): + if name == ProxylessNASNets.__name__: + return ProxylessNASNets + elif name == MobileNetV3.__name__: + return MobileNetV3 + elif name == ResNets.__name__: + return ResNets + else: + raise ValueError('unrecognized type of network: %s' % name) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/mobilenet_v3.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/mobilenet_v3.py new file mode 100644 index 0000000..25e4061 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/mobilenet_v3.py @@ -0,0 +1,218 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import copy +import torch.nn as nn + +from ofa_local.utils.layers import set_layer_from_config, MBConvLayer, ConvLayer, IdentityLayer, LinearLayer, ResidualBlock +from ofa_local.utils import MyNetwork, make_divisible, MyGlobalAvgPool2d + +__all__ = ['MobileNetV3', 'MobileNetV3Large'] + + +class MobileNetV3(MyNetwork): + + def __init__(self, first_conv, blocks, final_expand_layer, feature_mix_layer, classifier): + super(MobileNetV3, self).__init__() + + self.first_conv = first_conv + self.blocks = nn.ModuleList(blocks) + self.final_expand_layer = final_expand_layer + self.global_avg_pool = MyGlobalAvgPool2d(keep_dim=True) + self.feature_mix_layer = feature_mix_layer + self.classifier = classifier + + def forward(self, x): + x = self.first_conv(x) + for block in self.blocks: + x = block(x) + x = self.final_expand_layer(x) + x = self.global_avg_pool(x) # global average pooling + x = self.feature_mix_layer(x) + x = x.view(x.size(0), -1) + x = self.classifier(x) + return x + + @property + def module_str(self): + _str = self.first_conv.module_str + '\n' + for block in self.blocks: + _str += block.module_str + '\n' + _str += self.final_expand_layer.module_str + '\n' + _str += self.global_avg_pool.__repr__() + '\n' + _str += self.feature_mix_layer.module_str + '\n' + _str += self.classifier.module_str + return _str + + @property + def config(self): + return { + 'name': MobileNetV3.__name__, + 'bn': self.get_bn_param(), + 'first_conv': self.first_conv.config, + 'blocks': [ + block.config for block in self.blocks + ], + 'final_expand_layer': self.final_expand_layer.config, + 'feature_mix_layer': self.feature_mix_layer.config, + 'classifier': self.classifier.config, + } + + @staticmethod + def build_from_config(config): + first_conv = set_layer_from_config(config['first_conv']) + final_expand_layer = set_layer_from_config(config['final_expand_layer']) + feature_mix_layer = set_layer_from_config(config['feature_mix_layer']) + classifier = set_layer_from_config(config['classifier']) + + blocks = [] + for block_config in config['blocks']: + blocks.append(ResidualBlock.build_from_config(block_config)) + + net = MobileNetV3(first_conv, blocks, final_expand_layer, feature_mix_layer, classifier) + if 'bn' in config: + net.set_bn_param(**config['bn']) + else: + net.set_bn_param(momentum=0.1, eps=1e-5) + + return net + + def zero_last_gamma(self): + for m in self.modules(): + if isinstance(m, ResidualBlock): + if isinstance(m.conv, MBConvLayer) and isinstance(m.shortcut, IdentityLayer): + m.conv.point_linear.bn.weight.data.zero_() + + @property + def grouped_block_index(self): + info_list = [] + block_index_list = [] + for i, block in enumerate(self.blocks[1:], 1): + if block.shortcut is None and len(block_index_list) > 0: + info_list.append(block_index_list) + block_index_list = [] + block_index_list.append(i) + if len(block_index_list) > 0: + info_list.append(block_index_list) + return info_list + + @staticmethod + def build_net_via_cfg(cfg, input_channel, last_channel, n_classes, dropout_rate): + # first conv layer + first_conv = ConvLayer( + 3, input_channel, kernel_size=3, stride=2, use_bn=True, act_func='h_swish', ops_order='weight_bn_act' + ) + # build mobile blocks + feature_dim = input_channel + blocks = [] + for stage_id, block_config_list in cfg.items(): + for k, mid_channel, out_channel, use_se, act_func, stride, expand_ratio in block_config_list: + mb_conv = MBConvLayer( + feature_dim, out_channel, k, stride, expand_ratio, mid_channel, act_func, use_se + ) + if stride == 1 and out_channel == feature_dim: + shortcut = IdentityLayer(out_channel, out_channel) + else: + shortcut = None + blocks.append(ResidualBlock(mb_conv, shortcut)) + feature_dim = out_channel + # final expand layer + final_expand_layer = ConvLayer( + feature_dim, feature_dim * 6, kernel_size=1, use_bn=True, act_func='h_swish', ops_order='weight_bn_act', + ) + # feature mix layer + feature_mix_layer = ConvLayer( + feature_dim * 6, last_channel, kernel_size=1, bias=False, use_bn=False, act_func='h_swish', + ) + # classifier + classifier = LinearLayer(last_channel, n_classes, dropout_rate=dropout_rate) + + return first_conv, blocks, final_expand_layer, feature_mix_layer, classifier + + @staticmethod + def adjust_cfg(cfg, ks=None, expand_ratio=None, depth_param=None, stage_width_list=None): + for i, (stage_id, block_config_list) in enumerate(cfg.items()): + for block_config in block_config_list: + if ks is not None and stage_id != '0': + block_config[0] = ks + if expand_ratio is not None and stage_id != '0': + block_config[-1] = expand_ratio + block_config[1] = None + if stage_width_list is not None: + block_config[2] = stage_width_list[i] + if depth_param is not None and stage_id != '0': + new_block_config_list = [block_config_list[0]] + new_block_config_list += [copy.deepcopy(block_config_list[-1]) for _ in range(depth_param - 1)] + cfg[stage_id] = new_block_config_list + return cfg + + def load_state_dict(self, state_dict, **kwargs): + current_state_dict = self.state_dict() + + for key in state_dict: + if key not in current_state_dict: + assert '.mobile_inverted_conv.' in key + new_key = key.replace('.mobile_inverted_conv.', '.conv.') + else: + new_key = key + current_state_dict[new_key] = state_dict[key] + super(MobileNetV3, self).load_state_dict(current_state_dict) + + +class MobileNetV3Large(MobileNetV3): + + def __init__(self, n_classes=1000, width_mult=1.0, bn_param=(0.1, 1e-5), dropout_rate=0.2, + ks=None, expand_ratio=None, depth_param=None, stage_width_list=None): + input_channel = 16 + last_channel = 1280 + + input_channel = make_divisible(input_channel * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + last_channel = make_divisible(last_channel * width_mult, MyNetwork.CHANNEL_DIVISIBLE) \ + if width_mult > 1.0 else last_channel + + cfg = { + # k, exp, c, se, nl, s, e, + '0': [ + [3, 16, 16, False, 'relu', 1, 1], + ], + '1': [ + [3, 64, 24, False, 'relu', 2, None], # 4 + [3, 72, 24, False, 'relu', 1, None], # 3 + ], + '2': [ + [5, 72, 40, True, 'relu', 2, None], # 3 + [5, 120, 40, True, 'relu', 1, None], # 3 + [5, 120, 40, True, 'relu', 1, None], # 3 + ], + '3': [ + [3, 240, 80, False, 'h_swish', 2, None], # 6 + [3, 200, 80, False, 'h_swish', 1, None], # 2.5 + [3, 184, 80, False, 'h_swish', 1, None], # 2.3 + [3, 184, 80, False, 'h_swish', 1, None], # 2.3 + ], + '4': [ + [3, 480, 112, True, 'h_swish', 1, None], # 6 + [3, 672, 112, True, 'h_swish', 1, None], # 6 + ], + '5': [ + [5, 672, 160, True, 'h_swish', 2, None], # 6 + [5, 960, 160, True, 'h_swish', 1, None], # 6 + [5, 960, 160, True, 'h_swish', 1, None], # 6 + ] + } + + cfg = self.adjust_cfg(cfg, ks, expand_ratio, depth_param, stage_width_list) + # width multiplier on mobile setting, change `exp: 1` and `c: 2` + for stage_id, block_config_list in cfg.items(): + for block_config in block_config_list: + if block_config[1] is not None: + block_config[1] = make_divisible(block_config[1] * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + block_config[2] = make_divisible(block_config[2] * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + + first_conv, blocks, final_expand_layer, feature_mix_layer, classifier = self.build_net_via_cfg( + cfg, input_channel, last_channel, n_classes, dropout_rate + ) + super(MobileNetV3Large, self).__init__(first_conv, blocks, final_expand_layer, feature_mix_layer, classifier) + # set bn param + self.set_bn_param(*bn_param) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/proxyless_nets.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/proxyless_nets.py new file mode 100644 index 0000000..f1610be --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/proxyless_nets.py @@ -0,0 +1,210 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import json +import torch.nn as nn + +from ofa_local.utils.layers import set_layer_from_config, MBConvLayer, ConvLayer, IdentityLayer, LinearLayer, ResidualBlock +from ofa_local.utils import download_url, make_divisible, val2list, MyNetwork, MyGlobalAvgPool2d + +__all__ = ['proxyless_base', 'ProxylessNASNets', 'MobileNetV2'] + + +def proxyless_base(net_config=None, n_classes=None, bn_param=None, dropout_rate=None, + local_path='~/.torch/proxylessnas/'): + assert net_config is not None, 'Please input a network config' + if 'http' in net_config: + net_config_path = download_url(net_config, local_path) + else: + net_config_path = net_config + net_config_json = json.load(open(net_config_path, 'r')) + + if n_classes is not None: + net_config_json['classifier']['out_features'] = n_classes + if dropout_rate is not None: + net_config_json['classifier']['dropout_rate'] = dropout_rate + + net = ProxylessNASNets.build_from_config(net_config_json) + if bn_param is not None: + net.set_bn_param(*bn_param) + + return net + + +class ProxylessNASNets(MyNetwork): + + def __init__(self, first_conv, blocks, feature_mix_layer, classifier): + super(ProxylessNASNets, self).__init__() + + self.first_conv = first_conv + self.blocks = nn.ModuleList(blocks) + self.feature_mix_layer = feature_mix_layer + self.global_avg_pool = MyGlobalAvgPool2d(keep_dim=False) + self.classifier = classifier + + def forward(self, x): + x = self.first_conv(x) + for block in self.blocks: + x = block(x) + if self.feature_mix_layer is not None: + x = self.feature_mix_layer(x) + x = self.global_avg_pool(x) + x = self.classifier(x) + return x + + @property + def module_str(self): + _str = self.first_conv.module_str + '\n' + for block in self.blocks: + _str += block.module_str + '\n' + _str += self.feature_mix_layer.module_str + '\n' + _str += self.global_avg_pool.__repr__() + '\n' + _str += self.classifier.module_str + return _str + + @property + def config(self): + return { + 'name': ProxylessNASNets.__name__, + 'bn': self.get_bn_param(), + 'first_conv': self.first_conv.config, + 'blocks': [ + block.config for block in self.blocks + ], + 'feature_mix_layer': None if self.feature_mix_layer is None else self.feature_mix_layer.config, + 'classifier': self.classifier.config, + } + + @staticmethod + def build_from_config(config): + first_conv = set_layer_from_config(config['first_conv']) + feature_mix_layer = set_layer_from_config(config['feature_mix_layer']) + classifier = set_layer_from_config(config['classifier']) + + blocks = [] + for block_config in config['blocks']: + blocks.append(ResidualBlock.build_from_config(block_config)) + + net = ProxylessNASNets(first_conv, blocks, feature_mix_layer, classifier) + if 'bn' in config: + net.set_bn_param(**config['bn']) + else: + net.set_bn_param(momentum=0.1, eps=1e-3) + + return net + + def zero_last_gamma(self): + for m in self.modules(): + if isinstance(m, ResidualBlock): + if isinstance(m.conv, MBConvLayer) and isinstance(m.shortcut, IdentityLayer): + m.conv.point_linear.bn.weight.data.zero_() + + @property + def grouped_block_index(self): + info_list = [] + block_index_list = [] + for i, block in enumerate(self.blocks[1:], 1): + if block.shortcut is None and len(block_index_list) > 0: + info_list.append(block_index_list) + block_index_list = [] + block_index_list.append(i) + if len(block_index_list) > 0: + info_list.append(block_index_list) + return info_list + + def load_state_dict(self, state_dict, **kwargs): + current_state_dict = self.state_dict() + + for key in state_dict: + if key not in current_state_dict: + assert '.mobile_inverted_conv.' in key + new_key = key.replace('.mobile_inverted_conv.', '.conv.') + else: + new_key = key + current_state_dict[new_key] = state_dict[key] + super(ProxylessNASNets, self).load_state_dict(current_state_dict) + + +class MobileNetV2(ProxylessNASNets): + + def __init__(self, n_classes=1000, width_mult=1.0, bn_param=(0.1, 1e-3), dropout_rate=0.2, + ks=None, expand_ratio=None, depth_param=None, stage_width_list=None): + + ks = 3 if ks is None else ks + expand_ratio = 6 if expand_ratio is None else expand_ratio + + input_channel = 32 + last_channel = 1280 + + input_channel = make_divisible(input_channel * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + last_channel = make_divisible(last_channel * width_mult, MyNetwork.CHANNEL_DIVISIBLE) \ + if width_mult > 1.0 else last_channel + + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [expand_ratio, 24, 2, 2], + [expand_ratio, 32, 3, 2], + [expand_ratio, 64, 4, 2], + [expand_ratio, 96, 3, 1], + [expand_ratio, 160, 3, 2], + [expand_ratio, 320, 1, 1], + ] + + if depth_param is not None: + assert isinstance(depth_param, int) + for i in range(1, len(inverted_residual_setting) - 1): + inverted_residual_setting[i][2] = depth_param + + if stage_width_list is not None: + for i in range(len(inverted_residual_setting)): + inverted_residual_setting[i][1] = stage_width_list[i] + + ks = val2list(ks, sum([n for _, _, n, _ in inverted_residual_setting]) - 1) + _pt = 0 + + # first conv layer + first_conv = ConvLayer( + 3, input_channel, kernel_size=3, stride=2, use_bn=True, act_func='relu6', ops_order='weight_bn_act' + ) + # inverted residual blocks + blocks = [] + for t, c, n, s in inverted_residual_setting: + output_channel = make_divisible(c * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + for i in range(n): + if i == 0: + stride = s + else: + stride = 1 + if t == 1: + kernel_size = 3 + else: + kernel_size = ks[_pt] + _pt += 1 + mobile_inverted_conv = MBConvLayer( + in_channels=input_channel, out_channels=output_channel, kernel_size=kernel_size, stride=stride, + expand_ratio=t, + ) + if stride == 1: + if input_channel == output_channel: + shortcut = IdentityLayer(input_channel, input_channel) + else: + shortcut = None + else: + shortcut = None + blocks.append( + ResidualBlock(mobile_inverted_conv, shortcut) + ) + input_channel = output_channel + # 1x1_conv before global average pooling + feature_mix_layer = ConvLayer( + input_channel, last_channel, kernel_size=1, use_bn=True, act_func='relu6', ops_order='weight_bn_act', + ) + + classifier = LinearLayer(last_channel, n_classes, dropout_rate=dropout_rate) + + super(MobileNetV2, self).__init__(first_conv, blocks, feature_mix_layer, classifier) + + # set bn param + self.set_bn_param(*bn_param) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/resnets.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/resnets.py new file mode 100644 index 0000000..a354492 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/networks/resnets.py @@ -0,0 +1,192 @@ +import torch.nn as nn + +from ofa_local.utils.layers import set_layer_from_config, ConvLayer, IdentityLayer, LinearLayer +from ofa_local.utils.layers import ResNetBottleneckBlock, ResidualBlock +from ofa_local.utils import make_divisible, MyNetwork, MyGlobalAvgPool2d + +__all__ = ['ResNets', 'ResNet50', 'ResNet50D'] + + +class ResNets(MyNetwork): + + BASE_DEPTH_LIST = [2, 2, 4, 2] + STAGE_WIDTH_LIST = [256, 512, 1024, 2048] + + def __init__(self, input_stem, blocks, classifier): + super(ResNets, self).__init__() + + self.input_stem = nn.ModuleList(input_stem) + self.max_pooling = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False) + self.blocks = nn.ModuleList(blocks) + self.global_avg_pool = MyGlobalAvgPool2d(keep_dim=False) + self.classifier = classifier + + def forward(self, x): + for layer in self.input_stem: + x = layer(x) + x = self.max_pooling(x) + for block in self.blocks: + x = block(x) + x = self.global_avg_pool(x) + x = self.classifier(x) + return x + + @property + def module_str(self): + _str = '' + for layer in self.input_stem: + _str += layer.module_str + '\n' + _str += 'max_pooling(ks=3, stride=2)\n' + for block in self.blocks: + _str += block.module_str + '\n' + _str += self.global_avg_pool.__repr__() + '\n' + _str += self.classifier.module_str + return _str + + @property + def config(self): + return { + 'name': ResNets.__name__, + 'bn': self.get_bn_param(), + 'input_stem': [ + layer.config for layer in self.input_stem + ], + 'blocks': [ + block.config for block in self.blocks + ], + 'classifier': self.classifier.config, + } + + @staticmethod + def build_from_config(config): + classifier = set_layer_from_config(config['classifier']) + + input_stem = [] + for layer_config in config['input_stem']: + input_stem.append(set_layer_from_config(layer_config)) + blocks = [] + for block_config in config['blocks']: + blocks.append(set_layer_from_config(block_config)) + + net = ResNets(input_stem, blocks, classifier) + if 'bn' in config: + net.set_bn_param(**config['bn']) + else: + net.set_bn_param(momentum=0.1, eps=1e-5) + + return net + + def zero_last_gamma(self): + for m in self.modules(): + if isinstance(m, ResNetBottleneckBlock) and isinstance(m.downsample, IdentityLayer): + m.conv3.bn.weight.data.zero_() + + @property + def grouped_block_index(self): + info_list = [] + block_index_list = [] + for i, block in enumerate(self.blocks): + if not isinstance(block.downsample, IdentityLayer) and len(block_index_list) > 0: + info_list.append(block_index_list) + block_index_list = [] + block_index_list.append(i) + if len(block_index_list) > 0: + info_list.append(block_index_list) + return info_list + + def load_state_dict(self, state_dict, **kwargs): + super(ResNets, self).load_state_dict(state_dict) + + +class ResNet50(ResNets): + + def __init__(self, n_classes=1000, width_mult=1.0, bn_param=(0.1, 1e-5), dropout_rate=0, + expand_ratio=None, depth_param=None): + + expand_ratio = 0.25 if expand_ratio is None else expand_ratio + + input_channel = make_divisible(64 * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + stage_width_list = ResNets.STAGE_WIDTH_LIST.copy() + for i, width in enumerate(stage_width_list): + stage_width_list[i] = make_divisible(width * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + + depth_list = [3, 4, 6, 3] + if depth_param is not None: + for i, depth in enumerate(ResNets.BASE_DEPTH_LIST): + depth_list[i] = depth + depth_param + + stride_list = [1, 2, 2, 2] + + # build input stem + input_stem = [ConvLayer( + 3, input_channel, kernel_size=7, stride=2, use_bn=True, act_func='relu', ops_order='weight_bn_act', + )] + + # blocks + blocks = [] + for d, width, s in zip(depth_list, stage_width_list, stride_list): + for i in range(d): + stride = s if i == 0 else 1 + bottleneck_block = ResNetBottleneckBlock( + input_channel, width, kernel_size=3, stride=stride, expand_ratio=expand_ratio, + act_func='relu', downsample_mode='conv', + ) + blocks.append(bottleneck_block) + input_channel = width + # classifier + classifier = LinearLayer(input_channel, n_classes, dropout_rate=dropout_rate) + + super(ResNet50, self).__init__(input_stem, blocks, classifier) + + # set bn param + self.set_bn_param(*bn_param) + + +class ResNet50D(ResNets): + + def __init__(self, n_classes=1000, width_mult=1.0, bn_param=(0.1, 1e-5), dropout_rate=0, + expand_ratio=None, depth_param=None): + + expand_ratio = 0.25 if expand_ratio is None else expand_ratio + + input_channel = make_divisible(64 * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + mid_input_channel = make_divisible(input_channel // 2, MyNetwork.CHANNEL_DIVISIBLE) + stage_width_list = ResNets.STAGE_WIDTH_LIST.copy() + for i, width in enumerate(stage_width_list): + stage_width_list[i] = make_divisible(width * width_mult, MyNetwork.CHANNEL_DIVISIBLE) + + depth_list = [3, 4, 6, 3] + if depth_param is not None: + for i, depth in enumerate(ResNets.BASE_DEPTH_LIST): + depth_list[i] = depth + depth_param + + stride_list = [1, 2, 2, 2] + + # build input stem + input_stem = [ + ConvLayer(3, mid_input_channel, 3, stride=2, use_bn=True, act_func='relu'), + ResidualBlock( + ConvLayer(mid_input_channel, mid_input_channel, 3, stride=1, use_bn=True, act_func='relu'), + IdentityLayer(mid_input_channel, mid_input_channel) + ), + ConvLayer(mid_input_channel, input_channel, 3, stride=1, use_bn=True, act_func='relu') + ] + + # blocks + blocks = [] + for d, width, s in zip(depth_list, stage_width_list, stride_list): + for i in range(d): + stride = s if i == 0 else 1 + bottleneck_block = ResNetBottleneckBlock( + input_channel, width, kernel_size=3, stride=stride, expand_ratio=expand_ratio, + act_func='relu', downsample_mode='avgpool_conv', + ) + blocks.append(bottleneck_block) + input_channel = width + # classifier + classifier = LinearLayer(input_channel, n_classes, dropout_rate=dropout_rate) + + super(ResNet50D, self).__init__(input_stem, blocks, classifier) + + # set bn param + self.set_bn_param(*bn_param) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/__init__.py new file mode 100644 index 0000000..57a83b5 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/__init__.py @@ -0,0 +1,7 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .run_config import * +from .run_manager import * +from .distributed_run_manager import * diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/distributed_run_manager.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/distributed_run_manager.py new file mode 100644 index 0000000..de54a07 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/distributed_run_manager.py @@ -0,0 +1,381 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import os +import json +import time +import random +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.backends.cudnn as cudnn +from tqdm import tqdm + +from ofa_local.utils import cross_entropy_with_label_smoothing, cross_entropy_loss_with_soft_target, write_log, init_models +from ofa_local.utils import DistributedMetric, list_mean, get_net_info, accuracy, AverageMeter, mix_labels, mix_images +from ofa_local.utils import MyRandomResizedCrop + +__all__ = ['DistributedRunManager'] + + +class DistributedRunManager: + + def __init__(self, path, net, run_config, hvd_compression, backward_steps=1, is_root=False, init=True): + import horovod.torch as hvd + + self.path = path + self.net = net + self.run_config = run_config + self.is_root = is_root + + self.best_acc = 0.0 + self.start_epoch = 0 + + os.makedirs(self.path, exist_ok=True) + + self.net.cuda() + cudnn.benchmark = True + if init and self.is_root: + init_models(self.net, self.run_config.model_init) + if self.is_root: + # print net info + net_info = get_net_info(self.net, self.run_config.data_provider.data_shape) + with open('%s/net_info.txt' % self.path, 'w') as fout: + fout.write(json.dumps(net_info, indent=4) + '\n') + try: + fout.write(self.net.module_str + '\n') + except Exception: + fout.write('%s do not support `module_str`' % type(self.net)) + fout.write('%s\n' % self.run_config.data_provider.train.dataset.transform) + fout.write('%s\n' % self.run_config.data_provider.test.dataset.transform) + fout.write('%s\n' % self.net) + + # criterion + if isinstance(self.run_config.mixup_alpha, float): + self.train_criterion = cross_entropy_loss_with_soft_target + elif self.run_config.label_smoothing > 0: + self.train_criterion = lambda pred, target: \ + cross_entropy_with_label_smoothing(pred, target, self.run_config.label_smoothing) + else: + self.train_criterion = nn.CrossEntropyLoss() + self.test_criterion = nn.CrossEntropyLoss() + + # optimizer + if self.run_config.no_decay_keys: + keys = self.run_config.no_decay_keys.split('#') + net_params = [ + self.net.get_parameters(keys, mode='exclude'), # parameters with weight decay + self.net.get_parameters(keys, mode='include'), # parameters without weight decay + ] + else: + # noinspection PyBroadException + try: + net_params = self.network.weight_parameters() + except Exception: + net_params = [] + for param in self.network.parameters(): + if param.requires_grad: + net_params.append(param) + self.optimizer = self.run_config.build_optimizer(net_params) + self.optimizer = hvd.DistributedOptimizer( + self.optimizer, named_parameters=self.net.named_parameters(), compression=hvd_compression, + backward_passes_per_step=backward_steps, + ) + + """ save path and log path """ + + @property + def save_path(self): + if self.__dict__.get('_save_path', None) is None: + save_path = os.path.join(self.path, 'checkpoint') + os.makedirs(save_path, exist_ok=True) + self.__dict__['_save_path'] = save_path + return self.__dict__['_save_path'] + + @property + def logs_path(self): + if self.__dict__.get('_logs_path', None) is None: + logs_path = os.path.join(self.path, 'logs') + os.makedirs(logs_path, exist_ok=True) + self.__dict__['_logs_path'] = logs_path + return self.__dict__['_logs_path'] + + @property + def network(self): + return self.net + + @network.setter + def network(self, new_val): + self.net = new_val + + def write_log(self, log_str, prefix='valid', should_print=True, mode='a'): + if self.is_root: + write_log(self.logs_path, log_str, prefix, should_print, mode) + + """ save & load model & save_config & broadcast """ + + def save_config(self, extra_run_config=None, extra_net_config=None): + if self.is_root: + run_save_path = os.path.join(self.path, 'run.config') + if not os.path.isfile(run_save_path): + run_config = self.run_config.config + if extra_run_config is not None: + run_config.update(extra_run_config) + json.dump(run_config, open(run_save_path, 'w'), indent=4) + print('Run configs dump to %s' % run_save_path) + + try: + net_save_path = os.path.join(self.path, 'net.config') + net_config = self.net.config + if extra_net_config is not None: + net_config.update(extra_net_config) + json.dump(net_config, open(net_save_path, 'w'), indent=4) + print('Network configs dump to %s' % net_save_path) + except Exception: + print('%s do not support net config' % type(self.net)) + + def save_model(self, checkpoint=None, is_best=False, model_name=None): + if self.is_root: + if checkpoint is None: + checkpoint = {'state_dict': self.net.state_dict()} + + if model_name is None: + model_name = 'checkpoint.pth.tar' + + latest_fname = os.path.join(self.save_path, 'latest.txt') + model_path = os.path.join(self.save_path, model_name) + with open(latest_fname, 'w') as _fout: + _fout.write(model_path + '\n') + torch.save(checkpoint, model_path) + + if is_best: + best_path = os.path.join(self.save_path, 'model_best.pth.tar') + torch.save({'state_dict': checkpoint['state_dict']}, best_path) + + def load_model(self, model_fname=None): + if self.is_root: + latest_fname = os.path.join(self.save_path, 'latest.txt') + if model_fname is None and os.path.exists(latest_fname): + with open(latest_fname, 'r') as fin: + model_fname = fin.readline() + if model_fname[-1] == '\n': + model_fname = model_fname[:-1] + # noinspection PyBroadException + try: + if model_fname is None or not os.path.exists(model_fname): + model_fname = '%s/checkpoint.pth.tar' % self.save_path + with open(latest_fname, 'w') as fout: + fout.write(model_fname + '\n') + print("=> loading checkpoint '{}'".format(model_fname)) + checkpoint = torch.load(model_fname, map_location='cpu') + except Exception: + self.write_log('fail to load checkpoint from %s' % self.save_path, 'valid') + return + + self.net.load_state_dict(checkpoint['state_dict']) + if 'epoch' in checkpoint: + self.start_epoch = checkpoint['epoch'] + 1 + if 'best_acc' in checkpoint: + self.best_acc = checkpoint['best_acc'] + if 'optimizer' in checkpoint: + self.optimizer.load_state_dict(checkpoint['optimizer']) + + self.write_log("=> loaded checkpoint '{}'".format(model_fname), 'valid') + + # noinspection PyArgumentList + def broadcast(self): + import horovod.torch as hvd + self.start_epoch = hvd.broadcast(torch.LongTensor(1).fill_(self.start_epoch)[0], 0, name='start_epoch').item() + self.best_acc = hvd.broadcast(torch.Tensor(1).fill_(self.best_acc)[0], 0, name='best_acc').item() + hvd.broadcast_parameters(self.net.state_dict(), 0) + hvd.broadcast_optimizer_state(self.optimizer, 0) + + """ metric related """ + + def get_metric_dict(self): + return { + 'top1': DistributedMetric('top1'), + 'top5': DistributedMetric('top5'), + } + + def update_metric(self, metric_dict, output, labels): + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + metric_dict['top1'].update(acc1[0], output.size(0)) + metric_dict['top5'].update(acc5[0], output.size(0)) + + def get_metric_vals(self, metric_dict, return_dict=False): + if return_dict: + return { + key: metric_dict[key].avg.item() for key in metric_dict + } + else: + return [metric_dict[key].avg.item() for key in metric_dict] + + def get_metric_names(self): + return 'top1', 'top5' + + """ train & validate """ + + def validate(self, epoch=0, is_test=False, run_str='', net=None, data_loader=None, no_logs=False): + if net is None: + net = self.net + if data_loader is None: + if is_test: + data_loader = self.run_config.test_loader + else: + data_loader = self.run_config.valid_loader + + net.eval() + + losses = DistributedMetric('val_loss') + metric_dict = self.get_metric_dict() + + with torch.no_grad(): + with tqdm(total=len(data_loader), + desc='Validate Epoch #{} {}'.format(epoch + 1, run_str), + disable=no_logs or not self.is_root) as t: + for i, (images, labels) in enumerate(data_loader): + images, labels = images.cuda(), labels.cuda() + # compute output + output = net(images) + loss = self.test_criterion(output, labels) + # measure accuracy and record loss + losses.update(loss, images.size(0)) + self.update_metric(metric_dict, output, labels) + t.set_postfix({ + 'loss': losses.avg.item(), + **self.get_metric_vals(metric_dict, return_dict=True), + 'img_size': images.size(2), + }) + t.update(1) + return losses.avg.item(), self.get_metric_vals(metric_dict) + + def validate_all_resolution(self, epoch=0, is_test=False, net=None): + if net is None: + net = self.net + if isinstance(self.run_config.data_provider.image_size, list): + img_size_list, loss_list, top1_list, top5_list = [], [], [], [] + for img_size in self.run_config.data_provider.image_size: + img_size_list.append(img_size) + self.run_config.data_provider.assign_active_img_size(img_size) + self.reset_running_statistics(net=net) + loss, (top1, top5) = self.validate(epoch, is_test, net=net) + loss_list.append(loss) + top1_list.append(top1) + top5_list.append(top5) + return img_size_list, loss_list, top1_list, top5_list + else: + loss, (top1, top5) = self.validate(epoch, is_test, net=net) + return [self.run_config.data_provider.active_img_size], [loss], [top1], [top5] + + def train_one_epoch(self, args, epoch, warmup_epochs=5, warmup_lr=0): + self.net.train() + self.run_config.train_loader.sampler.set_epoch(epoch) # required by distributed sampler + MyRandomResizedCrop.EPOCH = epoch # required by elastic resolution + + nBatch = len(self.run_config.train_loader) + + losses = DistributedMetric('train_loss') + metric_dict = self.get_metric_dict() + data_time = AverageMeter() + + with tqdm(total=nBatch, + desc='Train Epoch #{}'.format(epoch + 1), + disable=not self.is_root) as t: + end = time.time() + for i, (images, labels) in enumerate(self.run_config.train_loader): + MyRandomResizedCrop.BATCH = i + data_time.update(time.time() - end) + if epoch < warmup_epochs: + new_lr = self.run_config.warmup_adjust_learning_rate( + self.optimizer, warmup_epochs * nBatch, nBatch, epoch, i, warmup_lr, + ) + else: + new_lr = self.run_config.adjust_learning_rate(self.optimizer, epoch - warmup_epochs, i, nBatch) + + images, labels = images.cuda(), labels.cuda() + target = labels + if isinstance(self.run_config.mixup_alpha, float): + # transform data + random.seed(int('%d%.3d' % (i, epoch))) + lam = random.betavariate(self.run_config.mixup_alpha, self.run_config.mixup_alpha) + images = mix_images(images, lam) + labels = mix_labels( + labels, lam, self.run_config.data_provider.n_classes, self.run_config.label_smoothing + ) + + # soft target + if args.teacher_model is not None: + args.teacher_model.train() + with torch.no_grad(): + soft_logits = args.teacher_model(images).detach() + soft_label = F.softmax(soft_logits, dim=1) + + # compute output + output = self.net(images) + + if args.teacher_model is None: + loss = self.train_criterion(output, labels) + loss_type = 'ce' + else: + if args.kd_type == 'ce': + kd_loss = cross_entropy_loss_with_soft_target(output, soft_label) + else: + kd_loss = F.mse_loss(output, soft_logits) + loss = args.kd_ratio * kd_loss + self.train_criterion(output, labels) + loss_type = '%.1fkd+ce' % args.kd_ratio + + # update + self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + # measure accuracy and record loss + losses.update(loss, images.size(0)) + self.update_metric(metric_dict, output, target) + + t.set_postfix({ + 'loss': losses.avg.item(), + **self.get_metric_vals(metric_dict, return_dict=True), + 'img_size': images.size(2), + 'lr': new_lr, + 'loss_type': loss_type, + 'data_time': data_time.avg, + }) + t.update(1) + end = time.time() + + return losses.avg.item(), self.get_metric_vals(metric_dict) + + def train(self, args, warmup_epochs=5, warmup_lr=0): + for epoch in range(self.start_epoch, self.run_config.n_epochs + warmup_epochs): + train_loss, (train_top1, train_top5) = self.train_one_epoch(args, epoch, warmup_epochs, warmup_lr) + img_size, val_loss, val_top1, val_top5 = self.validate_all_resolution(epoch, is_test=False) + + is_best = list_mean(val_top1) > self.best_acc + self.best_acc = max(self.best_acc, list_mean(val_top1)) + if self.is_root: + val_log = '[{0}/{1}]\tloss {2:.3f}\t{6} acc {3:.3f} ({4:.3f})\t{7} acc {5:.3f}\t' \ + 'Train {6} {top1:.3f}\tloss {train_loss:.3f}\t'. \ + format(epoch + 1 - warmup_epochs, self.run_config.n_epochs, list_mean(val_loss), + list_mean(val_top1), self.best_acc, list_mean(val_top5), *self.get_metric_names(), + top1=train_top1, train_loss=train_loss) + for i_s, v_a in zip(img_size, val_top1): + val_log += '(%d, %.3f), ' % (i_s, v_a) + self.write_log(val_log, prefix='valid', should_print=False) + + self.save_model({ + 'epoch': epoch, + 'best_acc': self.best_acc, + 'optimizer': self.optimizer.state_dict(), + 'state_dict': self.net.state_dict(), + }, is_best=is_best) + + def reset_running_statistics(self, net=None, subset_size=2000, subset_batch_size=200, data_loader=None): + from ofa.imagenet_classification.elastic_nn.utils import set_running_statistics + if net is None: + net = self.net + if data_loader is None: + data_loader = self.run_config.random_sub_train_loader(subset_size, subset_batch_size) + set_running_statistics(net, data_loader) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/run_config.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/run_config.py new file mode 100644 index 0000000..7cb7afb --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/run_config.py @@ -0,0 +1,161 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from ofa_local.utils import calc_learning_rate, build_optimizer +from ofa_local.imagenet_classification.data_providers import ImagenetDataProvider + +__all__ = ['RunConfig', 'ImagenetRunConfig', 'DistributedImageNetRunConfig'] + + +class RunConfig: + + def __init__(self, n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, model_init, validation_frequency, print_frequency): + self.n_epochs = n_epochs + self.init_lr = init_lr + self.lr_schedule_type = lr_schedule_type + self.lr_schedule_param = lr_schedule_param + + self.dataset = dataset + self.train_batch_size = train_batch_size + self.test_batch_size = test_batch_size + self.valid_size = valid_size + + self.opt_type = opt_type + self.opt_param = opt_param + self.weight_decay = weight_decay + self.label_smoothing = label_smoothing + self.no_decay_keys = no_decay_keys + + self.mixup_alpha = mixup_alpha + + self.model_init = model_init + self.validation_frequency = validation_frequency + self.print_frequency = print_frequency + + @property + def config(self): + config = {} + for key in self.__dict__: + if not key.startswith('_'): + config[key] = self.__dict__[key] + return config + + def copy(self): + return RunConfig(**self.config) + + """ learning rate """ + + def adjust_learning_rate(self, optimizer, epoch, batch=0, nBatch=None): + """ adjust learning of a given optimizer and return the new learning rate """ + new_lr = calc_learning_rate(epoch, self.init_lr, self.n_epochs, batch, nBatch, self.lr_schedule_type) + for param_group in optimizer.param_groups: + param_group['lr'] = new_lr + return new_lr + + def warmup_adjust_learning_rate(self, optimizer, T_total, nBatch, epoch, batch=0, warmup_lr=0): + T_cur = epoch * nBatch + batch + 1 + new_lr = T_cur / T_total * (self.init_lr - warmup_lr) + warmup_lr + for param_group in optimizer.param_groups: + param_group['lr'] = new_lr + return new_lr + + """ data provider """ + + @property + def data_provider(self): + raise NotImplementedError + + @property + def train_loader(self): + return self.data_provider.train + + @property + def valid_loader(self): + return self.data_provider.valid + + @property + def test_loader(self): + return self.data_provider.test + + def random_sub_train_loader(self, n_images, batch_size, num_worker=None, num_replicas=None, rank=None): + return self.data_provider.build_sub_train_loader(n_images, batch_size, num_worker, num_replicas, rank) + + """ optimizer """ + + def build_optimizer(self, net_params): + return build_optimizer(net_params, + self.opt_type, self.opt_param, self.init_lr, self.weight_decay, self.no_decay_keys) + + +class ImagenetRunConfig(RunConfig): + + def __init__(self, n_epochs=150, init_lr=0.05, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='imagenet', train_batch_size=256, test_batch_size=500, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.1, no_decay_keys=None, + mixup_alpha=None, model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=32, resize_scale=0.08, distort_color='tf', image_size=224, **kwargs): + super(ImagenetRunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, + model_init, validation_frequency, print_frequency + ) + + self.n_worker = n_worker + self.resize_scale = resize_scale + self.distort_color = distort_color + self.image_size = image_size + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == ImagenetDataProvider.name(): + DataProviderClass = ImagenetDataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + ) + return self.__dict__['_data_provider'] + + +class DistributedImageNetRunConfig(ImagenetRunConfig): + + def __init__(self, n_epochs=150, init_lr=0.05, lr_schedule_type='cosine', lr_schedule_param=None, + dataset='imagenet', train_batch_size=64, test_batch_size=64, valid_size=None, + opt_type='sgd', opt_param=None, weight_decay=4e-5, label_smoothing=0.1, no_decay_keys=None, + mixup_alpha=None, model_init='he_fout', validation_frequency=1, print_frequency=10, + n_worker=8, resize_scale=0.08, distort_color='tf', image_size=224, + **kwargs): + super(DistributedImageNetRunConfig, self).__init__( + n_epochs, init_lr, lr_schedule_type, lr_schedule_param, + dataset, train_batch_size, test_batch_size, valid_size, + opt_type, opt_param, weight_decay, label_smoothing, no_decay_keys, + mixup_alpha, model_init, validation_frequency, print_frequency, n_worker, resize_scale, distort_color, + image_size, **kwargs + ) + + self._num_replicas = kwargs['num_replicas'] + self._rank = kwargs['rank'] + + @property + def data_provider(self): + if self.__dict__.get('_data_provider', None) is None: + if self.dataset == ImagenetDataProvider.name(): + DataProviderClass = ImagenetDataProvider + else: + raise NotImplementedError + self.__dict__['_data_provider'] = DataProviderClass( + train_batch_size=self.train_batch_size, test_batch_size=self.test_batch_size, + valid_size=self.valid_size, n_worker=self.n_worker, resize_scale=self.resize_scale, + distort_color=self.distort_color, image_size=self.image_size, + num_replicas=self._num_replicas, rank=self._rank, + ) + return self.__dict__['_data_provider'] diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/run_manager.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/run_manager.py new file mode 100644 index 0000000..c513a85 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/imagenet_classification/run_manager/run_manager.py @@ -0,0 +1,375 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import os +import random +import time +import json +import numpy as np +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.optim +from tqdm import tqdm + +from ofa_local.utils import get_net_info, cross_entropy_loss_with_soft_target, cross_entropy_with_label_smoothing +from ofa_local.utils import AverageMeter, accuracy, write_log, mix_images, mix_labels, init_models +from ofa_local.utils import MyRandomResizedCrop + +__all__ = ['RunManager'] + + +class RunManager: + + def __init__(self, path, net, run_config, init=True, measure_latency=None, no_gpu=False): + self.path = path + self.net = net + self.run_config = run_config + + self.best_acc = 0 + self.start_epoch = 0 + + os.makedirs(self.path, exist_ok=True) + + # move network to GPU if available + if torch.cuda.is_available() and (not no_gpu): + self.device = torch.device('cuda:0') + self.net = self.net.to(self.device) + cudnn.benchmark = True + else: + self.device = torch.device('cpu') + # initialize model (default) + if init: + init_models(run_config.model_init) + + # net info + net_info = get_net_info(self.net, self.run_config.data_provider.data_shape, measure_latency, True) + with open('%s/net_info.txt' % self.path, 'w') as fout: + fout.write(json.dumps(net_info, indent=4) + '\n') + # noinspection PyBroadException + try: + fout.write(self.network.module_str + '\n') + except Exception: + pass + fout.write('%s\n' % self.run_config.data_provider.train.dataset.transform) + fout.write('%s\n' % self.run_config.data_provider.test.dataset.transform) + fout.write('%s\n' % self.network) + + # criterion + if isinstance(self.run_config.mixup_alpha, float): + self.train_criterion = cross_entropy_loss_with_soft_target + elif self.run_config.label_smoothing > 0: + self.train_criterion = \ + lambda pred, target: cross_entropy_with_label_smoothing(pred, target, self.run_config.label_smoothing) + else: + self.train_criterion = nn.CrossEntropyLoss() + self.test_criterion = nn.CrossEntropyLoss() + + # optimizer + if self.run_config.no_decay_keys: + keys = self.run_config.no_decay_keys.split('#') + net_params = [ + self.network.get_parameters(keys, mode='exclude'), # parameters with weight decay + self.network.get_parameters(keys, mode='include'), # parameters without weight decay + ] + else: + # noinspection PyBroadException + try: + net_params = self.network.weight_parameters() + except Exception: + net_params = [] + for param in self.network.parameters(): + if param.requires_grad: + net_params.append(param) + self.optimizer = self.run_config.build_optimizer(net_params) + + self.net = torch.nn.DataParallel(self.net) + + """ save path and log path """ + + @property + def save_path(self): + if self.__dict__.get('_save_path', None) is None: + save_path = os.path.join(self.path, 'checkpoint') + os.makedirs(save_path, exist_ok=True) + self.__dict__['_save_path'] = save_path + return self.__dict__['_save_path'] + + @property + def logs_path(self): + if self.__dict__.get('_logs_path', None) is None: + logs_path = os.path.join(self.path, 'logs') + os.makedirs(logs_path, exist_ok=True) + self.__dict__['_logs_path'] = logs_path + return self.__dict__['_logs_path'] + + @property + def network(self): + return self.net.module if isinstance(self.net, nn.DataParallel) else self.net + + def write_log(self, log_str, prefix='valid', should_print=True, mode='a'): + write_log(self.logs_path, log_str, prefix, should_print, mode) + + """ save and load models """ + + def save_model(self, checkpoint=None, is_best=False, model_name=None): + if checkpoint is None: + checkpoint = {'state_dict': self.network.state_dict()} + + if model_name is None: + model_name = 'checkpoint.pth.tar' + + checkpoint['dataset'] = self.run_config.dataset # add `dataset` info to the checkpoint + latest_fname = os.path.join(self.save_path, 'latest.txt') + model_path = os.path.join(self.save_path, model_name) + with open(latest_fname, 'w') as fout: + fout.write(model_path + '\n') + torch.save(checkpoint, model_path) + + if is_best: + best_path = os.path.join(self.save_path, 'model_best.pth.tar') + torch.save({'state_dict': checkpoint['state_dict']}, best_path) + + def load_model(self, model_fname=None): + latest_fname = os.path.join(self.save_path, 'latest.txt') + if model_fname is None and os.path.exists(latest_fname): + with open(latest_fname, 'r') as fin: + model_fname = fin.readline() + if model_fname[-1] == '\n': + model_fname = model_fname[:-1] + # noinspection PyBroadException + try: + if model_fname is None or not os.path.exists(model_fname): + model_fname = '%s/checkpoint.pth.tar' % self.save_path + with open(latest_fname, 'w') as fout: + fout.write(model_fname + '\n') + print("=> loading checkpoint '{}'".format(model_fname)) + checkpoint = torch.load(model_fname, map_location='cpu') + except Exception: + print('fail to load checkpoint from %s' % self.save_path) + return {} + + self.network.load_state_dict(checkpoint['state_dict']) + if 'epoch' in checkpoint: + self.start_epoch = checkpoint['epoch'] + 1 + if 'best_acc' in checkpoint: + self.best_acc = checkpoint['best_acc'] + if 'optimizer' in checkpoint: + self.optimizer.load_state_dict(checkpoint['optimizer']) + + print("=> loaded checkpoint '{}'".format(model_fname)) + return checkpoint + + def save_config(self, extra_run_config=None, extra_net_config=None): + """ dump run_config and net_config to the model_folder """ + run_save_path = os.path.join(self.path, 'run.config') + if not os.path.isfile(run_save_path): + run_config = self.run_config.config + if extra_run_config is not None: + run_config.update(extra_run_config) + json.dump(run_config, open(run_save_path, 'w'), indent=4) + print('Run configs dump to %s' % run_save_path) + + try: + net_save_path = os.path.join(self.path, 'net.config') + net_config = self.network.config + if extra_net_config is not None: + net_config.update(extra_net_config) + json.dump(net_config, open(net_save_path, 'w'), indent=4) + print('Network configs dump to %s' % net_save_path) + except Exception: + print('%s do not support net config' % type(self.network)) + + """ metric related """ + + def get_metric_dict(self): + return { + 'top1': AverageMeter(), + 'top5': AverageMeter(), + } + + def update_metric(self, metric_dict, output, labels): + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + metric_dict['top1'].update(acc1[0].item(), output.size(0)) + metric_dict['top5'].update(acc5[0].item(), output.size(0)) + + def get_metric_vals(self, metric_dict, return_dict=False): + if return_dict: + return { + key: metric_dict[key].avg for key in metric_dict + } + else: + return [metric_dict[key].avg for key in metric_dict] + + def get_metric_names(self): + return 'top1', 'top5' + + """ train and test """ + + def validate(self, epoch=0, is_test=False, run_str='', net=None, data_loader=None, no_logs=False, train_mode=False): + if net is None: + net = self.net + if not isinstance(net, nn.DataParallel): + net = nn.DataParallel(net) + + if data_loader is None: + data_loader = self.run_config.test_loader if is_test else self.run_config.valid_loader + + if train_mode: + net.train() + else: + net.eval() + + losses = AverageMeter() + metric_dict = self.get_metric_dict() + + with torch.no_grad(): + with tqdm(total=len(data_loader), + desc='Validate Epoch #{} {}'.format(epoch + 1, run_str), disable=no_logs) as t: + for i, (images, labels) in enumerate(data_loader): + images, labels = images.to(self.device), labels.to(self.device) + # compute output + output = net(images) + loss = self.test_criterion(output, labels) + # measure accuracy and record loss + self.update_metric(metric_dict, output, labels) + + losses.update(loss.item(), images.size(0)) + t.set_postfix({ + 'loss': losses.avg, + **self.get_metric_vals(metric_dict, return_dict=True), + 'img_size': images.size(2), + }) + t.update(1) + return losses.avg, self.get_metric_vals(metric_dict) + + def validate_all_resolution(self, epoch=0, is_test=False, net=None): + if net is None: + net = self.network + if isinstance(self.run_config.data_provider.image_size, list): + img_size_list, loss_list, top1_list, top5_list = [], [], [], [] + for img_size in self.run_config.data_provider.image_size: + img_size_list.append(img_size) + self.run_config.data_provider.assign_active_img_size(img_size) + self.reset_running_statistics(net=net) + loss, (top1, top5) = self.validate(epoch, is_test, net=net) + loss_list.append(loss) + top1_list.append(top1) + top5_list.append(top5) + return img_size_list, loss_list, top1_list, top5_list + else: + loss, (top1, top5) = self.validate(epoch, is_test, net=net) + return [self.run_config.data_provider.active_img_size], [loss], [top1], [top5] + + def train_one_epoch(self, args, epoch, warmup_epochs=0, warmup_lr=0): + # switch to train mode + self.net.train() + MyRandomResizedCrop.EPOCH = epoch # required by elastic resolution + + nBatch = len(self.run_config.train_loader) + + losses = AverageMeter() + metric_dict = self.get_metric_dict() + data_time = AverageMeter() + + with tqdm(total=nBatch, + desc='{} Train Epoch #{}'.format(self.run_config.dataset, epoch + 1)) as t: + end = time.time() + for i, (images, labels) in enumerate(self.run_config.train_loader): + MyRandomResizedCrop.BATCH = i + data_time.update(time.time() - end) + if epoch < warmup_epochs: + new_lr = self.run_config.warmup_adjust_learning_rate( + self.optimizer, warmup_epochs * nBatch, nBatch, epoch, i, warmup_lr, + ) + else: + new_lr = self.run_config.adjust_learning_rate(self.optimizer, epoch - warmup_epochs, i, nBatch) + + images, labels = images.to(self.device), labels.to(self.device) + target = labels + if isinstance(self.run_config.mixup_alpha, float): + # transform data + lam = random.betavariate(self.run_config.mixup_alpha, self.run_config.mixup_alpha) + images = mix_images(images, lam) + labels = mix_labels( + labels, lam, self.run_config.data_provider.n_classes, self.run_config.label_smoothing + ) + + # soft target + if args.teacher_model is not None: + args.teacher_model.train() + with torch.no_grad(): + soft_logits = args.teacher_model(images).detach() + soft_label = F.softmax(soft_logits, dim=1) + + # compute output + output = self.net(images) + loss = self.train_criterion(output, labels) + + if args.teacher_model is None: + loss_type = 'ce' + else: + if args.kd_type == 'ce': + kd_loss = cross_entropy_loss_with_soft_target(output, soft_label) + else: + kd_loss = F.mse_loss(output, soft_logits) + loss = args.kd_ratio * kd_loss + loss + loss_type = '%.1fkd+ce' % args.kd_ratio + + # compute gradient and do SGD step + self.net.zero_grad() # or self.optimizer.zero_grad() + loss.backward() + self.optimizer.step() + + # measure accuracy and record loss + losses.update(loss.item(), images.size(0)) + self.update_metric(metric_dict, output, target) + + t.set_postfix({ + 'loss': losses.avg, + **self.get_metric_vals(metric_dict, return_dict=True), + 'img_size': images.size(2), + 'lr': new_lr, + 'loss_type': loss_type, + 'data_time': data_time.avg, + }) + t.update(1) + end = time.time() + return losses.avg, self.get_metric_vals(metric_dict) + + def train(self, args, warmup_epoch=0, warmup_lr=0): + for epoch in range(self.start_epoch, self.run_config.n_epochs + warmup_epoch): + train_loss, (train_top1, train_top5) = self.train_one_epoch(args, epoch, warmup_epoch, warmup_lr) + + if (epoch + 1) % self.run_config.validation_frequency == 0: + img_size, val_loss, val_acc, val_acc5 = self.validate_all_resolution(epoch=epoch, is_test=False) + + is_best = np.mean(val_acc) > self.best_acc + self.best_acc = max(self.best_acc, np.mean(val_acc)) + val_log = 'Valid [{0}/{1}]\tloss {2:.3f}\t{5} {3:.3f} ({4:.3f})'. \ + format(epoch + 1 - warmup_epoch, self.run_config.n_epochs, + np.mean(val_loss), np.mean(val_acc), self.best_acc, self.get_metric_names()[0]) + val_log += '\t{2} {0:.3f}\tTrain {1} {top1:.3f}\tloss {train_loss:.3f}\t'. \ + format(np.mean(val_acc5), *self.get_metric_names(), top1=train_top1, train_loss=train_loss) + for i_s, v_a in zip(img_size, val_acc): + val_log += '(%d, %.3f), ' % (i_s, v_a) + self.write_log(val_log, prefix='valid', should_print=False) + else: + is_best = False + + self.save_model({ + 'epoch': epoch, + 'best_acc': self.best_acc, + 'optimizer': self.optimizer.state_dict(), + 'state_dict': self.network.state_dict(), + }, is_best=is_best) + + def reset_running_statistics(self, net=None, subset_size=2000, subset_batch_size=200, data_loader=None): + from ofa.imagenet_classification.elastic_nn.utils import set_running_statistics + if net is None: + net = self.network + if data_loader is None: + data_loader = self.run_config.random_sub_train_loader(subset_size, subset_batch_size) + set_running_statistics(net, data_loader) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/model_zoo.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/model_zoo.py new file mode 100644 index 0000000..a70700d --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/model_zoo.py @@ -0,0 +1,87 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import json +import torch + +from ofa_local.utils import download_url +from ofa_local.imagenet_classification.networks import get_net_by_name, proxyless_base +from ofa_local.imagenet_classification.elastic_nn.networks import OFAMobileNetV3, OFAProxylessNASNets, OFAResNets + +__all__ = [ + 'ofa_specialized', 'ofa_net', + 'proxylessnas_net', 'proxylessnas_mobile', 'proxylessnas_cpu', 'proxylessnas_gpu', +] + + +def ofa_specialized(net_id, pretrained=True): + url_base = 'https://hanlab.mit.edu/files/OnceForAll/ofa_specialized/' + net_config = json.load(open( + download_url(url_base + net_id + '/net.config', model_dir='.torch/ofa_specialized/%s/' % net_id) + )) + net = get_net_by_name(net_config['name']).build_from_config(net_config) + + image_size = json.load(open( + download_url(url_base + net_id + '/run.config', model_dir='.torch/ofa_specialized/%s/' % net_id) + ))['image_size'] + + if pretrained: + init = torch.load( + download_url(url_base + net_id + '/init', model_dir='.torch/ofa_specialized/%s/' % net_id), + map_location='cpu' + )['state_dict'] + net.load_state_dict(init) + return net, image_size + + +def ofa_net(net_id, pretrained=True): + if net_id == 'ofa_proxyless_d234_e346_k357_w1.3': + net = OFAProxylessNASNets( + dropout_rate=0, width_mult=1.3, ks_list=[3, 5, 7], expand_ratio_list=[3, 4, 6], depth_list=[2, 3, 4], + ) + elif net_id == 'ofa_mbv3_d234_e346_k357_w1.0': + net = OFAMobileNetV3( + dropout_rate=0, width_mult=1.0, ks_list=[3, 5, 7], expand_ratio_list=[3, 4, 6], depth_list=[2, 3, 4], + ) + elif net_id == 'ofa_mbv3_d234_e346_k357_w1.2': + net = OFAMobileNetV3( + dropout_rate=0, width_mult=1.2, ks_list=[3, 5, 7], expand_ratio_list=[3, 4, 6], depth_list=[2, 3, 4], + ) + elif net_id == 'ofa_resnet50': + net = OFAResNets( + dropout_rate=0, depth_list=[0, 1, 2], expand_ratio_list=[0.2, 0.25, 0.35], width_mult_list=[0.65, 0.8, 1.0] + ) + net_id = 'ofa_resnet50_d=0+1+2_e=0.2+0.25+0.35_w=0.65+0.8+1.0' + else: + raise ValueError('Not supported: %s' % net_id) + + if pretrained: + url_base = 'https://hanlab.mit.edu/files/OnceForAll/ofa_nets/' + init = torch.load( + download_url(url_base + net_id, model_dir='.torch/ofa_nets'), + map_location='cpu')['state_dict'] + net.load_state_dict(init) + return net + + +def proxylessnas_net(net_id, pretrained=True): + net = proxyless_base( + net_config='https://hanlab.mit.edu/files/proxylessNAS/%s.config' % net_id, + ) + if pretrained: + net.load_state_dict(torch.load( + download_url('https://hanlab.mit.edu/files/proxylessNAS/%s.pth' % net_id), map_location='cpu' + )['state_dict']) + + +def proxylessnas_mobile(pretrained=True): + return proxylessnas_net('proxyless_mobile', pretrained) + + +def proxylessnas_cpu(pretrained=True): + return proxylessnas_net('proxyless_cpu', pretrained) + + +def proxylessnas_gpu(pretrained=True): + return proxylessnas_net('proxyless_gpu', pretrained) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/__init__.py new file mode 100644 index 0000000..804fd48 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/__init__.py @@ -0,0 +1,7 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .acc_dataset import * +from .acc_predictor import * +from .arch_encoder import * diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/acc_dataset.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/acc_dataset.py new file mode 100644 index 0000000..c6e7ea0 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/acc_dataset.py @@ -0,0 +1,181 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import os +import json +import numpy as np +from tqdm import tqdm +import torch +import torch.utils.data + +from ofa.utils import list_mean + +__all__ = ['net_setting2id', 'net_id2setting', 'AccuracyDataset'] + + +def net_setting2id(net_setting): + return json.dumps(net_setting) + + +def net_id2setting(net_id): + return json.loads(net_id) + + +class RegDataset(torch.utils.data.Dataset): + + def __init__(self, inputs, targets): + super(RegDataset, self).__init__() + self.inputs = inputs + self.targets = targets + + def __getitem__(self, index): + return self.inputs[index], self.targets[index] + + def __len__(self): + return self.inputs.size(0) + + +class AccuracyDataset: + + def __init__(self, path): + self.path = path + os.makedirs(self.path, exist_ok=True) + + @property + def net_id_path(self): + return os.path.join(self.path, 'net_id.dict') + + @property + def acc_src_folder(self): + return os.path.join(self.path, 'src') + + @property + def acc_dict_path(self): + return os.path.join(self.path, 'acc.dict') + + # TODO: support parallel building + def build_acc_dataset(self, run_manager, ofa_network, n_arch=1000, image_size_list=None): + # load net_id_list, random sample if not exist + if os.path.isfile(self.net_id_path): + net_id_list = json.load(open(self.net_id_path)) + else: + net_id_list = set() + while len(net_id_list) < n_arch: + net_setting = ofa_network.sample_active_subnet() + net_id = net_setting2id(net_setting) + net_id_list.add(net_id) + net_id_list = list(net_id_list) + net_id_list.sort() + json.dump(net_id_list, open(self.net_id_path, 'w'), indent=4) + + image_size_list = [128, 160, 192, 224] if image_size_list is None else image_size_list + + with tqdm(total=len(net_id_list) * len(image_size_list), desc='Building Acc Dataset') as t: + for image_size in image_size_list: + # load val dataset into memory + val_dataset = [] + run_manager.run_config.data_provider.assign_active_img_size(image_size) + for images, labels in run_manager.run_config.valid_loader: + val_dataset.append((images, labels)) + # save path + os.makedirs(self.acc_src_folder, exist_ok=True) + acc_save_path = os.path.join(self.acc_src_folder, '%d.dict' % image_size) + acc_dict = {} + # load existing acc dict + if os.path.isfile(acc_save_path): + existing_acc_dict = json.load(open(acc_save_path, 'r')) + else: + existing_acc_dict = {} + for net_id in net_id_list: + net_setting = net_id2setting(net_id) + key = net_setting2id({**net_setting, 'image_size': image_size}) + if key in existing_acc_dict: + acc_dict[key] = existing_acc_dict[key] + t.set_postfix({ + 'net_id': net_id, + 'image_size': image_size, + 'info_val': acc_dict[key], + 'status': 'loading', + }) + t.update() + continue + ofa_network.set_active_subnet(**net_setting) + run_manager.reset_running_statistics(ofa_network) + net_setting_str = ','.join(['%s_%s' % ( + key, '%.1f' % list_mean(val) if isinstance(val, list) else val + ) for key, val in net_setting.items()]) + loss, (top1, top5) = run_manager.validate( + run_str=net_setting_str, net=ofa_network, data_loader=val_dataset, no_logs=True, + ) + info_val = top1 + + t.set_postfix({ + 'net_id': net_id, + 'image_size': image_size, + 'info_val': info_val, + }) + t.update() + + acc_dict.update({ + key: info_val + }) + json.dump(acc_dict, open(acc_save_path, 'w'), indent=4) + + def merge_acc_dataset(self, image_size_list=None): + # load existing data + merged_acc_dict = {} + for fname in os.listdir(self.acc_src_folder): + if '.dict' not in fname: + continue + image_size = int(fname.split('.dict')[0]) + if image_size_list is not None and image_size not in image_size_list: + print('Skip ', fname) + continue + full_path = os.path.join(self.acc_src_folder, fname) + partial_acc_dict = json.load(open(full_path)) + merged_acc_dict.update(partial_acc_dict) + print('loaded %s' % full_path) + json.dump(merged_acc_dict, open(self.acc_dict_path, 'w'), indent=4) + return merged_acc_dict + + def build_acc_data_loader(self, arch_encoder, n_training_sample=None, batch_size=256, n_workers=16): + # load data + acc_dict = json.load(open(self.acc_dict_path)) + X_all = [] + Y_all = [] + with tqdm(total=len(acc_dict), desc='Loading data') as t: + for k, v in acc_dict.items(): + dic = json.loads(k) + X_all.append(arch_encoder.arch2feature(dic)) + Y_all.append(v / 100.) # range: 0 - 1 + t.update() + base_acc = np.mean(Y_all) + # convert to torch tensor + X_all = torch.tensor(X_all, dtype=torch.float) + Y_all = torch.tensor(Y_all) + + # random shuffle + shuffle_idx = torch.randperm(len(X_all)) + X_all = X_all[shuffle_idx] + Y_all = Y_all[shuffle_idx] + + # split data + idx = X_all.size(0) // 5 * 4 if n_training_sample is None else n_training_sample + val_idx = X_all.size(0) // 5 * 4 + X_train, Y_train = X_all[:idx], Y_all[:idx] + X_test, Y_test = X_all[val_idx:], Y_all[val_idx:] + print('Train Size: %d,' % len(X_train), 'Valid Size: %d' % len(X_test)) + + # build data loader + train_dataset = RegDataset(X_train, Y_train) + val_dataset = RegDataset(X_test, Y_test) + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=batch_size, shuffle=True, pin_memory=False, num_workers=n_workers + ) + valid_loader = torch.utils.data.DataLoader( + val_dataset, batch_size=batch_size, shuffle=False, pin_memory=False, num_workers=n_workers + ) + + return train_loader, valid_loader, base_acc diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/acc_predictor.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/acc_predictor.py new file mode 100644 index 0000000..faf5136 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/acc_predictor.py @@ -0,0 +1,50 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import os +import numpy as np +import torch +import torch.nn as nn + +__all__ = ['AccuracyPredictor'] + + +class AccuracyPredictor(nn.Module): + + def __init__(self, arch_encoder, hidden_size=400, n_layers=3, + checkpoint_path=None, device='cuda:0'): + super(AccuracyPredictor, self).__init__() + self.arch_encoder = arch_encoder + self.hidden_size = hidden_size + self.n_layers = n_layers + self.device = device + + # build layers + layers = [] + for i in range(self.n_layers): + layers.append(nn.Sequential( + nn.Linear(self.arch_encoder.n_dim if i == 0 else self.hidden_size, self.hidden_size), + nn.ReLU(inplace=True), + )) + layers.append(nn.Linear(self.hidden_size, 1, bias=False)) + self.layers = nn.Sequential(*layers) + self.base_acc = nn.Parameter(torch.zeros(1, device=self.device), requires_grad=False) + + if checkpoint_path is not None and os.path.exists(checkpoint_path): + checkpoint = torch.load(checkpoint_path, map_location='cpu') + if 'state_dict' in checkpoint: + checkpoint = checkpoint['state_dict'] + self.load_state_dict(checkpoint) + print('Loaded checkpoint from %s' % checkpoint_path) + + self.layers = self.layers.to(self.device) + + def forward(self, x): + y = self.layers(x).squeeze() + return y + self.base_acc + + def predict_acc(self, arch_dict_list): + X = [self.arch_encoder.arch2feature(arch_dict) for arch_dict in arch_dict_list] + X = torch.tensor(np.array(X)).float().to(self.device) + return self.forward(X) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/arch_encoder.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/arch_encoder.py new file mode 100644 index 0000000..88726b4 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/accuracy_predictor/arch_encoder.py @@ -0,0 +1,315 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + + +import random +import numpy as np +from ofa.imagenet_classification.networks import ResNets + +__all__ = ['MobileNetArchEncoder', 'ResNetArchEncoder'] + + +class MobileNetArchEncoder: + SPACE_TYPE = 'mbv3' + + def __init__(self, image_size_list=None, ks_list=None, expand_list=None, depth_list=None, n_stage=None): + self.image_size_list = [224] if image_size_list is None else image_size_list + self.ks_list = [3, 5, 7] if ks_list is None else ks_list + self.expand_list = [3, 4, 6] if expand_list is None else [int(expand) for expand in expand_list] + self.depth_list = [2, 3, 4] if depth_list is None else depth_list + if n_stage is not None: + self.n_stage = n_stage + elif self.SPACE_TYPE == 'mbv2': + self.n_stage = 6 + elif self.SPACE_TYPE == 'mbv3': + self.n_stage = 5 + else: + raise NotImplementedError + + # build info dict + self.n_dim = 0 + self.r_info = dict(id2val={}, val2id={}, L=[], R=[]) + self._build_info_dict(target='r') + + self.k_info = dict(id2val=[], val2id=[], L=[], R=[]) + self.e_info = dict(id2val=[], val2id=[], L=[], R=[]) + self._build_info_dict(target='k') + self._build_info_dict(target='e') + + @property + def max_n_blocks(self): + if self.SPACE_TYPE == 'mbv3': + return self.n_stage * max(self.depth_list) + elif self.SPACE_TYPE == 'mbv2': + return (self.n_stage - 1) * max(self.depth_list) + 1 + else: + raise NotImplementedError + + def _build_info_dict(self, target): + if target == 'r': + target_dict = self.r_info + target_dict['L'].append(self.n_dim) + for img_size in self.image_size_list: + target_dict['val2id'][img_size] = self.n_dim + target_dict['id2val'][self.n_dim] = img_size + self.n_dim += 1 + target_dict['R'].append(self.n_dim) + else: + if target == 'k': + target_dict = self.k_info + choices = self.ks_list + elif target == 'e': + target_dict = self.e_info + choices = self.expand_list + else: + raise NotImplementedError + for i in range(self.max_n_blocks): + target_dict['val2id'].append({}) + target_dict['id2val'].append({}) + target_dict['L'].append(self.n_dim) + for k in choices: + target_dict['val2id'][i][k] = self.n_dim + target_dict['id2val'][i][self.n_dim] = k + self.n_dim += 1 + target_dict['R'].append(self.n_dim) + + def arch2feature(self, arch_dict): + ks, e, d, r = arch_dict['ks'], arch_dict['e'], arch_dict['d'], arch_dict['image_size'] + + feature = np.zeros(self.n_dim) + for i in range(self.max_n_blocks): + nowd = i % max(self.depth_list) + stg = i // max(self.depth_list) + if nowd < d[stg]: + feature[self.k_info['val2id'][i][ks[i]]] = 1 + feature[self.e_info['val2id'][i][e[i]]] = 1 + feature[self.r_info['val2id'][r]] = 1 + return feature + + def feature2arch(self, feature): + img_sz = self.r_info['id2val'][ + int(np.argmax(feature[self.r_info['L'][0]:self.r_info['R'][0]])) + self.r_info['L'][0] + ] + assert img_sz in self.image_size_list + arch_dict = {'ks': [], 'e': [], 'd': [], 'image_size': img_sz} + + d = 0 + for i in range(self.max_n_blocks): + skip = True + for j in range(self.k_info['L'][i], self.k_info['R'][i]): + if feature[j] == 1: + arch_dict['ks'].append(self.k_info['id2val'][i][j]) + skip = False + break + + for j in range(self.e_info['L'][i], self.e_info['R'][i]): + if feature[j] == 1: + arch_dict['e'].append(self.e_info['id2val'][i][j]) + assert not skip + skip = False + break + + if skip: + arch_dict['e'].append(0) + arch_dict['ks'].append(0) + else: + d += 1 + + if (i + 1) % max(self.depth_list) == 0 or (i + 1) == self.max_n_blocks: + arch_dict['d'].append(d) + d = 0 + return arch_dict + + def random_sample_arch(self): + return { + 'ks': random.choices(self.ks_list, k=self.max_n_blocks), + 'e': random.choices(self.expand_list, k=self.max_n_blocks), + 'd': random.choices(self.depth_list, k=self.n_stage), + 'image_size': random.choice(self.image_size_list) + } + + def mutate_resolution(self, arch_dict, mutate_prob): + if random.random() < mutate_prob: + arch_dict['image_size'] = random.choice(self.image_size_list) + return arch_dict + + def mutate_arch(self, arch_dict, mutate_prob): + for i in range(self.max_n_blocks): + if random.random() < mutate_prob: + arch_dict['ks'][i] = random.choice(self.ks_list) + arch_dict['e'][i] = random.choice(self.expand_list) + + for i in range(self.n_stage): + if random.random() < mutate_prob: + arch_dict['d'][i] = random.choice(self.depth_list) + return arch_dict + + +class ResNetArchEncoder: + + def __init__(self, image_size_list=None, depth_list=None, expand_list=None, width_mult_list=None, + base_depth_list=None): + self.image_size_list = [224] if image_size_list is None else image_size_list + self.expand_list = [0.2, 0.25, 0.35] if expand_list is None else expand_list + self.depth_list = [0, 1, 2] if depth_list is None else depth_list + self.width_mult_list = [0.65, 0.8, 1.0] if width_mult_list is None else width_mult_list + + self.base_depth_list = ResNets.BASE_DEPTH_LIST if base_depth_list is None else base_depth_list + + """" build info dict """ + self.n_dim = 0 + # resolution + self.r_info = dict(id2val={}, val2id={}, L=[], R=[]) + self._build_info_dict(target='r') + # input stem skip + self.input_stem_d_info = dict(id2val={}, val2id={}, L=[], R=[]) + self._build_info_dict(target='input_stem_d') + # width_mult + self.width_mult_info = dict(id2val=[], val2id=[], L=[], R=[]) + self._build_info_dict(target='width_mult') + # expand ratio + self.e_info = dict(id2val=[], val2id=[], L=[], R=[]) + self._build_info_dict(target='e') + + @property + def n_stage(self): + return len(self.base_depth_list) + + @property + def max_n_blocks(self): + return sum(self.base_depth_list) + self.n_stage * max(self.depth_list) + + def _build_info_dict(self, target): + if target == 'r': + target_dict = self.r_info + target_dict['L'].append(self.n_dim) + for img_size in self.image_size_list: + target_dict['val2id'][img_size] = self.n_dim + target_dict['id2val'][self.n_dim] = img_size + self.n_dim += 1 + target_dict['R'].append(self.n_dim) + elif target == 'input_stem_d': + target_dict = self.input_stem_d_info + target_dict['L'].append(self.n_dim) + for skip in [0, 1]: + target_dict['val2id'][skip] = self.n_dim + target_dict['id2val'][self.n_dim] = skip + self.n_dim += 1 + target_dict['R'].append(self.n_dim) + elif target == 'e': + target_dict = self.e_info + choices = self.expand_list + for i in range(self.max_n_blocks): + target_dict['val2id'].append({}) + target_dict['id2val'].append({}) + target_dict['L'].append(self.n_dim) + for e in choices: + target_dict['val2id'][i][e] = self.n_dim + target_dict['id2val'][i][self.n_dim] = e + self.n_dim += 1 + target_dict['R'].append(self.n_dim) + elif target == 'width_mult': + target_dict = self.width_mult_info + choices = list(range(len(self.width_mult_list))) + for i in range(self.n_stage + 2): + target_dict['val2id'].append({}) + target_dict['id2val'].append({}) + target_dict['L'].append(self.n_dim) + for w in choices: + target_dict['val2id'][i][w] = self.n_dim + target_dict['id2val'][i][self.n_dim] = w + self.n_dim += 1 + target_dict['R'].append(self.n_dim) + + def arch2feature(self, arch_dict): + d, e, w, r = arch_dict['d'], arch_dict['e'], arch_dict['w'], arch_dict['image_size'] + input_stem_skip = 1 if d[0] > 0 else 0 + d = d[1:] + + feature = np.zeros(self.n_dim) + feature[self.r_info['val2id'][r]] = 1 + feature[self.input_stem_d_info['val2id'][input_stem_skip]] = 1 + for i in range(self.n_stage + 2): + feature[self.width_mult_info['val2id'][i][w[i]]] = 1 + + start_pt = 0 + for i, base_depth in enumerate(self.base_depth_list): + depth = base_depth + d[i] + for j in range(start_pt, start_pt + depth): + feature[self.e_info['val2id'][j][e[j]]] = 1 + start_pt += max(self.depth_list) + base_depth + + return feature + + def feature2arch(self, feature): + img_sz = self.r_info['id2val'][ + int(np.argmax(feature[self.r_info['L'][0]:self.r_info['R'][0]])) + self.r_info['L'][0] + ] + input_stem_skip = self.input_stem_d_info['id2val'][ + int(np.argmax(feature[self.input_stem_d_info['L'][0]:self.input_stem_d_info['R'][0]])) + + self.input_stem_d_info['L'][0] + ] * 2 + assert img_sz in self.image_size_list + arch_dict = {'d': [input_stem_skip], 'e': [], 'w': [], 'image_size': img_sz} + + for i in range(self.n_stage + 2): + arch_dict['w'].append( + self.width_mult_info['id2val'][i][ + int(np.argmax(feature[self.width_mult_info['L'][i]:self.width_mult_info['R'][i]])) + + self.width_mult_info['L'][i] + ] + ) + + d = 0 + skipped = 0 + stage_id = 0 + for i in range(self.max_n_blocks): + skip = True + for j in range(self.e_info['L'][i], self.e_info['R'][i]): + if feature[j] == 1: + arch_dict['e'].append(self.e_info['id2val'][i][j]) + skip = False + break + if skip: + arch_dict['e'].append(0) + skipped += 1 + else: + d += 1 + + if i + 1 == self.max_n_blocks or (skipped + d) % \ + (max(self.depth_list) + self.base_depth_list[stage_id]) == 0: + arch_dict['d'].append(d - self.base_depth_list[stage_id]) + d, skipped = 0, 0 + stage_id += 1 + return arch_dict + + def random_sample_arch(self): + return { + 'd': [random.choice([0, 2])] + random.choices(self.depth_list, k=self.n_stage), + 'e': random.choices(self.expand_list, k=self.max_n_blocks), + 'w': random.choices(list(range(len(self.width_mult_list))), k=self.n_stage + 2), + 'image_size': random.choice(self.image_size_list) + } + + def mutate_resolution(self, arch_dict, mutate_prob): + if random.random() < mutate_prob: + arch_dict['image_size'] = random.choice(self.image_size_list) + return arch_dict + + def mutate_arch(self, arch_dict, mutate_prob): + # input stem skip + if random.random() < mutate_prob: + arch_dict['d'][0] = random.choice([0, 2]) + # depth + for i in range(1, len(arch_dict['d'])): + if random.random() < mutate_prob: + arch_dict['d'][i] = random.choice(self.depth_list) + # width_mult + for i in range(len(arch_dict['w'])): + if random.random() < mutate_prob: + arch_dict['w'][i] = random.choice(list(range(len(self.width_mult_list)))) + # expand ratio + for i in range(len(arch_dict['e'])): + if random.random() < mutate_prob: + arch_dict['e'][i] = random.choice(self.expand_list) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/efficiency_predictor/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/efficiency_predictor/__init__.py new file mode 100644 index 0000000..804cfd2 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/efficiency_predictor/__init__.py @@ -0,0 +1,71 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import os +import copy +from .latency_lookup_table import * + + +class BaseEfficiencyModel: + + def __init__(self, ofa_net): + self.ofa_net = ofa_net + + def get_active_subnet_config(self, arch_dict): + arch_dict = copy.deepcopy(arch_dict) + image_size = arch_dict.pop('image_size') + self.ofa_net.set_active_subnet(**arch_dict) + active_net_config = self.ofa_net.get_active_net_config() + return active_net_config, image_size + + def get_efficiency(self, arch_dict): + raise NotImplementedError + + +class ProxylessNASFLOPsModel(BaseEfficiencyModel): + + def get_efficiency(self, arch_dict): + active_net_config, image_size = self.get_active_subnet_config(arch_dict) + return ProxylessNASLatencyTable.count_flops_given_config(active_net_config, image_size) + + +class Mbv3FLOPsModel(BaseEfficiencyModel): + + def get_efficiency(self, arch_dict): + active_net_config, image_size = self.get_active_subnet_config(arch_dict) + return MBv3LatencyTable.count_flops_given_config(active_net_config, image_size) + + +class ResNet50FLOPsModel(BaseEfficiencyModel): + + def get_efficiency(self, arch_dict): + active_net_config, image_size = self.get_active_subnet_config(arch_dict) + return ResNet50LatencyTable.count_flops_given_config(active_net_config, image_size) + +class ProxylessNASLatencyModel(BaseEfficiencyModel): + + def __init__(self, ofa_net, lookup_table_path_dict): + super(ProxylessNASLatencyModel, self).__init__(ofa_net) + self.latency_tables = {} + for image_size, path in lookup_table_path_dict.items(): + self.latency_tables[image_size] = ProxylessNASLatencyTable( + local_dir='/tmp/.ofa_latency_tools/', url=os.path.join(path, '%d_lookup_table.yaml' % image_size)) + + def get_efficiency(self, arch_dict): + active_net_config, image_size = self.get_active_subnet_config(arch_dict) + return self.latency_tables[image_size].predict_network_latency_given_config(active_net_config, image_size) + + +class Mbv3LatencyModel(BaseEfficiencyModel): + + def __init__(self, ofa_net, lookup_table_path_dict): + super(Mbv3LatencyModel, self).__init__(ofa_net) + self.latency_tables = {} + for image_size, path in lookup_table_path_dict.items(): + self.latency_tables[image_size] = MBv3LatencyTable( + local_dir='/tmp/.ofa_latency_tools/', url=os.path.join(path, '%d_lookup_table.yaml' % image_size)) + + def get_efficiency(self, arch_dict): + active_net_config, image_size = self.get_active_subnet_config(arch_dict) + return self.latency_tables[image_size].predict_network_latency_given_config(active_net_config, image_size) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/efficiency_predictor/latency_lookup_table.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/efficiency_predictor/latency_lookup_table.py new file mode 100644 index 0000000..80681da --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/efficiency_predictor/latency_lookup_table.py @@ -0,0 +1,387 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import yaml +from ofa.utils import download_url, make_divisible, MyNetwork + +__all__ = ['count_conv_flop', 'ProxylessNASLatencyTable', 'MBv3LatencyTable', 'ResNet50LatencyTable'] + + +def count_conv_flop(out_size, in_channels, out_channels, kernel_size, groups): + out_h = out_w = out_size + delta_ops = in_channels * out_channels * kernel_size * kernel_size * out_h * out_w / groups + return delta_ops + + +class LatencyTable(object): + + def __init__(self, local_dir='~/.ofa/latency_tools/', + url='https://hanlab.mit.edu/files/proxylessNAS/LatencyTools/mobile_trim.yaml'): + if url.startswith('http'): + fname = download_url(url, local_dir, overwrite=True) + else: + fname = url + with open(fname, 'r') as fp: + self.lut = yaml.load(fp) + + @staticmethod + def repr_shape(shape): + if isinstance(shape, (list, tuple)): + return 'x'.join(str(_) for _ in shape) + elif isinstance(shape, str): + return shape + else: + return TypeError + + def query(self, **kwargs): + raise NotImplementedError + + def predict_network_latency(self, net, image_size): + raise NotImplementedError + + def predict_network_latency_given_config(self, net_config, image_size): + raise NotImplementedError + + @staticmethod + def count_flops_given_config(net_config, image_size=224): + raise NotImplementedError + + +class ProxylessNASLatencyTable(LatencyTable): + + def query(self, l_type: str, input_shape, output_shape, expand=None, ks=None, stride=None, id_skip=None): + """ + :param l_type: + Layer type must be one of the followings + 1. `Conv`: The initial 3x3 conv with stride 2. + 2. `Conv_1`: feature_mix_layer + 3. `Logits`: All operations after `Conv_1`. + 4. `expanded_conv`: MobileInvertedResidual + :param input_shape: input shape (h, w, #channels) + :param output_shape: output shape (h, w, #channels) + :param expand: expansion ratio + :param ks: kernel size + :param stride: + :param id_skip: indicate whether has the residual connection + """ + infos = [l_type, 'input:%s' % self.repr_shape(input_shape), 'output:%s' % self.repr_shape(output_shape), ] + + if l_type in ('expanded_conv',): + assert None not in (expand, ks, stride, id_skip) + infos += ['expand:%d' % expand, 'kernel:%d' % ks, 'stride:%d' % stride, 'idskip:%d' % id_skip] + key = '-'.join(infos) + return self.lut[key]['mean'] + + def predict_network_latency(self, net, image_size=224): + predicted_latency = 0 + # first conv + predicted_latency += self.query( + 'Conv', [image_size, image_size, 3], + [(image_size + 1) // 2, (image_size + 1) // 2, net.first_conv.out_channels] + ) + # blocks + fsize = (image_size + 1) // 2 + for block in net.blocks: + mb_conv = block.conv + shortcut = block.shortcut + + if mb_conv is None: + continue + if shortcut is None: + idskip = 0 + else: + idskip = 1 + out_fz = int((fsize - 1) / mb_conv.stride + 1) # fsize // mb_conv.stride + block_latency = self.query( + 'expanded_conv', [fsize, fsize, mb_conv.in_channels], [out_fz, out_fz, mb_conv.out_channels], + expand=mb_conv.expand_ratio, ks=mb_conv.kernel_size, stride=mb_conv.stride, id_skip=idskip + ) + predicted_latency += block_latency + fsize = out_fz + # feature mix layer + predicted_latency += self.query( + 'Conv_1', [fsize, fsize, net.feature_mix_layer.in_channels], + [fsize, fsize, net.feature_mix_layer.out_channels] + ) + # classifier + predicted_latency += self.query( + 'Logits', [fsize, fsize, net.classifier.in_features], [net.classifier.out_features] # 1000 + ) + return predicted_latency + + def predict_network_latency_given_config(self, net_config, image_size=224): + predicted_latency = 0 + # first conv + predicted_latency += self.query( + 'Conv', [image_size, image_size, 3], + [(image_size + 1) // 2, (image_size + 1) // 2, net_config['first_conv']['out_channels']] + ) + # blocks + fsize = (image_size + 1) // 2 + for block in net_config['blocks']: + mb_conv = block['mobile_inverted_conv'] if 'mobile_inverted_conv' in block else block['conv'] + shortcut = block['shortcut'] + + if mb_conv is None: + continue + if shortcut is None: + idskip = 0 + else: + idskip = 1 + out_fz = int((fsize - 1) / mb_conv['stride'] + 1) + block_latency = self.query( + 'expanded_conv', [fsize, fsize, mb_conv['in_channels']], [out_fz, out_fz, mb_conv['out_channels']], + expand=mb_conv['expand_ratio'], ks=mb_conv['kernel_size'], stride=mb_conv['stride'], id_skip=idskip + ) + predicted_latency += block_latency + fsize = out_fz + # feature mix layer + predicted_latency += self.query( + 'Conv_1', [fsize, fsize, net_config['feature_mix_layer']['in_channels']], + [fsize, fsize, net_config['feature_mix_layer']['out_channels']] + ) + # classifier + predicted_latency += self.query( + 'Logits', [fsize, fsize, net_config['classifier']['in_features']], + [net_config['classifier']['out_features']] # 1000 + ) + return predicted_latency + + @staticmethod + def count_flops_given_config(net_config, image_size=224): + flops = 0 + # first conv + flops += count_conv_flop((image_size + 1) // 2, 3, net_config['first_conv']['out_channels'], 3, 1) + # blocks + fsize = (image_size + 1) // 2 + for block in net_config['blocks']: + mb_conv = block['mobile_inverted_conv'] if 'mobile_inverted_conv' in block else block['conv'] + if mb_conv is None: + continue + out_fz = int((fsize - 1) / mb_conv['stride'] + 1) + if mb_conv['mid_channels'] is None: + mb_conv['mid_channels'] = round(mb_conv['in_channels'] * mb_conv['expand_ratio']) + if mb_conv['expand_ratio'] != 1: + # inverted bottleneck + flops += count_conv_flop(fsize, mb_conv['in_channels'], mb_conv['mid_channels'], 1, 1) + # depth conv + flops += count_conv_flop(out_fz, mb_conv['mid_channels'], mb_conv['mid_channels'], + mb_conv['kernel_size'], mb_conv['mid_channels']) + # point linear + flops += count_conv_flop(out_fz, mb_conv['mid_channels'], mb_conv['out_channels'], 1, 1) + fsize = out_fz + # feature mix layer + flops += count_conv_flop(fsize, net_config['feature_mix_layer']['in_channels'], + net_config['feature_mix_layer']['out_channels'], 1, 1) + # classifier + flops += count_conv_flop(1, net_config['classifier']['in_features'], + net_config['classifier']['out_features'], 1, 1) + return flops / 1e6 # MFLOPs + + +class MBv3LatencyTable(LatencyTable): + + def query(self, l_type: str, input_shape, output_shape, mid=None, ks=None, stride=None, id_skip=None, + se=None, h_swish=None): + infos = [l_type, 'input:%s' % self.repr_shape(input_shape), 'output:%s' % self.repr_shape(output_shape), ] + + if l_type in ('expanded_conv',): + assert None not in (mid, ks, stride, id_skip, se, h_swish) + infos += ['expand:%d' % mid, 'kernel:%d' % ks, 'stride:%d' % stride, 'idskip:%d' % id_skip, + 'se:%d' % se, 'hs:%d' % h_swish] + key = '-'.join(infos) + return self.lut[key]['mean'] + + def predict_network_latency(self, net, image_size=224): + predicted_latency = 0 + # first conv + predicted_latency += self.query( + 'Conv', [image_size, image_size, 3], + [(image_size + 1) // 2, (image_size + 1) // 2, net.first_conv.out_channels] + ) + # blocks + fsize = (image_size + 1) // 2 + for block in net.blocks: + mb_conv = block.conv + shortcut = block.shortcut + + if mb_conv is None: + continue + if shortcut is None: + idskip = 0 + else: + idskip = 1 + out_fz = int((fsize - 1) / mb_conv.stride + 1) + block_latency = self.query( + 'expanded_conv', [fsize, fsize, mb_conv.in_channels], [out_fz, out_fz, mb_conv.out_channels], + mid=mb_conv.depth_conv.conv.in_channels, ks=mb_conv.kernel_size, stride=mb_conv.stride, id_skip=idskip, + se=1 if mb_conv.use_se else 0, h_swish=1 if mb_conv.act_func == 'h_swish' else 0, + ) + predicted_latency += block_latency + fsize = out_fz + # final expand layer + predicted_latency += self.query( + 'Conv_1', [fsize, fsize, net.final_expand_layer.in_channels], + [fsize, fsize, net.final_expand_layer.out_channels], + ) + # global average pooling + predicted_latency += self.query( + 'AvgPool2D', [fsize, fsize, net.final_expand_layer.out_channels], + [1, 1, net.final_expand_layer.out_channels], + ) + # feature mix layer + predicted_latency += self.query( + 'Conv_2', [1, 1, net.feature_mix_layer.in_channels], + [1, 1, net.feature_mix_layer.out_channels] + ) + # classifier + predicted_latency += self.query( + 'Logits', [1, 1, net.classifier.in_features], [net.classifier.out_features] + ) + return predicted_latency + + def predict_network_latency_given_config(self, net_config, image_size=224): + predicted_latency = 0 + # first conv + predicted_latency += self.query( + 'Conv', [image_size, image_size, 3], + [(image_size + 1) // 2, (image_size + 1) // 2, net_config['first_conv']['out_channels']] + ) + # blocks + fsize = (image_size + 1) // 2 + for block in net_config['blocks']: + mb_conv = block['mobile_inverted_conv'] if 'mobile_inverted_conv' in block else block['conv'] + shortcut = block['shortcut'] + + if mb_conv is None: + continue + if shortcut is None: + idskip = 0 + else: + idskip = 1 + out_fz = int((fsize - 1) / mb_conv['stride'] + 1) + if mb_conv['mid_channels'] is None: + mb_conv['mid_channels'] = round(mb_conv['in_channels'] * mb_conv['expand_ratio']) + block_latency = self.query( + 'expanded_conv', [fsize, fsize, mb_conv['in_channels']], [out_fz, out_fz, mb_conv['out_channels']], + mid=mb_conv['mid_channels'], ks=mb_conv['kernel_size'], stride=mb_conv['stride'], id_skip=idskip, + se=1 if mb_conv['use_se'] else 0, h_swish=1 if mb_conv['act_func'] == 'h_swish' else 0, + ) + predicted_latency += block_latency + fsize = out_fz + # final expand layer + predicted_latency += self.query( + 'Conv_1', [fsize, fsize, net_config['final_expand_layer']['in_channels']], + [fsize, fsize, net_config['final_expand_layer']['out_channels']], + ) + # global average pooling + predicted_latency += self.query( + 'AvgPool2D', [fsize, fsize, net_config['final_expand_layer']['out_channels']], + [1, 1, net_config['final_expand_layer']['out_channels']], + ) + # feature mix layer + predicted_latency += self.query( + 'Conv_2', [1, 1, net_config['feature_mix_layer']['in_channels']], + [1, 1, net_config['feature_mix_layer']['out_channels']] + ) + # classifier + predicted_latency += self.query( + 'Logits', [1, 1, net_config['classifier']['in_features']], [net_config['classifier']['out_features']] + ) + return predicted_latency + + @staticmethod + def count_flops_given_config(net_config, image_size=224): + flops = 0 + # first conv + flops += count_conv_flop((image_size + 1) // 2, 3, net_config['first_conv']['out_channels'], 3, 1) + # blocks + fsize = (image_size + 1) // 2 + for block in net_config['blocks']: + mb_conv = block['mobile_inverted_conv'] if 'mobile_inverted_conv' in block else block['conv'] + if mb_conv is None: + continue + out_fz = int((fsize - 1) / mb_conv['stride'] + 1) + if mb_conv['mid_channels'] is None: + mb_conv['mid_channels'] = round(mb_conv['in_channels'] * mb_conv['expand_ratio']) + if mb_conv['expand_ratio'] != 1: + # inverted bottleneck + flops += count_conv_flop(fsize, mb_conv['in_channels'], mb_conv['mid_channels'], 1, 1) + # depth conv + flops += count_conv_flop(out_fz, mb_conv['mid_channels'], mb_conv['mid_channels'], + mb_conv['kernel_size'], mb_conv['mid_channels']) + if mb_conv['use_se']: + # SE layer + se_mid = make_divisible(mb_conv['mid_channels'] // 4, divisor=MyNetwork.CHANNEL_DIVISIBLE) + flops += count_conv_flop(1, mb_conv['mid_channels'], se_mid, 1, 1) + flops += count_conv_flop(1, se_mid, mb_conv['mid_channels'], 1, 1) + # point linear + flops += count_conv_flop(out_fz, mb_conv['mid_channels'], mb_conv['out_channels'], 1, 1) + fsize = out_fz + # final expand layer + flops += count_conv_flop(fsize, net_config['final_expand_layer']['in_channels'], + net_config['final_expand_layer']['out_channels'], 1, 1) + # feature mix layer + flops += count_conv_flop(1, net_config['feature_mix_layer']['in_channels'], + net_config['feature_mix_layer']['out_channels'], 1, 1) + # classifier + flops += count_conv_flop(1, net_config['classifier']['in_features'], + net_config['classifier']['out_features'], 1, 1) + return flops / 1e6 # MFLOPs + + +class ResNet50LatencyTable(LatencyTable): + + def query(self, **kwargs): + raise NotImplementedError + + def predict_network_latency(self, net, image_size): + raise NotImplementedError + + def predict_network_latency_given_config(self, net_config, image_size): + raise NotImplementedError + + @staticmethod + def count_flops_given_config(net_config, image_size=224): + flops = 0 + # input stem + for layer_config in net_config['input_stem']: + if layer_config['name'] != 'ConvLayer': + layer_config = layer_config['conv'] + in_channel = layer_config['in_channels'] + out_channel = layer_config['out_channels'] + out_image_size = int((image_size - 1) / layer_config['stride'] + 1) + + flops += count_conv_flop(out_image_size, in_channel, out_channel, + layer_config['kernel_size'], layer_config.get('groups', 1)) + image_size = out_image_size + # max pooling + image_size = int((image_size - 1) / 2 + 1) + # ResNetBottleneckBlocks + for block_config in net_config['blocks']: + in_channel = block_config['in_channels'] + out_channel = block_config['out_channels'] + + out_image_size = int((image_size - 1) / block_config['stride'] + 1) + mid_channel = block_config['mid_channels'] if block_config['mid_channels'] is not None \ + else round(out_channel * block_config['expand_ratio']) + mid_channel = make_divisible(mid_channel, MyNetwork.CHANNEL_DIVISIBLE) + + # conv1 + flops += count_conv_flop(image_size, in_channel, mid_channel, 1, 1) + # conv2 + flops += count_conv_flop(out_image_size, mid_channel, mid_channel, + block_config['kernel_size'], block_config['groups']) + # conv3 + flops += count_conv_flop(out_image_size, mid_channel, out_channel, 1, 1) + # downsample + if block_config['stride'] == 1 and in_channel == out_channel: + pass + else: + flops += count_conv_flop(out_image_size, in_channel, out_channel, 1, 1) + image_size = out_image_size + # final classifier + flops += count_conv_flop(1, net_config['classifier']['in_features'], + net_config['classifier']['out_features'], 1, 1) + return flops / 1e6 # MFLOPs diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/search_algorithm/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/search_algorithm/__init__.py new file mode 100644 index 0000000..13817d2 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/search_algorithm/__init__.py @@ -0,0 +1,5 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .evolution import * diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/search_algorithm/evolution.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/search_algorithm/evolution.py new file mode 100644 index 0000000..511e890 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/nas/search_algorithm/evolution.py @@ -0,0 +1,134 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import copy +import random +import numpy as np +from tqdm import tqdm + +__all__ = ['EvolutionFinder'] + + +class EvolutionFinder: + + def __init__(self, efficiency_predictor, accuracy_predictor, **kwargs): + self.efficiency_predictor = efficiency_predictor + self.accuracy_predictor = accuracy_predictor + + # evolution hyper-parameters + self.arch_mutate_prob = kwargs.get('arch_mutate_prob', 0.1) + self.resolution_mutate_prob = kwargs.get('resolution_mutate_prob', 0.5) + self.population_size = kwargs.get('population_size', 100) + self.max_time_budget = kwargs.get('max_time_budget', 500) + self.parent_ratio = kwargs.get('parent_ratio', 0.25) + self.mutation_ratio = kwargs.get('mutation_ratio', 0.5) + + @property + def arch_manager(self): + return self.accuracy_predictor.arch_encoder + + def update_hyper_params(self, new_param_dict): + self.__dict__.update(new_param_dict) + + def random_valid_sample(self, constraint): + while True: + sample = self.arch_manager.random_sample_arch() + efficiency = self.efficiency_predictor.get_efficiency(sample) + if efficiency <= constraint: + return sample, efficiency + + def mutate_sample(self, sample, constraint): + while True: + new_sample = copy.deepcopy(sample) + + self.arch_manager.mutate_resolution(new_sample, self.resolution_mutate_prob) + self.arch_manager.mutate_arch(new_sample, self.arch_mutate_prob) + + efficiency = self.efficiency_predictor.get_efficiency(new_sample) + if efficiency <= constraint: + return new_sample, efficiency + + def crossover_sample(self, sample1, sample2, constraint): + while True: + new_sample = copy.deepcopy(sample1) + for key in new_sample.keys(): + if not isinstance(new_sample[key], list): + new_sample[key] = random.choice([sample1[key], sample2[key]]) + else: + for i in range(len(new_sample[key])): + new_sample[key][i] = random.choice([sample1[key][i], sample2[key][i]]) + + efficiency = self.efficiency_predictor.get_efficiency(new_sample) + if efficiency <= constraint: + return new_sample, efficiency + + def run_evolution_search(self, constraint, verbose=False, **kwargs): + """Run a single roll-out of regularized evolution to a fixed time budget.""" + self.update_hyper_params(kwargs) + + mutation_numbers = int(round(self.mutation_ratio * self.population_size)) + parents_size = int(round(self.parent_ratio * self.population_size)) + + best_valids = [-100] + population = [] # (validation, sample, latency) tuples + child_pool = [] + efficiency_pool = [] + best_info = None + if verbose: + print('Generate random population...') + for _ in range(self.population_size): + sample, efficiency = self.random_valid_sample(constraint) + child_pool.append(sample) + efficiency_pool.append(efficiency) + + accs = self.accuracy_predictor.predict_acc(child_pool) + for i in range(mutation_numbers): + population.append((accs[i].item(), child_pool[i], efficiency_pool[i])) + + if verbose: + print('Start Evolution...') + # After the population is seeded, proceed with evolving the population. + with tqdm(total=self.max_time_budget, desc='Searching with constraint (%s)' % constraint, + disable=(not verbose)) as t: + for i in range(self.max_time_budget): + parents = sorted(population, key=lambda x: x[0])[::-1][:parents_size] + acc = parents[0][0] + t.set_postfix({ + 'acc': parents[0][0] + }) + if not verbose and (i + 1) % 100 == 0: + print('Iter: {} Acc: {}'.format(i + 1, parents[0][0])) + + if acc > best_valids[-1]: + best_valids.append(acc) + best_info = parents[0] + else: + best_valids.append(best_valids[-1]) + + population = parents + child_pool = [] + efficiency_pool = [] + + for j in range(mutation_numbers): + par_sample = population[np.random.randint(parents_size)][1] + # Mutate + new_sample, efficiency = self.mutate_sample(par_sample, constraint) + child_pool.append(new_sample) + efficiency_pool.append(efficiency) + + for j in range(self.population_size - mutation_numbers): + par_sample1 = population[np.random.randint(parents_size)][1] + par_sample2 = population[np.random.randint(parents_size)][1] + # Crossover + new_sample, efficiency = self.crossover_sample(par_sample1, par_sample2, constraint) + child_pool.append(new_sample) + efficiency_pool.append(efficiency) + + accs = self.accuracy_predictor.predict_acc(child_pool) + for j in range(self.population_size): + population.append((accs[j].item(), child_pool[j], efficiency_pool[j])) + + t.update(1) + + return best_valids, best_info diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/__init__.py new file mode 100644 index 0000000..7454357 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/__init__.py @@ -0,0 +1,5 @@ +from .accuracy_predictor import AccuracyPredictor +from .flops_table import FLOPsTable +from .latency_table import LatencyTable +from .evolution_finder import EvolutionFinder, ArchManager +from .imagenet_eval_helper import evaluate_ofa_subnet, evaluate_ofa_specialized diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/accuracy_predictor.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/accuracy_predictor.py new file mode 100644 index 0000000..10d033a --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/accuracy_predictor.py @@ -0,0 +1,85 @@ +import torch.nn as nn +import torch + +import copy + +from ofa.utils import download_url + + +# Helper for constructing the one-hot vectors. +def construct_maps(keys): + d = dict() + keys = list(set(keys)) + for k in keys: + if k not in d: + d[k] = len(list(d.keys())) + return d + + +ks_map = construct_maps(keys=(3, 5, 7)) +ex_map = construct_maps(keys=(3, 4, 6)) +dp_map = construct_maps(keys=(2, 3, 4)) + + +class AccuracyPredictor: + def __init__(self, pretrained=True, device='cuda:0'): + self.device = device + + self.model = nn.Sequential( + nn.Linear(128, 400), + nn.ReLU(), + nn.Linear(400, 400), + nn.ReLU(), + nn.Linear(400, 400), + nn.ReLU(), + nn.Linear(400, 1), + ) + if pretrained: + # load pretrained model + fname = download_url("https://hanlab.mit.edu/files/OnceForAll/tutorial/acc_predictor.pth") + self.model.load_state_dict( + torch.load(fname, map_location=torch.device('cpu')) + ) + self.model = self.model.to(self.device) + + # TODO: merge it with serialization utils. + @torch.no_grad() + def predict_accuracy(self, population): + all_feats = [] + for sample in population: + ks_list = copy.deepcopy(sample['ks']) + ex_list = copy.deepcopy(sample['e']) + d_list = copy.deepcopy(sample['d']) + r = copy.deepcopy(sample['r'])[0] + feats = AccuracyPredictor.spec2feats(ks_list, ex_list, d_list, r).reshape(1, -1).to(self.device) + all_feats.append(feats) + all_feats = torch.cat(all_feats, 0) + pred = self.model(all_feats).cpu() + return pred + + @staticmethod + def spec2feats(ks_list, ex_list, d_list, r): + # This function converts a network config to a feature vector (128-D). + start = 0 + end = 4 + for d in d_list: + for j in range(start+d, end): + ks_list[j] = 0 + ex_list[j] = 0 + start += 4 + end += 4 + + # convert to onehot + ks_onehot = [0 for _ in range(60)] + ex_onehot = [0 for _ in range(60)] + r_onehot = [0 for _ in range(8)] + + for i in range(20): + start = i * 3 + if ks_list[i] != 0: + ks_onehot[start + ks_map[ks_list[i]]] = 1 + if ex_list[i] != 0: + ex_onehot[start + ex_map[ex_list[i]]] = 1 + + r_onehot[(r - 112) // 16] = 1 + return torch.Tensor(ks_onehot + ex_onehot + r_onehot) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/evolution_finder.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/evolution_finder.py new file mode 100644 index 0000000..ffb5537 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/evolution_finder.py @@ -0,0 +1,213 @@ +import copy +import random +from tqdm import tqdm +import numpy as np + +__all__ = ['EvolutionFinder'] + + +class ArchManager: + def __init__(self): + self.num_blocks = 20 + self.num_stages = 5 + self.kernel_sizes = [3, 5, 7] + self.expand_ratios = [3, 4, 6] + self.depths = [2, 3, 4] + self.resolutions = [160, 176, 192, 208, 224] + + def random_sample(self): + sample = {} + d = [] + e = [] + ks = [] + for i in range(self.num_stages): + d.append(random.choice(self.depths)) + + for i in range(self.num_blocks): + e.append(random.choice(self.expand_ratios)) + ks.append(random.choice(self.kernel_sizes)) + + sample = { + 'wid': None, + 'ks': ks, + 'e': e, + 'd': d, + 'r': [random.choice(self.resolutions)] + } + + return sample + + def random_resample(self, sample, i): + assert i >= 0 and i < self.num_blocks + sample['ks'][i] = random.choice(self.kernel_sizes) + sample['e'][i] = random.choice(self.expand_ratios) + + def random_resample_depth(self, sample, i): + assert i >= 0 and i < self.num_stages + sample['d'][i] = random.choice(self.depths) + + def random_resample_resolution(self, sample): + sample['r'][0] = random.choice(self.resolutions) + + +class EvolutionFinder: + valid_constraint_range = { + 'flops': [150, 600], + 'note10': [15, 60], + } + + def __init__(self, constraint_type, efficiency_constraint, + efficiency_predictor, accuracy_predictor, **kwargs): + self.constraint_type = constraint_type + if not constraint_type in self.valid_constraint_range.keys(): + self.invite_reset_constraint_type() + self.efficiency_constraint = efficiency_constraint + if not (efficiency_constraint <= self.valid_constraint_range[constraint_type][1] and + efficiency_constraint >= self.valid_constraint_range[constraint_type][0]): + self.invite_reset_constraint() + + self.efficiency_predictor = efficiency_predictor + self.accuracy_predictor = accuracy_predictor + self.arch_manager = ArchManager() + self.num_blocks = self.arch_manager.num_blocks + self.num_stages = self.arch_manager.num_stages + + self.mutate_prob = kwargs.get('mutate_prob', 0.1) + self.population_size = kwargs.get('population_size', 100) + self.max_time_budget = kwargs.get('max_time_budget', 500) + self.parent_ratio = kwargs.get('parent_ratio', 0.25) + self.mutation_ratio = kwargs.get('mutation_ratio', 0.5) + + def invite_reset_constraint_type(self): + print('Invalid constraint type! Please input one of:', list(self.valid_constraint_range.keys())) + new_type = input() + while new_type not in self.valid_constraint_range.keys(): + print('Invalid constraint type! Please input one of:', list(self.valid_constraint_range.keys())) + new_type = input() + self.constraint_type = new_type + + def invite_reset_constraint(self): + print('Invalid constraint_value! Please input an integer in interval: [%d, %d]!' % ( + self.valid_constraint_range[self.constraint_type][0], + self.valid_constraint_range[self.constraint_type][1]) + ) + + new_cons = input() + while (not new_cons.isdigit()) or (int(new_cons) > self.valid_constraint_range[self.constraint_type][1]) or \ + (int(new_cons) < self.valid_constraint_range[self.constraint_type][0]): + print('Invalid constraint_value! Please input an integer in interval: [%d, %d]!' % ( + self.valid_constraint_range[self.constraint_type][0], + self.valid_constraint_range[self.constraint_type][1]) + ) + new_cons = input() + new_cons = int(new_cons) + self.efficiency_constraint = new_cons + + def set_efficiency_constraint(self, new_constraint): + self.efficiency_constraint = new_constraint + + def random_sample(self): + constraint = self.efficiency_constraint + while True: + sample = self.arch_manager.random_sample() + efficiency = self.efficiency_predictor.predict_efficiency(sample) + if efficiency <= constraint: + return sample, efficiency + + def mutate_sample(self, sample): + constraint = self.efficiency_constraint + while True: + new_sample = copy.deepcopy(sample) + + if random.random() < self.mutate_prob: + self.arch_manager.random_resample_resolution(new_sample) + + for i in range(self.num_blocks): + if random.random() < self.mutate_prob: + self.arch_manager.random_resample(new_sample, i) + + for i in range(self.num_stages): + if random.random() < self.mutate_prob: + self.arch_manager.random_resample_depth(new_sample, i) + + efficiency = self.efficiency_predictor.predict_efficiency(new_sample) + if efficiency <= constraint: + return new_sample, efficiency + + def crossover_sample(self, sample1, sample2): + constraint = self.efficiency_constraint + while True: + new_sample = copy.deepcopy(sample1) + for key in new_sample.keys(): + if not isinstance(new_sample[key], list): + continue + for i in range(len(new_sample[key])): + new_sample[key][i] = random.choice([sample1[key][i], sample2[key][i]]) + + efficiency = self.efficiency_predictor.predict_efficiency(new_sample) + if efficiency <= constraint: + return new_sample, efficiency + + def run_evolution_search(self, verbose=False): + """Run a single roll-out of regularized evolution to a fixed time budget.""" + max_time_budget = self.max_time_budget + population_size = self.population_size + mutation_numbers = int(round(self.mutation_ratio * population_size)) + parents_size = int(round(self.parent_ratio * population_size)) + constraint = self.efficiency_constraint + + best_valids = [-100] + population = [] # (validation, sample, latency) tuples + child_pool = [] + efficiency_pool = [] + best_info = None + if verbose: + print('Generate random population...') + for _ in range(population_size): + sample, efficiency = self.random_sample() + child_pool.append(sample) + efficiency_pool.append(efficiency) + + accs = self.accuracy_predictor.predict_accuracy(child_pool) + for i in range(mutation_numbers): + population.append((accs[i].item(), child_pool[i], efficiency_pool[i])) + + if verbose: + print('Start Evolution...') + # After the population is seeded, proceed with evolving the population. + for iter in tqdm(range(max_time_budget), desc='Searching with %s constraint (%s)' % (self.constraint_type, self.efficiency_constraint)): + parents = sorted(population, key=lambda x: x[0])[::-1][:parents_size] + acc = parents[0][0] + if verbose: + print('Iter: {} Acc: {}'.format(iter - 1, parents[0][0])) + + if acc > best_valids[-1]: + best_valids.append(acc) + best_info = parents[0] + else: + best_valids.append(best_valids[-1]) + + population = parents + child_pool = [] + efficiency_pool = [] + + for i in range(mutation_numbers): + par_sample = population[np.random.randint(parents_size)][1] + # Mutate + new_sample, efficiency = self.mutate_sample(par_sample) + child_pool.append(new_sample) + efficiency_pool.append(efficiency) + + for i in range(population_size - mutation_numbers): + par_sample1 = population[np.random.randint(parents_size)][1] + par_sample2 = population[np.random.randint(parents_size)][1] + # Crossover + new_sample, efficiency = self.crossover_sample(par_sample1, par_sample2) + child_pool.append(new_sample) + efficiency_pool.append(efficiency) + + accs = self.accuracy_predictor.predict_accuracy(child_pool) + for i in range(population_size): + population.append((accs[i].item(), child_pool[i], efficiency_pool[i])) + + return best_valids, best_info diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/flops_table.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/flops_table.py new file mode 100644 index 0000000..97bc0af --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/flops_table.py @@ -0,0 +1,224 @@ +import time +import copy +import torch +import torch.nn as nn +import numpy as np +from ofa.utils.layers import * + +__all__ = ['FLOPsTable'] + + +def rm_bn_from_net(net): + for m in net.modules(): + if isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d): + m.forward = lambda x: x + + +class FLOPsTable: + def __init__(self, pred_type='flops', device='cuda:0', multiplier=1.2, batch_size=64, load_efficiency_table=None): + assert pred_type in ['flops', 'latency'] + self.multiplier = multiplier + self.pred_type = pred_type + self.device = device + self.batch_size = batch_size + self.efficiency_dict = {} + if load_efficiency_table is not None: + self.efficiency_dict = np.load(load_efficiency_table, allow_pickle=True).item() + else: + self.build_lut(batch_size) + + @torch.no_grad() + def measure_single_layer_latency(self, layer: nn.Module, input_size: tuple, warmup_steps=10, measure_steps=50): + total_time = 0 + inputs = torch.randn(*input_size, device=self.device) + layer.eval() + rm_bn_from_net(layer) + network = layer.to(self.device) + torch.cuda.synchronize() + for i in range(warmup_steps): + network(inputs) + torch.cuda.synchronize() + + torch.cuda.synchronize() + st = time.time() + for i in range(measure_steps): + network(inputs) + torch.cuda.synchronize() + ed = time.time() + total_time += ed - st + + latency = total_time / measure_steps * 1000 + + return latency + + @torch.no_grad() + def measure_single_layer_flops(self, layer: nn.Module, input_size: tuple): + import thop + inputs = torch.randn(*input_size, device=self.device) + network = layer.to(self.device) + layer.eval() + rm_bn_from_net(layer) + flops, params = thop.profile(network, (inputs,), verbose=False) + return flops / 1e6 + + def build_lut(self, batch_size=1, resolutions=[160, 176, 192, 208, 224]): + for resolution in resolutions: + self.build_single_lut(batch_size, resolution) + + np.save('local_lut.npy', self.efficiency_dict) + + def build_single_lut(self, batch_size=1, base_resolution=224): + print('Building the %s lookup table (resolution=%d)...' % (self.pred_type, base_resolution)) + # block, input_size, in_channels, out_channels, expand_ratio, kernel_size, stride, act, se + configurations = [ + (ConvLayer, base_resolution, 3, 16, 3, 2, 'relu'), + (ResidualBlock, base_resolution // 2, 16, 16, [1], [3, 5, 7], 1, 'relu', False), + (ResidualBlock, base_resolution // 2, 16, 24, [3, 4, 6], [3, 5, 7], 2, 'relu', False), + (ResidualBlock, base_resolution // 4, 24, 24, [3, 4, 6], [3, 5, 7], 1, 'relu', False), + (ResidualBlock, base_resolution // 4, 24, 24, [3, 4, 6], [3, 5, 7], 1, 'relu', False), + (ResidualBlock, base_resolution // 4, 24, 24, [3, 4, 6], [3, 5, 7], 1, 'relu', False), + (ResidualBlock, base_resolution // 4, 24, 40, [3, 4, 6], [3, 5, 7], 2, 'relu', True), + (ResidualBlock, base_resolution // 8, 40, 40, [3, 4, 6], [3, 5, 7], 1, 'relu', True), + (ResidualBlock, base_resolution // 8, 40, 40, [3, 4, 6], [3, 5, 7], 1, 'relu', True), + (ResidualBlock, base_resolution // 8, 40, 40, [3, 4, 6], [3, 5, 7], 1, 'relu', True), + (ResidualBlock, base_resolution // 8, 40, 80, [3, 4, 6], [3, 5, 7], 2, 'h_swish', False), + (ResidualBlock, base_resolution // 16, 80, 80, [3, 4, 6], [3, 5, 7], 1, 'h_swish', False), + (ResidualBlock, base_resolution // 16, 80, 80, [3, 4, 6], [3, 5, 7], 1, 'h_swish', False), + (ResidualBlock, base_resolution // 16, 80, 80, [3, 4, 6], [3, 5, 7], 1, 'h_swish', False), + (ResidualBlock, base_resolution // 16, 80, 112, [3, 4, 6], [3, 5, 7], 1, 'h_swish', True), + (ResidualBlock, base_resolution // 16, 112, 112, [3, 4, 6], [3, 5, 7], 1, 'h_swish', True), + (ResidualBlock, base_resolution // 16, 112, 112, [3, 4, 6], [3, 5, 7], 1, 'h_swish', True), + (ResidualBlock, base_resolution // 16, 112, 112, [3, 4, 6], [3, 5, 7], 1, 'h_swish', True), + (ResidualBlock, base_resolution // 16, 112, 160, [3, 4, 6], [3, 5, 7], 2, 'h_swish', True), + (ResidualBlock, base_resolution // 32, 160, 160, [3, 4, 6], [3, 5, 7], 1, 'h_swish', True), + (ResidualBlock, base_resolution // 32, 160, 160, [3, 4, 6], [3, 5, 7], 1, 'h_swish', True), + (ResidualBlock, base_resolution // 32, 160, 160, [3, 4, 6], [3, 5, 7], 1, 'h_swish', True), + (ConvLayer, base_resolution // 32, 160, 960, 1, 1, 'h_swish'), + (ConvLayer, 1, 960, 1280, 1, 1, 'h_swish'), + (LinearLayer, 1, 1280, 1000, 1, 1), + ] + + efficiency_dict = { + 'mobile_inverted_blocks': [], + 'other_blocks': {} + } + + for layer_idx in range(len(configurations)): + config = configurations[layer_idx] + op_type = config[0] + if op_type == ResidualBlock: + _, input_size, in_channels, out_channels, expand_list, ks_list, stride, act, se = config + in_channels = int(round(in_channels * self.multiplier)) + out_channels = int(round(out_channels * self.multiplier)) + template_config = { + 'name': ResidualBlock.__name__, + 'mobile_inverted_conv': { + 'name': MBConvLayer.__name__, + 'in_channels': in_channels, + 'out_channels': out_channels, + 'kernel_size': kernel_size, + 'stride': stride, + 'expand_ratio': 0, + # 'mid_channels': None, + 'act_func': act, + 'use_se': se, + }, + 'shortcut': { + 'name': IdentityLayer.__name__, + 'in_channels': in_channels, + 'out_channels': out_channels, + } if (in_channels == out_channels and stride == 1) else None + } + sub_dict = {} + for ks in ks_list: + for e in expand_list: + build_config = copy.deepcopy(template_config) + build_config['mobile_inverted_conv']['expand_ratio'] = e + build_config['mobile_inverted_conv']['kernel_size'] = ks + + layer = ResidualBlock.build_from_config(build_config) + input_shape = (batch_size, in_channels, input_size, input_size) + + if self.pred_type == 'flops': + measure_result = self.measure_single_layer_flops(layer, input_shape) / batch_size + elif self.pred_type == 'latency': + measure_result = self.measure_single_layer_latency(layer, input_shape) + + sub_dict[(ks, e)] = measure_result + + efficiency_dict['mobile_inverted_blocks'].append(sub_dict) + + elif op_type == ConvLayer: + _, input_size, in_channels, out_channels, kernel_size, stride, activation = config + in_channels = int(round(in_channels * self.multiplier)) + out_channels = int(round(out_channels * self.multiplier)) + build_config = { + # 'name': ConvLayer.__name__, + 'in_channels': in_channels, + 'out_channels': out_channels, + 'kernel_size': kernel_size, + 'stride': stride, + 'dilation': 1, + 'groups': 1, + 'bias': False, + 'use_bn': True, + 'has_shuffle': False, + 'act_func': activation, + } + layer = ConvLayer.build_from_config(build_config) + input_shape = (batch_size, in_channels, input_size, input_size) + + if self.pred_type == 'flops': + measure_result = self.measure_single_layer_flops(layer, input_shape) / batch_size + elif self.pred_type == 'latency': + measure_result = self.measure_single_layer_latency(layer, input_shape) + + efficiency_dict['other_blocks'][layer_idx] = measure_result + + elif op_type == LinearLayer: + _, input_size, in_channels, out_channels, kernel_size, stride = config + in_channels = int(round(in_channels * self.multiplier)) + out_channels = int(round(out_channels * self.multiplier)) + build_config = { + # 'name': LinearLayer.__name__, + 'in_features': in_channels, + 'out_features': out_channels + } + layer = LinearLayer.build_from_config(build_config) + input_shape = (batch_size, in_channels) + + if self.pred_type == 'flops': + measure_result = self.measure_single_layer_flops(layer, input_shape) / batch_size + elif self.pred_type == 'latency': + measure_result = self.measure_single_layer_latency(layer, input_shape) + + efficiency_dict['other_blocks'][layer_idx] = measure_result + + else: + raise NotImplementedError + + self.efficiency_dict[base_resolution] = efficiency_dict + print('Built the %s lookup table (resolution=%d)!' % (self.pred_type, base_resolution)) + return efficiency_dict + + def predict_efficiency(self, sample): + input_size = sample.get('r', [224]) + input_size = input_size[0] + assert 'ks' in sample and 'e' in sample and 'd' in sample + assert len(sample['ks']) == len(sample['e']) and len(sample['ks']) == 20 + assert len(sample['d']) == 5 + total_stats = 0. + for i in range(20): + stage = i // 4 + depth_max = sample['d'][stage] + depth = i % 4 + 1 + if depth > depth_max: + continue + ks, e = sample['ks'][i], sample['e'][i] + total_stats += self.efficiency_dict[input_size]['mobile_inverted_blocks'][i + 1][(ks, e)] + + for key in self.efficiency_dict[input_size]['other_blocks']: + total_stats += self.efficiency_dict[input_size]['other_blocks'][key] + + total_stats += self.efficiency_dict[input_size]['mobile_inverted_blocks'][0][(3, 1)] + return total_stats diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/imagenet_eval_helper.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/imagenet_eval_helper.py new file mode 100644 index 0000000..385f5dd --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/imagenet_eval_helper.py @@ -0,0 +1,241 @@ +import os.path as osp +import numpy as np +import math +from tqdm import tqdm + +import torch.nn as nn +import torch.backends.cudnn as cudnn +import torch.utils.data +from torchvision import transforms, datasets + +from ofa.utils import AverageMeter, accuracy +from ofa.model_zoo import ofa_specialized +from ofa.imagenet_classification.elastic_nn.utils import set_running_statistics + + +def evaluate_ofa_subnet(ofa_net, path, net_config, data_loader, batch_size, device='cuda:0'): + assert 'ks' in net_config and 'd' in net_config and 'e' in net_config + assert len(net_config['ks']) == 20 and len(net_config['e']) == 20 and len(net_config['d']) == 5 + ofa_net.set_active_subnet(ks=net_config['ks'], d=net_config['d'], e=net_config['e']) + subnet = ofa_net.get_active_subnet().to(device) + calib_bn(subnet, path, net_config['r'][0], batch_size) + top1 = validate(subnet, path, net_config['r'][0], data_loader, batch_size, device) + return top1 + + +def calib_bn(net, path, image_size, batch_size, num_images=2000): + # print('Creating dataloader for resetting BN running statistics...') + dataset = datasets.ImageFolder( + osp.join( + path, + 'train'), + transforms.Compose([ + transforms.RandomResizedCrop(image_size), + transforms.RandomHorizontalFlip(), + transforms.ColorJitter(brightness=32. / 255., saturation=0.5), + transforms.ToTensor(), + transforms.Normalize( + mean=[ + 0.485, + 0.456, + 0.406], + std=[ + 0.229, + 0.224, + 0.225] + ), + ]) + ) + chosen_indexes = np.random.choice(list(range(len(dataset))), num_images) + sub_sampler = torch.utils.data.sampler.SubsetRandomSampler(chosen_indexes) + data_loader = torch.utils.data.DataLoader( + dataset, + sampler=sub_sampler, + batch_size=batch_size, + num_workers=16, + pin_memory=True, + drop_last=False, + ) + # print('Resetting BN running statistics (this may take 10-20 seconds)...') + set_running_statistics(net, data_loader) + + + +def validate(net, path, image_size, data_loader, batch_size=100, device='cuda:0'): + if 'cuda' in device: + net = torch.nn.DataParallel(net).to(device) + else: + net = net.to(device) + + data_loader.dataset.transform = transforms.Compose([ + transforms.Resize(int(math.ceil(image_size / 0.875))), + transforms.CenterCrop(image_size), + transforms.ToTensor(), + transforms.Normalize( + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225] + ), + ]) + + cudnn.benchmark = True + criterion = nn.CrossEntropyLoss().to(device) + + net.eval() + net = net.to(device) + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + + with torch.no_grad(): + with tqdm(total=len(data_loader), desc='Validate') as t: + for i, (images, labels) in enumerate(data_loader): + images, labels = images.to(device), labels.to(device) + # compute output + output = net(images) + loss = criterion(output, labels) + # measure accuracy and record loss + acc1, acc5 = accuracy(output, labels, topk=(1, 5)) + + losses.update(loss.item(), images.size(0)) + top1.update(acc1[0].item(), images.size(0)) + top5.update(acc5[0].item(), images.size(0)) + t.set_postfix({ + 'loss': losses.avg, + 'top1': top1.avg, + 'top5': top5.avg, + 'img_size': images.size(2), + }) + t.update(1) + + + print('Results: loss=%.5f,\t top1=%.1f,\t top5=%.1f' % (losses.avg, top1.avg, top5.avg)) + return top1.avg + + +def evaluate_ofa_specialized(path, data_loader, batch_size=100, device='cuda:0'): + def select_platform_name(): + valid_platform_name = [ + 'pixel1', 'pixel2', 'note10', 'note8', 's7edge', 'lg-g8', '1080ti', 'v100', 'tx2', 'cpu', 'flops' + ] + + print("Please select a hardware platform from ('pixel1', 'pixel2', 'note10', 'note8', 's7edge', 'lg-g8', '1080ti', 'v100', 'tx2', 'cpu', 'flops')!\n") + + while True: + platform_name = input() + platform_name = platform_name.lower() + if platform_name in valid_platform_name: + return platform_name + print("Platform name is invalid! Please select in ('pixel1', 'pixel2', 'note10', 'note8', 's7edge', 'lg-g8', '1080ti', 'v100', 'tx2', 'cpu', 'flops')!\n") + + def select_netid(platform_name): + platform_efficiency_map = { + 'pixel1': { + 143: 'pixel1_lat@143ms_top1@80.1_finetune@75', + 132: 'pixel1_lat@132ms_top1@79.8_finetune@75', + 79: 'pixel1_lat@79ms_top1@78.7_finetune@75', + 58: 'pixel1_lat@58ms_top1@76.9_finetune@75', + 40: 'pixel1_lat@40ms_top1@74.9_finetune@25', + 28: 'pixel1_lat@28ms_top1@73.3_finetune@25', + 20: 'pixel1_lat@20ms_top1@71.4_finetune@25', + }, + + 'pixel2': { + 62: 'pixel2_lat@62ms_top1@75.8_finetune@25', + 50: 'pixel2_lat@50ms_top1@74.7_finetune@25', + 35: 'pixel2_lat@35ms_top1@73.4_finetune@25', + 25: 'pixel2_lat@25ms_top1@71.5_finetune@25', + }, + + 'note10': { + 64: 'note10_lat@64ms_top1@80.2_finetune@75', + 50: 'note10_lat@50ms_top1@79.7_finetune@75', + 41: 'note10_lat@41ms_top1@79.3_finetune@75', + 30: 'note10_lat@30ms_top1@78.4_finetune@75', + 22: 'note10_lat@22ms_top1@76.6_finetune@25', + 16: 'note10_lat@16ms_top1@75.5_finetune@25', + 11: 'note10_lat@11ms_top1@73.6_finetune@25', + 8: 'note10_lat@8ms_top1@71.4_finetune@25', + }, + + 'note8': { + 65: 'note8_lat@65ms_top1@76.1_finetune@25', + 49: 'note8_lat@49ms_top1@74.9_finetune@25', + 31: 'note8_lat@31ms_top1@72.8_finetune@25', + 22: 'note8_lat@22ms_top1@70.4_finetune@25', + }, + + 's7edge': { + 88: 's7edge_lat@88ms_top1@76.3_finetune@25', + 58: 's7edge_lat@58ms_top1@74.7_finetune@25', + 41: 's7edge_lat@41ms_top1@73.1_finetune@25', + 29: 's7edge_lat@29ms_top1@70.5_finetune@25', + }, + + 'lg-g8': { + 24: 'LG-G8_lat@24ms_top1@76.4_finetune@25', + 16: 'LG-G8_lat@16ms_top1@74.7_finetune@25', + 11: 'LG-G8_lat@11ms_top1@73.0_finetune@25', + 8: 'LG-G8_lat@8ms_top1@71.1_finetune@25', + }, + + '1080ti': { + 27: '1080ti_gpu64@27ms_top1@76.4_finetune@25', + 22: '1080ti_gpu64@22ms_top1@75.3_finetune@25', + 15: '1080ti_gpu64@15ms_top1@73.8_finetune@25', + 12: '1080ti_gpu64@12ms_top1@72.6_finetune@25', + }, + + 'v100': { + 11: 'v100_gpu64@11ms_top1@76.1_finetune@25', + 9: 'v100_gpu64@9ms_top1@75.3_finetune@25', + 6: 'v100_gpu64@6ms_top1@73.0_finetune@25', + 5: 'v100_gpu64@5ms_top1@71.6_finetune@25', + }, + + 'tx2': { + 96: 'tx2_gpu16@96ms_top1@75.8_finetune@25', + 80: 'tx2_gpu16@80ms_top1@75.4_finetune@25', + 47: 'tx2_gpu16@47ms_top1@72.9_finetune@25', + 35: 'tx2_gpu16@35ms_top1@70.3_finetune@25', + }, + + 'cpu': { + 17: 'cpu_lat@17ms_top1@75.7_finetune@25', + 15: 'cpu_lat@15ms_top1@74.6_finetune@25', + 11: 'cpu_lat@11ms_top1@72.0_finetune@25', + 10: 'cpu_lat@10ms_top1@71.1_finetune@25', + }, + + 'flops': { + 595: 'flops@595M_top1@80.0_finetune@75', + 482: 'flops@482M_top1@79.6_finetune@75', + 389: 'flops@389M_top1@79.1_finetune@75', + } + } + + sub_efficiency_map = platform_efficiency_map[platform_name] + if not platform_name == 'flops': + print("Now, please specify a latency constraint for model specialization among", sorted(list(sub_efficiency_map.keys())), 'ms. (Please just input the number.) \n') + else: + print("Now, please specify a FLOPs constraint for model specialization among", sorted(list(sub_efficiency_map.keys())), 'MFLOPs. (Please just input the number.) \n') + + while True: + efficiency_constraint = input() + if not efficiency_constraint.isdigit(): + print('Sorry, please input an integer! \n') + continue + efficiency_constraint = int(efficiency_constraint) + if not efficiency_constraint in sub_efficiency_map.keys(): + print('Sorry, please choose a value from: ', sorted(list(sub_efficiency_map.keys())), '.\n') + continue + return sub_efficiency_map[efficiency_constraint] + + platform_name = select_platform_name() + net_id = select_netid(platform_name) + + net, image_size = ofa_specialized(net_id=net_id, pretrained=True) + + validate(net, path, image_size, data_loader, batch_size, device) + + return net_id + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/latency_table.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/latency_table.py new file mode 100644 index 0000000..7369183 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/tutorial/latency_table.py @@ -0,0 +1,164 @@ +import yaml +from ofa.utils import download_url + + +class LatencyEstimator(object): + + def __init__(self, local_dir='~/.hancai/latency_tools/', + url='https://hanlab.mit.edu/files/proxylessNAS/LatencyTools/mobile_trim.yaml'): + if url.startswith('http'): + fname = download_url(url, local_dir, overwrite=True) + else: + fname = url + + with open(fname, 'r') as fp: + self.lut = yaml.load(fp) + + @staticmethod + def repr_shape(shape): + if isinstance(shape, (list, tuple)): + return 'x'.join(str(_) for _ in shape) + elif isinstance(shape, str): + return shape + else: + return TypeError + + def query(self, l_type: str, input_shape, output_shape, mid=None, ks=None, stride=None, id_skip=None, + se=None, h_swish=None): + infos = [l_type, 'input:%s' % self.repr_shape(input_shape), 'output:%s' % self.repr_shape(output_shape), ] + + if l_type in ('expanded_conv',): + assert None not in (mid, ks, stride, id_skip, se, h_swish) + infos += ['expand:%d' % mid, 'kernel:%d' % ks, 'stride:%d' % stride, 'idskip:%d' % id_skip, + 'se:%d' % se, 'hs:%d' % h_swish] + key = '-'.join(infos) + return self.lut[key]['mean'] + + def predict_network_latency(self, net, image_size=224): + predicted_latency = 0 + # first conv + predicted_latency += self.query( + 'Conv', [image_size, image_size, 3], + [(image_size + 1) // 2, (image_size + 1) // 2, net.first_conv.out_channels] + ) + # blocks + fsize = (image_size + 1) // 2 + for block in net.blocks: + mb_conv = block.mobile_inverted_conv + shortcut = block.shortcut + + if mb_conv is None: + continue + if shortcut is None: + idskip = 0 + else: + idskip = 1 + out_fz = int((fsize - 1) / mb_conv.stride + 1) + block_latency = self.query( + 'expanded_conv', [fsize, fsize, mb_conv.in_channels], [out_fz, out_fz, mb_conv.out_channels], + mid=mb_conv.depth_conv.conv.in_channels, ks=mb_conv.kernel_size, stride=mb_conv.stride, id_skip=idskip, + se=1 if mb_conv.use_se else 0, h_swish=1 if mb_conv.act_func == 'h_swish' else 0, + ) + predicted_latency += block_latency + fsize = out_fz + # final expand layer + predicted_latency += self.query( + 'Conv_1', [fsize, fsize, net.final_expand_layer.in_channels], + [fsize, fsize, net.final_expand_layer.out_channels], + ) + # global average pooling + predicted_latency += self.query( + 'AvgPool2D', [fsize, fsize, net.final_expand_layer.out_channels], + [1, 1, net.final_expand_layer.out_channels], + ) + # feature mix layer + predicted_latency += self.query( + 'Conv_2', [1, 1, net.feature_mix_layer.in_channels], + [1, 1, net.feature_mix_layer.out_channels] + ) + # classifier + predicted_latency += self.query( + 'Logits', [1, 1, net.classifier.in_features], [net.classifier.out_features] + ) + return predicted_latency + + def predict_network_latency_given_spec(self, spec): + image_size = spec['r'][0] + predicted_latency = 0 + # first conv + predicted_latency += self.query( + 'Conv', [image_size, image_size, 3], + [(image_size + 1) // 2, (image_size + 1) // 2, 24] + ) + # blocks + fsize = (image_size + 1) // 2 + # first block + predicted_latency += self.query( + 'expanded_conv', [fsize, fsize, 24], [fsize, fsize, 24], + mid=24, ks=3, stride=1, id_skip=1, se=0, h_swish=0, + ) + in_channel = 24 + stride_stages = [2, 2, 2, 1, 2] + width_stages = [32, 48, 96, 136, 192] + act_stages = ['relu', 'relu', 'h_swish', 'h_swish', 'h_swish'] + se_stages = [False, True, False, True, True] + for i in range(20): + stage = i // 4 + depth_max = spec['d'][stage] + depth = i % 4 + 1 + if depth > depth_max: + continue + ks, e = spec['ks'][i], spec['e'][i] + if i % 4 == 0: + stride = stride_stages[stage] + idskip = 0 + else: + stride = 1 + idskip = 1 + out_channel = width_stages[stage] + out_fz = int((fsize - 1) / stride + 1) + + mid_channel = round(in_channel * e) + block_latency = self.query( + 'expanded_conv', [fsize, fsize, in_channel], [out_fz, out_fz, out_channel], + mid=mid_channel, ks=ks, stride=stride, id_skip=idskip, + se=1 if se_stages[stage] else 0, h_swish=1 if act_stages[stage] == 'h_swish' else 0, + ) + predicted_latency += block_latency + fsize = out_fz + in_channel = out_channel + # final expand layer + predicted_latency += self.query( + 'Conv_1', [fsize, fsize, 192], + [fsize, fsize, 1152], + ) + # global average pooling + predicted_latency += self.query( + 'AvgPool2D', [fsize, fsize, 1152], + [1, 1, 1152], + ) + # feature mix layer + predicted_latency += self.query( + 'Conv_2', [1, 1, 1152], + [1, 1, 1536] + ) + # classifier + predicted_latency += self.query( + 'Logits', [1, 1, 1536], [1000] + ) + return predicted_latency + + +class LatencyTable: + def __init__(self, device='note10', resolutions=(160, 176, 192, 208, 224)): + self.latency_tables = {} + + for image_size in resolutions: + self.latency_tables[image_size] = LatencyEstimator( + url='https://hanlab.mit.edu/files/OnceForAll/tutorial/latency_table@%s/%d_lookup_table.yaml' % ( + device, image_size) + ) + print('Built latency table for image size: %d.' % image_size) + + def predict_efficiency(self, spec: dict): + return self.latency_tables[spec['r'][0]].predict_network_latency_given_spec(spec) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/__init__.py new file mode 100644 index 0000000..1839557 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/__init__.py @@ -0,0 +1,10 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +from .pytorch_modules import * +from .pytorch_utils import * +from .my_modules import * +from .flops_counter import * +from .common_tools import * +from .my_dataloader import * diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/common_tools.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/common_tools.py new file mode 100644 index 0000000..9d63b4a --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/common_tools.py @@ -0,0 +1,284 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import numpy as np +import os +import sys +import torch + +try: + from urllib import urlretrieve +except ImportError: + from urllib.request import urlretrieve + +__all__ = [ + 'sort_dict', 'get_same_padding', + 'get_split_list', 'list_sum', 'list_mean', 'list_join', + 'subset_mean', 'sub_filter_start_end', 'min_divisible_value', 'val2list', + 'download_url', + 'write_log', 'pairwise_accuracy', 'accuracy', + 'AverageMeter', 'MultiClassAverageMeter', + 'DistributedMetric', 'DistributedTensor', +] + + +def sort_dict(src_dict, reverse=False, return_dict=True): + output = sorted(src_dict.items(), key=lambda x: x[1], reverse=reverse) + if return_dict: + return dict(output) + else: + return output + + +def get_same_padding(kernel_size): + if isinstance(kernel_size, tuple): + assert len(kernel_size) == 2, 'invalid kernel size: %s' % kernel_size + p1 = get_same_padding(kernel_size[0]) + p2 = get_same_padding(kernel_size[1]) + return p1, p2 + assert isinstance(kernel_size, int), 'kernel size should be either `int` or `tuple`' + assert kernel_size % 2 > 0, 'kernel size should be odd number' + return kernel_size // 2 + + +def get_split_list(in_dim, child_num, accumulate=False): + in_dim_list = [in_dim // child_num] * child_num + for _i in range(in_dim % child_num): + in_dim_list[_i] += 1 + if accumulate: + for i in range(1, child_num): + in_dim_list[i] += in_dim_list[i - 1] + return in_dim_list + + +def list_sum(x): + return x[0] if len(x) == 1 else x[0] + list_sum(x[1:]) + + +def list_mean(x): + return list_sum(x) / len(x) + + +def list_join(val_list, sep='\t'): + return sep.join([str(val) for val in val_list]) + + +def subset_mean(val_list, sub_indexes): + sub_indexes = val2list(sub_indexes, 1) + return list_mean([val_list[idx] for idx in sub_indexes]) + + +def sub_filter_start_end(kernel_size, sub_kernel_size): + center = kernel_size // 2 + dev = sub_kernel_size // 2 + start, end = center - dev, center + dev + 1 + assert end - start == sub_kernel_size + return start, end + + +def min_divisible_value(n1, v1): + """ make sure v1 is divisible by n1, otherwise decrease v1 """ + if v1 >= n1: + return n1 + while n1 % v1 != 0: + v1 -= 1 + return v1 + + +def val2list(val, repeat_time=1): + if isinstance(val, list) or isinstance(val, np.ndarray): + return val + elif isinstance(val, tuple): + return list(val) + else: + return [val for _ in range(repeat_time)] + + +def download_url(url, model_dir='~/.torch/', overwrite=False): + target_dir = url.split('/')[-1] + model_dir = os.path.expanduser(model_dir) + try: + if not os.path.exists(model_dir): + os.makedirs(model_dir) + model_dir = os.path.join(model_dir, target_dir) + cached_file = model_dir + if not os.path.exists(cached_file) or overwrite: + sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file)) + urlretrieve(url, cached_file) + return cached_file + except Exception as e: + # remove lock file so download can be executed next time. + os.remove(os.path.join(model_dir, 'download.lock')) + sys.stderr.write('Failed to download from url %s' % url + '\n' + str(e) + '\n') + return None + + +def write_log(logs_path, log_str, prefix='valid', should_print=True, mode='a'): + if not os.path.exists(logs_path): + os.makedirs(logs_path, exist_ok=True) + """ prefix: valid, train, test """ + if prefix in ['valid', 'test']: + with open(os.path.join(logs_path, 'valid_console.txt'), mode) as fout: + fout.write(log_str + '\n') + fout.flush() + if prefix in ['valid', 'test', 'train']: + with open(os.path.join(logs_path, 'train_console.txt'), mode) as fout: + if prefix in ['valid', 'test']: + fout.write('=' * 10) + fout.write(log_str + '\n') + fout.flush() + else: + with open(os.path.join(logs_path, '%s.txt' % prefix), mode) as fout: + fout.write(log_str + '\n') + fout.flush() + if should_print: + print(log_str) + + +def pairwise_accuracy(la, lb, n_samples=200000): + n = len(la) + assert n == len(lb) + total = 0 + count = 0 + for _ in range(n_samples): + i = np.random.randint(n) + j = np.random.randint(n) + while i == j: + j = np.random.randint(n) + if la[i] >= la[j] and lb[i] >= lb[j]: + count += 1 + if la[i] < la[j] and lb[i] < lb[j]: + count += 1 + total += 1 + return float(count) / total + + +def accuracy(output, target, topk=(1,)): + """ Computes the precision@k for the specified values of k """ + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.reshape(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +class AverageMeter(object): + """ + Computes and stores the average and current value + Copied from: https://github.com/pytorch/examples/blob/master/imagenet/main.py + """ + + def __init__(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +class MultiClassAverageMeter: + + """ Multi Binary Classification Tasks """ + def __init__(self, num_classes, balanced=False, **kwargs): + + super(MultiClassAverageMeter, self).__init__() + self.num_classes = num_classes + self.balanced = balanced + + self.counts = [] + for k in range(self.num_classes): + self.counts.append(np.ndarray((2, 2), dtype=np.float32)) + + self.reset() + + def reset(self): + for k in range(self.num_classes): + self.counts[k].fill(0) + + def add(self, outputs, targets): + outputs = outputs.data.cpu().numpy() + targets = targets.data.cpu().numpy() + + for k in range(self.num_classes): + output = np.argmax(outputs[:, k, :], axis=1) + target = targets[:, k] + + x = output + 2 * target + bincount = np.bincount(x.astype(np.int32), minlength=2 ** 2) + + self.counts[k] += bincount.reshape((2, 2)) + + def value(self): + mean = 0 + for k in range(self.num_classes): + if self.balanced: + value = np.mean((self.counts[k] / np.maximum(np.sum(self.counts[k], axis=1), 1)[:, None]).diagonal()) + else: + value = np.sum(self.counts[k].diagonal()) / np.maximum(np.sum(self.counts[k]), 1) + + mean += value / self.num_classes * 100. + return mean + + +class DistributedMetric(object): + """ + Horovod: average metrics from distributed training. + """ + def __init__(self, name): + self.name = name + self.sum = torch.zeros(1)[0] + self.count = torch.zeros(1)[0] + + def update(self, val, delta_n=1): + import horovod.torch as hvd + val *= delta_n + self.sum += hvd.allreduce(val.detach().cpu(), name=self.name) + self.count += delta_n + + @property + def avg(self): + return self.sum / self.count + + +class DistributedTensor(object): + + def __init__(self, name): + self.name = name + self.sum = None + self.count = torch.zeros(1)[0] + self.synced = False + + def update(self, val, delta_n=1): + val *= delta_n + if self.sum is None: + self.sum = val.detach() + else: + self.sum += val.detach() + self.count += delta_n + + @property + def avg(self): + import horovod.torch as hvd + if not self.synced: + self.sum = hvd.allreduce(self.sum, name=self.name) + self.synced = True + return self.sum / self.count diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/flops_counter.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/flops_counter.py new file mode 100644 index 0000000..751984c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/flops_counter.py @@ -0,0 +1,97 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import torch +import torch.nn as nn + +from .my_modules import MyConv2d + +__all__ = ['profile'] + + +def count_convNd(m, _, y): + cin = m.in_channels + + kernel_ops = m.weight.size()[2] * m.weight.size()[3] + ops_per_element = kernel_ops + output_elements = y.nelement() + + # cout x oW x oH + total_ops = cin * output_elements * ops_per_element // m.groups + m.total_ops = torch.zeros(1).fill_(total_ops) + + +def count_linear(m, _, __): + total_ops = m.in_features * m.out_features + + m.total_ops = torch.zeros(1).fill_(total_ops) + + +register_hooks = { + nn.Conv1d: count_convNd, + nn.Conv2d: count_convNd, + nn.Conv3d: count_convNd, + MyConv2d: count_convNd, + ###################################### + nn.Linear: count_linear, + ###################################### + nn.Dropout: None, + nn.Dropout2d: None, + nn.Dropout3d: None, + nn.BatchNorm2d: None, +} + + +def profile(model, input_size, custom_ops=None): + handler_collection = [] + custom_ops = {} if custom_ops is None else custom_ops + + def add_hooks(m_): + if len(list(m_.children())) > 0: + return + + m_.register_buffer('total_ops', torch.zeros(1)) + m_.register_buffer('total_params', torch.zeros(1)) + + for p in m_.parameters(): + m_.total_params += torch.zeros(1).fill_(p.numel()) + + m_type = type(m_) + fn = None + + if m_type in custom_ops: + fn = custom_ops[m_type] + elif m_type in register_hooks: + fn = register_hooks[m_type] + + if fn is not None: + _handler = m_.register_forward_hook(fn) + handler_collection.append(_handler) + + original_device = model.parameters().__next__().device + training = model.training + + model.eval() + model.apply(add_hooks) + + x = torch.zeros(input_size).to(original_device) + with torch.no_grad(): + model(x) + + total_ops = 0 + total_params = 0 + for m in model.modules(): + if len(list(m.children())) > 0: # skip for non-leaf module + continue + total_ops += m.total_ops + total_params += m.total_params + + total_ops = total_ops.item() + total_params = total_params.item() + + model.train(training).to(original_device) + for handler in handler_collection: + handler.remove() + + return total_ops, total_params diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/layers.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/layers.py new file mode 100644 index 0000000..581aa87 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/layers.py @@ -0,0 +1,727 @@ +###################################################################################### +# Copyright (c) Han Cai, Once for All, ICLR 2020 [GitHub OFA] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +import torch.nn as nn +from torch.distributions import Bernoulli +from collections import OrderedDict +from ofa_local.utils import get_same_padding, min_divisible_value, SEModule, ShuffleLayer +from ofa_local.utils import MyNetwork, MyModule +from ofa_local.utils import build_activation, make_divisible + +__all__ = [ + 'set_layer_from_config', + 'ConvLayer', 'IdentityLayer', 'LinearLayer', 'MultiHeadLinearLayer', 'ZeroLayer', 'MBConvLayer', + 'ResidualBlock', 'ResNetBottleneckBlock', +] + + +class DropBlock(nn.Module): + def __init__(self, block_size): + super(DropBlock, self).__init__() + + self.block_size = block_size + + def forward(self, x, gamma): + # shape: (bsize, channels, height, width) + + if self.training: + batch_size, channels, height, width = x.shape + + bernoulli = Bernoulli(gamma) + mask = bernoulli.sample( + (batch_size, channels, height - (self.block_size - 1), width - (self.block_size - 1))).cuda() + # print((x.sample[-2], x.sample[-1])) + block_mask = self._compute_block_mask(mask) + # print (block_mask.size()) + # print (x.size()) + countM = block_mask.size()[0] * block_mask.size()[1] * block_mask.size()[2] * block_mask.size()[3] + count_ones = block_mask.sum() + + return block_mask * x * (countM / count_ones) + else: + return x + + def _compute_block_mask(self, mask): + left_padding = int((self.block_size - 1) / 2) + right_padding = int(self.block_size / 2) + + batch_size, channels, height, width = mask.shape + # print ("mask", mask[0][0]) + non_zero_idxs = mask.nonzero() + nr_blocks = non_zero_idxs.shape[0] + + offsets = torch.stack( + [ + torch.arange(self.block_size).view(-1, 1).expand(self.block_size, self.block_size).reshape(-1), + # - left_padding, + torch.arange(self.block_size).repeat(self.block_size), # - left_padding + ] + ).t().cuda() + offsets = torch.cat((torch.zeros(self.block_size ** 2, 2).cuda().long(), offsets.long()), 1) + + if nr_blocks > 0: + non_zero_idxs = non_zero_idxs.repeat(self.block_size ** 2, 1) + offsets = offsets.repeat(nr_blocks, 1).view(-1, 4) + offsets = offsets.long() + + block_idxs = non_zero_idxs + offsets + # block_idxs += left_padding + padded_mask = F.pad(mask, (left_padding, right_padding, left_padding, right_padding)) + padded_mask[block_idxs[:, 0], block_idxs[:, 1], block_idxs[:, 2], block_idxs[:, 3]] = 1. + else: + padded_mask = F.pad(mask, (left_padding, right_padding, left_padding, right_padding)) + + block_mask = 1 - padded_mask # [:height, :width] + return block_mask + + +def set_layer_from_config(layer_config): + if layer_config is None: + return None + + name2layer = { + ConvLayer.__name__: ConvLayer, + IdentityLayer.__name__: IdentityLayer, + LinearLayer.__name__: LinearLayer, + MultiHeadLinearLayer.__name__: MultiHeadLinearLayer, + ZeroLayer.__name__: ZeroLayer, + MBConvLayer.__name__: MBConvLayer, + 'MBInvertedConvLayer': MBConvLayer, + ########################################################## + ResidualBlock.__name__: ResidualBlock, + ResNetBottleneckBlock.__name__: ResNetBottleneckBlock, + } + + layer_name = layer_config.pop('name') + layer = name2layer[layer_name] + return layer.build_from_config(layer_config) + + +class My2DLayer(MyModule): + + def __init__(self, in_channels, out_channels, + use_bn=True, act_func='relu', dropout_rate=0, ops_order='weight_bn_act'): + super(My2DLayer, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.use_bn = use_bn + self.act_func = act_func + self.dropout_rate = dropout_rate + self.ops_order = ops_order + + """ modules """ + modules = {} + # batch norm + if self.use_bn: + if self.bn_before_weight: + modules['bn'] = nn.BatchNorm2d(in_channels) + else: + modules['bn'] = nn.BatchNorm2d(out_channels) + else: + modules['bn'] = None + # activation + modules['act'] = build_activation(self.act_func, self.ops_list[0] != 'act' and self.use_bn) + # dropout + if self.dropout_rate > 0: + modules['dropout'] = nn.Dropout2d(self.dropout_rate, inplace=True) + else: + modules['dropout'] = None + # weight + modules['weight'] = self.weight_op() + + # add modules + for op in self.ops_list: + if modules[op] is None: + continue + elif op == 'weight': + # dropout before weight operation + if modules['dropout'] is not None: + self.add_module('dropout', modules['dropout']) + for key in modules['weight']: + self.add_module(key, modules['weight'][key]) + else: + self.add_module(op, modules[op]) + + @property + def ops_list(self): + return self.ops_order.split('_') + + @property + def bn_before_weight(self): + for op in self.ops_list: + if op == 'bn': + return True + elif op == 'weight': + return False + raise ValueError('Invalid ops_order: %s' % self.ops_order) + + def weight_op(self): + raise NotImplementedError + + """ Methods defined in MyModule """ + + def forward(self, x): + # similar to nn.Sequential + for module in self._modules.values(): + x = module(x) + return x + + @property + def module_str(self): + raise NotImplementedError + + @property + def config(self): + return { + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'use_bn': self.use_bn, + 'act_func': self.act_func, + 'dropout_rate': self.dropout_rate, + 'ops_order': self.ops_order, + } + + @staticmethod + def build_from_config(config): + raise NotImplementedError + + +class ConvLayer(My2DLayer): + + def __init__(self, in_channels, out_channels, + kernel_size=3, stride=1, dilation=1, groups=1, bias=False, has_shuffle=False, use_se=False, + use_bn=True, act_func='relu', dropout_rate=0, ops_order='weight_bn_act'): + # default normal 3x3_Conv with bn and relu + self.kernel_size = kernel_size + self.stride = stride + self.dilation = dilation + self.groups = groups + self.bias = bias + self.has_shuffle = has_shuffle + self.use_se = use_se + + super(ConvLayer, self).__init__(in_channels, out_channels, use_bn, act_func, dropout_rate, ops_order) + if self.use_se: + self.add_module('se', SEModule(self.out_channels)) + + def weight_op(self): + padding = get_same_padding(self.kernel_size) + if isinstance(padding, int): + padding *= self.dilation + else: + padding[0] *= self.dilation + padding[1] *= self.dilation + + weight_dict = OrderedDict({ + 'conv': nn.Conv2d( + self.in_channels, self.out_channels, kernel_size=self.kernel_size, stride=self.stride, padding=padding, + dilation=self.dilation, groups=min_divisible_value(self.in_channels, self.groups), bias=self.bias + ) + }) + if self.has_shuffle and self.groups > 1: + weight_dict['shuffle'] = ShuffleLayer(self.groups) + + return weight_dict + + @property + def module_str(self): + if isinstance(self.kernel_size, int): + kernel_size = (self.kernel_size, self.kernel_size) + else: + kernel_size = self.kernel_size + if self.groups == 1: + if self.dilation > 1: + conv_str = '%dx%d_DilatedConv' % (kernel_size[0], kernel_size[1]) + else: + conv_str = '%dx%d_Conv' % (kernel_size[0], kernel_size[1]) + else: + if self.dilation > 1: + conv_str = '%dx%d_DilatedGroupConv' % (kernel_size[0], kernel_size[1]) + else: + conv_str = '%dx%d_GroupConv' % (kernel_size[0], kernel_size[1]) + conv_str += '_O%d' % self.out_channels + if self.use_se: + conv_str = 'SE_' + conv_str + conv_str += '_' + self.act_func.upper() + if self.use_bn: + if isinstance(self.bn, nn.GroupNorm): + conv_str += '_GN%d' % self.bn.num_groups + elif isinstance(self.bn, nn.BatchNorm2d): + conv_str += '_BN' + return conv_str + + @property + def config(self): + return { + 'name': ConvLayer.__name__, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'dilation': self.dilation, + 'groups': self.groups, + 'bias': self.bias, + 'has_shuffle': self.has_shuffle, + 'use_se': self.use_se, + **super(ConvLayer, self).config + } + + @staticmethod + def build_from_config(config): + return ConvLayer(**config) + + +class IdentityLayer(My2DLayer): + + def __init__(self, in_channels, out_channels, + use_bn=False, act_func=None, dropout_rate=0, ops_order='weight_bn_act'): + super(IdentityLayer, self).__init__(in_channels, out_channels, use_bn, act_func, dropout_rate, ops_order) + + def weight_op(self): + return None + + @property + def module_str(self): + return 'Identity' + + @property + def config(self): + return { + 'name': IdentityLayer.__name__, + **super(IdentityLayer, self).config, + } + + @staticmethod + def build_from_config(config): + return IdentityLayer(**config) + + +class LinearLayer(MyModule): + + def __init__(self, in_features, out_features, bias=True, + use_bn=False, act_func=None, dropout_rate=0, ops_order='weight_bn_act'): + super(LinearLayer, self).__init__() + + self.in_features = in_features + self.out_features = out_features + self.bias = bias + + self.use_bn = use_bn + self.act_func = act_func + self.dropout_rate = dropout_rate + self.ops_order = ops_order + + """ modules """ + modules = {} + # batch norm + if self.use_bn: + if self.bn_before_weight: + modules['bn'] = nn.BatchNorm1d(in_features) + else: + modules['bn'] = nn.BatchNorm1d(out_features) + else: + modules['bn'] = None + # activation + modules['act'] = build_activation(self.act_func, self.ops_list[0] != 'act') + # dropout + if self.dropout_rate > 0: + modules['dropout'] = nn.Dropout(self.dropout_rate, inplace=True) + else: + modules['dropout'] = None + # linear + modules['weight'] = {'linear': nn.Linear(self.in_features, self.out_features, self.bias)} + + # add modules + for op in self.ops_list: + if modules[op] is None: + continue + elif op == 'weight': + if modules['dropout'] is not None: + self.add_module('dropout', modules['dropout']) + for key in modules['weight']: + self.add_module(key, modules['weight'][key]) + else: + self.add_module(op, modules[op]) + + @property + def ops_list(self): + return self.ops_order.split('_') + + @property + def bn_before_weight(self): + for op in self.ops_list: + if op == 'bn': + return True + elif op == 'weight': + return False + raise ValueError('Invalid ops_order: %s' % self.ops_order) + + def forward(self, x): + for module in self._modules.values(): + x = module(x) + return x + + @property + def module_str(self): + return '%dx%d_Linear' % (self.in_features, self.out_features) + + @property + def config(self): + return { + 'name': LinearLayer.__name__, + 'in_features': self.in_features, + 'out_features': self.out_features, + 'bias': self.bias, + 'use_bn': self.use_bn, + 'act_func': self.act_func, + 'dropout_rate': self.dropout_rate, + 'ops_order': self.ops_order, + } + + @staticmethod + def build_from_config(config): + return LinearLayer(**config) + + +class MultiHeadLinearLayer(MyModule): + + def __init__(self, in_features, out_features, num_heads=1, bias=True, dropout_rate=0): + super(MultiHeadLinearLayer, self).__init__() + self.in_features = in_features + self.out_features = out_features + self.num_heads = num_heads + + self.bias = bias + self.dropout_rate = dropout_rate + + if self.dropout_rate > 0: + self.dropout = nn.Dropout(self.dropout_rate, inplace=True) + else: + self.dropout = None + + self.layers = nn.ModuleList() + for k in range(num_heads): + layer = nn.Linear(in_features, out_features, self.bias) + self.layers.append(layer) + + def forward(self, inputs): + if self.dropout is not None: + inputs = self.dropout(inputs) + + outputs = [] + for layer in self.layers: + output = layer.forward(inputs) + outputs.append(output) + + outputs = torch.stack(outputs, dim=1) + return outputs + + @property + def module_str(self): + return self.__repr__() + + @property + def config(self): + return { + 'name': MultiHeadLinearLayer.__name__, + 'in_features': self.in_features, + 'out_features': self.out_features, + 'num_heads': self.num_heads, + 'bias': self.bias, + 'dropout_rate': self.dropout_rate, + } + + @staticmethod + def build_from_config(config): + return MultiHeadLinearLayer(**config) + + def __repr__(self): + return 'MultiHeadLinear(in_features=%d, out_features=%d, num_heads=%d, bias=%s, dropout_rate=%s)' % ( + self.in_features, self.out_features, self.num_heads, self.bias, self.dropout_rate + ) + + +class ZeroLayer(MyModule): + + def __init__(self): + super(ZeroLayer, self).__init__() + + def forward(self, x): + raise ValueError + + @property + def module_str(self): + return 'Zero' + + @property + def config(self): + return { + 'name': ZeroLayer.__name__, + } + + @staticmethod + def build_from_config(config): + return ZeroLayer() + + +class MBConvLayer(MyModule): + + def __init__(self, in_channels, out_channels, + kernel_size=3, stride=1, expand_ratio=6, mid_channels=None, act_func='relu6', use_se=False, + groups=None): + super(MBConvLayer, self).__init__() + + self.in_channels = in_channels + self.out_channels = out_channels + + self.kernel_size = kernel_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + self.act_func = act_func + self.use_se = use_se + self.groups = groups + + if self.mid_channels is None: + feature_dim = round(self.in_channels * self.expand_ratio) + else: + feature_dim = self.mid_channels + + if self.expand_ratio == 1: + self.inverted_bottleneck = None + else: + self.inverted_bottleneck = nn.Sequential(OrderedDict([ + ('conv', nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', build_activation(self.act_func, inplace=True)), + ])) + + pad = get_same_padding(self.kernel_size) + groups = feature_dim if self.groups is None else min_divisible_value(feature_dim, self.groups) + depth_conv_modules = [ + ('conv', nn.Conv2d(feature_dim, feature_dim, kernel_size, stride, pad, groups=groups, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', build_activation(self.act_func, inplace=True)) + ] + if self.use_se: + depth_conv_modules.append(('se', SEModule(feature_dim))) + self.depth_conv = nn.Sequential(OrderedDict(depth_conv_modules)) + + self.point_linear = nn.Sequential(OrderedDict([ + ('conv', nn.Conv2d(feature_dim, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + + def forward(self, x): + if self.inverted_bottleneck: + x = self.inverted_bottleneck(x) + x = self.depth_conv(x) + x = self.point_linear(x) + return x + + @property + def module_str(self): + if self.mid_channels is None: + expand_ratio = self.expand_ratio + else: + expand_ratio = self.mid_channels // self.in_channels + layer_str = '%dx%d_MBConv%d_%s' % (self.kernel_size, self.kernel_size, expand_ratio, self.act_func.upper()) + if self.use_se: + layer_str = 'SE_' + layer_str + layer_str += '_O%d' % self.out_channels + if self.groups is not None: + layer_str += '_G%d' % self.groups + if isinstance(self.point_linear.bn, nn.GroupNorm): + layer_str += '_GN%d' % self.point_linear.bn.num_groups + elif isinstance(self.point_linear.bn, nn.BatchNorm2d): + layer_str += '_BN' + + return layer_str + + @property + def config(self): + return { + 'name': MBConvLayer.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + 'act_func': self.act_func, + 'use_se': self.use_se, + 'groups': self.groups, + } + + @staticmethod + def build_from_config(config): + return MBConvLayer(**config) + + +class ResidualBlock(MyModule): + + def __init__(self, conv, shortcut, dropout_rate, dropblock, block_size): + super(ResidualBlock, self).__init__() + + self.conv = conv + self.shortcut = shortcut + # hayeon + self.num_batches_tracked = 0 + self.dropout_rate = dropout_rate + self.dropblock = dropblock + self.block_size = block_size + self.DropBlock = DropBlock(block_size=self.block_size) + + def forward(self, x): + # hayeon + self.num_batches_tracked += 1 + + if self.conv is None or isinstance(self.conv, ZeroLayer): + res = x + elif self.shortcut is None or isinstance(self.shortcut, ZeroLayer): + res = self.conv(x) + else: + res = self.conv(x) + self.shortcut(x) + + # hayeon + if self.dropout_rate > 0: + if self.dropblock: + feat_size = res.size()[2] + keep_rate = max(1.0 - self.dropout_rate / (20*2000) * (self.num_batches_tracked), 1.0 - self.drop_rate) + gamma = (1 - keep_rate) / self.block_size**2 * feat_size**2 / (feat_size - self.block_size + 1)**2 + res = self.DropBlock(res, gamma=gamma) + else: + res = F.dropout(res, p=self.dropout_rate, training=self.training, inplace=True) + return res + + @property + def module_str(self): + return '(%s, %s)' % ( + self.conv.module_str if self.conv is not None else None, + self.shortcut.module_str if self.shortcut is not None else None + ) + + @property + def config(self): + return { + 'name': ResidualBlock.__name__, + 'conv': self.conv.config if self.conv is not None else None, + 'shortcut': self.shortcut.config if self.shortcut is not None else None, + } + + @staticmethod + def build_from_config(config): + conv_config = config['conv'] if 'conv' in config else config['mobile_inverted_conv'] + conv = set_layer_from_config(conv_config) + shortcut = set_layer_from_config(config['shortcut']) + return ResidualBlock(conv, shortcut) + + @property + def mobile_inverted_conv(self): + return self.conv + + +class ResNetBottleneckBlock(MyModule): + + def __init__(self, in_channels, out_channels, + kernel_size=3, stride=1, expand_ratio=0.25, mid_channels=None, act_func='relu', groups=1, + downsample_mode='avgpool_conv'): + super(ResNetBottleneckBlock, self).__init__() + + self.in_channels = in_channels + self.out_channels = out_channels + + self.kernel_size = kernel_size + self.stride = stride + self.expand_ratio = expand_ratio + self.mid_channels = mid_channels + self.act_func = act_func + self.groups = groups + + self.downsample_mode = downsample_mode + + if self.mid_channels is None: + feature_dim = round(self.out_channels * self.expand_ratio) + else: + feature_dim = self.mid_channels + + feature_dim = make_divisible(feature_dim, MyNetwork.CHANNEL_DIVISIBLE) + self.mid_channels = feature_dim + + # build modules + self.conv1 = nn.Sequential(OrderedDict([ + ('conv', nn.Conv2d(self.in_channels, feature_dim, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', build_activation(self.act_func, inplace=True)), + ])) + + pad = get_same_padding(self.kernel_size) + self.conv2 = nn.Sequential(OrderedDict([ + ('conv', nn.Conv2d(feature_dim, feature_dim, kernel_size, stride, pad, groups=groups, bias=False)), + ('bn', nn.BatchNorm2d(feature_dim)), + ('act', build_activation(self.act_func, inplace=True)) + ])) + + self.conv3 = nn.Sequential(OrderedDict([ + ('conv', nn.Conv2d(feature_dim, self.out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(self.out_channels)), + ])) + + if stride == 1 and in_channels == out_channels: + self.downsample = IdentityLayer(in_channels, out_channels) + elif self.downsample_mode == 'conv': + self.downsample = nn.Sequential(OrderedDict([ + ('conv', nn.Conv2d(in_channels, out_channels, 1, stride, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + elif self.downsample_mode == 'avgpool_conv': + self.downsample = nn.Sequential(OrderedDict([ + ('avg_pool', nn.AvgPool2d(kernel_size=stride, stride=stride, padding=0, ceil_mode=True)), + ('conv', nn.Conv2d(in_channels, out_channels, 1, 1, 0, bias=False)), + ('bn', nn.BatchNorm2d(out_channels)), + ])) + else: + raise NotImplementedError + + self.final_act = build_activation(self.act_func, inplace=True) + + def forward(self, x): + residual = self.downsample(x) + + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + + x = x + residual + x = self.final_act(x) + return x + + @property + def module_str(self): + return '(%s, %s)' % ( + '%dx%d_BottleneckConv_%d->%d->%d_S%d_G%d' % ( + self.kernel_size, self.kernel_size, self.in_channels, self.mid_channels, self.out_channels, + self.stride, self.groups + ), + 'Identity' if isinstance(self.downsample, IdentityLayer) else self.downsample_mode, + ) + + @property + def config(self): + return { + 'name': ResNetBottleneckBlock.__name__, + 'in_channels': self.in_channels, + 'out_channels': self.out_channels, + 'kernel_size': self.kernel_size, + 'stride': self.stride, + 'expand_ratio': self.expand_ratio, + 'mid_channels': self.mid_channels, + 'act_func': self.act_func, + 'groups': self.groups, + 'downsample_mode': self.downsample_mode, + } + + @staticmethod + def build_from_config(config): + return ResNetBottleneckBlock(**config) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/__init__.py new file mode 100644 index 0000000..c3b06c3 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/__init__.py @@ -0,0 +1,4 @@ +from .my_data_loader import * +from .my_data_worker import * +from .my_distributed_sampler import * +from .my_random_resize_crop import * diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_data_loader.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_data_loader.py new file mode 100644 index 0000000..f5af640 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_data_loader.py @@ -0,0 +1,962 @@ +r"""Definition of the DataLoader and associated iterators that subclass _BaseDataLoaderIter + +To support these two classes, in `./_utils` we define many utility methods and +functions to be run in multiprocessing. E.g., the data loading worker loop is +in `./_utils/worker.py`. +""" + +import threading +import itertools +import warnings +import multiprocessing as python_multiprocessing +import torch +import torch.multiprocessing as multiprocessing +from torch._utils import ExceptionWrapper +from torch._six import queue, string_classes +from torch.utils.data.dataset import IterableDataset +from torch.utils.data import Sampler, SequentialSampler, RandomSampler, BatchSampler +from torch.utils.data import _utils + +from .my_data_worker import worker_loop + +__all__ = ['MyDataLoader'] + +get_worker_info = _utils.worker.get_worker_info + +# This function used to be defined in this file. However, it was moved to +# _utils/collate.py. Although it is rather hard to access this from user land +# (one has to explicitly directly `import torch.utils.data.dataloader`), there +# probably is user code out there using it. This aliasing maintains BC in this +# aspect. +default_collate = _utils.collate.default_collate + + +class _DatasetKind(object): + Map = 0 + Iterable = 1 + + @staticmethod + def create_fetcher(kind, dataset, auto_collation, collate_fn, drop_last): + if kind == _DatasetKind.Map: + return _utils.fetch._MapDatasetFetcher(dataset, auto_collation, collate_fn, drop_last) + else: + return _utils.fetch._IterableDatasetFetcher(dataset, auto_collation, collate_fn, drop_last) + + +class _InfiniteConstantSampler(Sampler): + r"""Analogous to ``itertools.repeat(None, None)``. + Used as sampler for :class:`~torch.utils.data.IterableDataset`. + + Arguments: + data_source (Dataset): dataset to sample from + """ + + def __init__(self): + super(_InfiniteConstantSampler, self).__init__(None) + + def __iter__(self): + while True: + yield None + + +class MyDataLoader(object): + r""" + Data loader. Combines a dataset and a sampler, and provides an iterable over + the given dataset. + + The :class:`~torch.utils.data.DataLoader` supports both map-style and + iterable-style datasets with single- or multi-process loading, customizing + loading order and optional automatic batching (collation) and memory pinning. + + See :py:mod:`torch.utils.data` documentation page for more details. + + Arguments: + dataset (Dataset): dataset from which to load the data. + batch_size (int, optional): how many samples per batch to load + (default: ``1``). + shuffle (bool, optional): set to ``True`` to have the data reshuffled + at every epoch (default: ``False``). + sampler (Sampler, optional): defines the strategy to draw samples from + the dataset. If specified, :attr:`shuffle` must be ``False``. + batch_sampler (Sampler, optional): like :attr:`sampler`, but returns a batch of + indices at a time. Mutually exclusive with :attr:`batch_size`, + :attr:`shuffle`, :attr:`sampler`, and :attr:`drop_last`. + num_workers (int, optional): how many subprocesses to use for data + loading. ``0`` means that the data will be loaded in the main process. + (default: ``0``) + collate_fn (callable, optional): merges a list of samples to form a + mini-batch of Tensor(s). Used when using batched loading from a + map-style dataset. + pin_memory (bool, optional): If ``True``, the data loader will copy Tensors + into CUDA pinned memory before returning them. If your data elements + are a custom type, or your :attr:`collate_fn` returns a batch that is a custom type, + see the example below. + drop_last (bool, optional): set to ``True`` to drop the last incomplete batch, + if the dataset size is not divisible by the batch size. If ``False`` and + the size of dataset is not divisible by the batch size, then the last batch + will be smaller. (default: ``False``) + timeout (numeric, optional): if positive, the timeout value for collecting a batch + from workers. Should always be non-negative. (default: ``0``) + worker_init_fn (callable, optional): If not ``None``, this will be called on each + worker subprocess with the worker id (an int in ``[0, num_workers - 1]``) as + input, after seeding and before data loading. (default: ``None``) + + + .. warning:: If the ``spawn`` start method is used, :attr:`worker_init_fn` + cannot be an unpicklable object, e.g., a lambda function. See + :ref:`multiprocessing-best-practices` on more details related + to multiprocessing in PyTorch. + + .. note:: ``len(dataloader)`` heuristic is based on the length of the sampler used. + When :attr:`dataset` is an :class:`~torch.utils.data.IterableDataset`, + ``len(dataset)`` (if implemented) is returned instead, regardless + of multi-process loading configurations, because PyTorch trust + user :attr:`dataset` code in correctly handling multi-process + loading to avoid duplicate data. See `Dataset Types`_ for more + details on these two types of datasets and how + :class:`~torch.utils.data.IterableDataset` interacts with `Multi-process data loading`_. + """ + + __initialized = False + + def __init__(self, dataset, batch_size=1, shuffle=False, sampler=None, + batch_sampler=None, num_workers=0, collate_fn=None, + pin_memory=False, drop_last=False, timeout=0, + worker_init_fn=None, multiprocessing_context=None): + torch._C._log_api_usage_once("python.data_loader") + + if num_workers < 0: + raise ValueError('num_workers option should be non-negative; ' + 'use num_workers=0 to disable multiprocessing.') + + if timeout < 0: + raise ValueError('timeout option should be non-negative') + + self.dataset = dataset + self.num_workers = num_workers + self.pin_memory = pin_memory + self.timeout = timeout + self.worker_init_fn = worker_init_fn + self.multiprocessing_context = multiprocessing_context + + # Arg-check dataset related before checking samplers because we want to + # tell users that iterable-style datasets are incompatible with custom + # samplers first, so that they don't learn that this combo doesn't work + # after spending time fixing the custom sampler errors. + if isinstance(dataset, IterableDataset): + self._dataset_kind = _DatasetKind.Iterable + # NOTE [ Custom Samplers and `IterableDataset` ] + # + # `IterableDataset` does not support custom `batch_sampler` or + # `sampler` since the key is irrelevant (unless we support + # generator-style dataset one day...). + # + # For `sampler`, we always create a dummy sampler. This is an + # infinite sampler even when the dataset may have an implemented + # finite `__len__` because in multi-process data loading, naive + # settings will return duplicated data (which may be desired), and + # thus using a sampler with length matching that of dataset will + # cause data lost (you may have duplicates of the first couple + # batches, but never see anything afterwards). Therefore, + # `Iterabledataset` always uses an infinite sampler, an instance of + # `_InfiniteConstantSampler` defined above. + # + # A custom `batch_sampler` essentially only controls the batch size. + # However, it is unclear how useful it would be since an iterable-style + # dataset can handle that within itself. Moreover, it is pointless + # in multi-process data loading as the assignment order of batches + # to workers is an implementation detail so users can not control + # how to batchify each worker's iterable. Thus, we disable this + # option. If this turns out to be useful in future, we can re-enable + # this, and support custom samplers that specify the assignments to + # specific workers. + if shuffle is not False: + raise ValueError( + "DataLoader with IterableDataset: expected unspecified " + "shuffle option, but got shuffle={}".format(shuffle)) + elif sampler is not None: + # See NOTE [ Custom Samplers and IterableDataset ] + raise ValueError( + "DataLoader with IterableDataset: expected unspecified " + "sampler option, but got sampler={}".format(sampler)) + elif batch_sampler is not None: + # See NOTE [ Custom Samplers and IterableDataset ] + raise ValueError( + "DataLoader with IterableDataset: expected unspecified " + "batch_sampler option, but got batch_sampler={}".format(batch_sampler)) + else: + self._dataset_kind = _DatasetKind.Map + + if sampler is not None and shuffle: + raise ValueError('sampler option is mutually exclusive with ' + 'shuffle') + + if batch_sampler is not None: + # auto_collation with custom batch_sampler + if batch_size != 1 or shuffle or sampler is not None or drop_last: + raise ValueError('batch_sampler option is mutually exclusive ' + 'with batch_size, shuffle, sampler, and ' + 'drop_last') + batch_size = None + drop_last = False + elif batch_size is None: + # no auto_collation + if shuffle or drop_last: + raise ValueError('batch_size=None option disables auto-batching ' + 'and is mutually exclusive with ' + 'shuffle, and drop_last') + + if sampler is None: # give default samplers + if self._dataset_kind == _DatasetKind.Iterable: + # See NOTE [ Custom Samplers and IterableDataset ] + sampler = _InfiniteConstantSampler() + else: # map-style + if shuffle: + sampler = RandomSampler(dataset) + else: + sampler = SequentialSampler(dataset) + + if batch_size is not None and batch_sampler is None: + # auto_collation without custom batch_sampler + batch_sampler = BatchSampler(sampler, batch_size, drop_last) + + self.batch_size = batch_size + self.drop_last = drop_last + self.sampler = sampler + self.batch_sampler = batch_sampler + + if collate_fn is None: + if self._auto_collation: + collate_fn = _utils.collate.default_collate + else: + collate_fn = _utils.collate.default_convert + + self.collate_fn = collate_fn + self.__initialized = True + self._IterableDataset_len_called = None # See NOTE [ IterableDataset and __len__ ] + + @property + def multiprocessing_context(self): + return self.__multiprocessing_context + + @multiprocessing_context.setter + def multiprocessing_context(self, multiprocessing_context): + if multiprocessing_context is not None: + if self.num_workers > 0: + if not multiprocessing._supports_context: + raise ValueError('multiprocessing_context relies on Python >= 3.4, with ' + 'support for different start methods') + + if isinstance(multiprocessing_context, string_classes): + valid_start_methods = multiprocessing.get_all_start_methods() + if multiprocessing_context not in valid_start_methods: + raise ValueError( + ('multiprocessing_context option ' + 'should specify a valid start method in {}, but got ' + 'multiprocessing_context={}').format(valid_start_methods, multiprocessing_context)) + multiprocessing_context = multiprocessing.get_context(multiprocessing_context) + + if not isinstance(multiprocessing_context, python_multiprocessing.context.BaseContext): + raise ValueError(('multiprocessing_context option should be a valid context ' + 'object or a string specifying the start method, but got ' + 'multiprocessing_context={}').format(multiprocessing_context)) + else: + raise ValueError(('multiprocessing_context can only be used with ' + 'multi-process loading (num_workers > 0), but got ' + 'num_workers={}').format(self.num_workers)) + + self.__multiprocessing_context = multiprocessing_context + + def __setattr__(self, attr, val): + if self.__initialized and attr in ('batch_size', 'batch_sampler', 'sampler', 'drop_last', 'dataset'): + raise ValueError('{} attribute should not be set after {} is ' + 'initialized'.format(attr, self.__class__.__name__)) + + super(MyDataLoader, self).__setattr__(attr, val) + + def __iter__(self): + if self.num_workers == 0: + return _SingleProcessDataLoaderIter(self) + else: + return _MultiProcessingDataLoaderIter(self) + + @property + def _auto_collation(self): + return self.batch_sampler is not None + + @property + def _index_sampler(self): + # The actual sampler used for generating indices for `_DatasetFetcher` + # (see _utils/fetch.py) to read data at each time. This would be + # `.batch_sampler` if in auto-collation mode, and `.sampler` otherwise. + # We can't change `.sampler` and `.batch_sampler` attributes for BC + # reasons. + if self._auto_collation: + return self.batch_sampler + else: + return self.sampler + + def __len__(self): + if self._dataset_kind == _DatasetKind.Iterable: + # NOTE [ IterableDataset and __len__ ] + # + # For `IterableDataset`, `__len__` could be inaccurate when one naively + # does multi-processing data loading, since the samples will be duplicated. + # However, no real use case should be actually using that behavior, so + # it should count as a user error. We should generally trust user + # code to do the proper thing (e.g., configure each replica differently + # in `__iter__`), and give us the correct `__len__` if they choose to + # implement it (this will still throw if the dataset does not implement + # a `__len__`). + # + # To provide a further warning, we track if `__len__` was called on the + # `DataLoader`, save the returned value in `self._len_called`, and warn + # if the iterator ends up yielding more than this number of samples. + length = self._IterableDataset_len_called = len(self.dataset) + return length + else: + return len(self._index_sampler) + + +class _BaseDataLoaderIter(object): + def __init__(self, loader): + self._dataset = loader.dataset + self._dataset_kind = loader._dataset_kind + self._IterableDataset_len_called = loader._IterableDataset_len_called + self._auto_collation = loader._auto_collation + self._drop_last = loader.drop_last + self._index_sampler = loader._index_sampler + self._num_workers = loader.num_workers + self._pin_memory = loader.pin_memory and torch.cuda.is_available() + self._timeout = loader.timeout + self._collate_fn = loader.collate_fn + self._sampler_iter = iter(self._index_sampler) + self._base_seed = torch.empty((), dtype=torch.int64).random_().item() + self._num_yielded = 0 + + def __iter__(self): + return self + + def _next_index(self): + return next(self._sampler_iter) # may raise StopIteration + + def _next_data(self): + raise NotImplementedError + + def __next__(self): + data = self._next_data() + self._num_yielded += 1 + if self._dataset_kind == _DatasetKind.Iterable and \ + self._IterableDataset_len_called is not None and \ + self._num_yielded > self._IterableDataset_len_called: + warn_msg = ("Length of IterableDataset {} was reported to be {} (when accessing len(dataloader)), but {} " + "samples have been fetched. ").format(self._dataset, self._IterableDataset_len_called, + self._num_yielded) + if self._num_workers > 0: + warn_msg += ("For multiprocessing data-loading, this could be caused by not properly configuring the " + "IterableDataset replica at each worker. Please see " + "https://pytorch.org/docs/stable/data.html#torch.utils.data.IterableDataset for examples.") + warnings.warn(warn_msg) + return data + + next = __next__ # Python 2 compatibility + + def __len__(self): + return len(self._index_sampler) + + def __getstate__(self): + # across multiple threads for HOGWILD. + # Probably the best way to do this is by moving the sample pushing + # to a separate thread and then just sharing the data queue + # but signalling the end is tricky without a non-blocking API + raise NotImplementedError("{} cannot be pickled", self.__class__.__name__) + + +class _SingleProcessDataLoaderIter(_BaseDataLoaderIter): + def __init__(self, loader): + super(_SingleProcessDataLoaderIter, self).__init__(loader) + assert self._timeout == 0 + assert self._num_workers == 0 + + self._dataset_fetcher = _DatasetKind.create_fetcher( + self._dataset_kind, self._dataset, self._auto_collation, self._collate_fn, self._drop_last) + + def _next_data(self): + index = self._next_index() # may raise StopIteration + data = self._dataset_fetcher.fetch(index) # may raise StopIteration + if self._pin_memory: + data = _utils.pin_memory.pin_memory(data) + return data + + +class _MultiProcessingDataLoaderIter(_BaseDataLoaderIter): + r"""Iterates once over the DataLoader's dataset, as specified by the sampler""" + + # NOTE [ Data Loader Multiprocessing Shutdown Logic ] + # + # Preliminary: + # + # Our data model looks like this (queues are indicated with curly brackets): + # + # main process || + # | || + # {index_queue} || + # | || + # worker processes || DATA + # | || + # {worker_result_queue} || FLOW + # | || + # pin_memory_thread of main process || DIRECTION + # | || + # {data_queue} || + # | || + # data output \/ + # + # P.S. `worker_result_queue` and `pin_memory_thread` part may be omitted if + # `pin_memory=False`. + # + # + # Terminating multiprocessing logic requires very careful design. In + # particular, we need to make sure that + # + # 1. The iterator gracefully exits the workers when its last reference is + # gone or it is depleted. + # + # In this case, the workers should be gracefully exited because the + # main process may still need to continue to run, and we want cleaning + # up code in the workers to be executed (e.g., releasing GPU memory). + # Naturally, we implement the shutdown logic in `__del__` of + # DataLoaderIterator. + # + # We delay the discussion on the logic in this case until later. + # + # 2. The iterator exits the workers when the loader process and/or worker + # processes exits normally or with error. + # + # We set all workers and `pin_memory_thread` to have `daemon=True`. + # + # You may ask, why can't we make the workers non-daemonic, and + # gracefully exit using the same logic as we have in `__del__` when the + # iterator gets deleted (see 1 above)? + # + # First of all, `__del__` is **not** guaranteed to be called when + # interpreter exits. Even if it is called, by the time it executes, + # many Python core library resources may alreay be freed, and even + # simple things like acquiring an internal lock of a queue may hang. + # Therefore, in this case, we actually need to prevent `__del__` from + # being executed, and rely on the automatic termination of daemonic + # children. Thus, we register an `atexit` hook that sets a global flag + # `_utils.python_exit_status`. Since `atexit` hooks are executed in the + # reverse order of registration, we are guaranteed that this flag is + # set before library resources we use are freed. (Hooks freeing those + # resources are registered at importing the Python core libraries at + # the top of this file.) So in `__del__`, we check if + # `_utils.python_exit_status` is set or `None` (freed), and perform + # no-op if so. + # + # Another problem with `__del__` is also related to the library cleanup + # calls. When a process ends, it shuts the all its daemonic children + # down with a SIGTERM (instead of joining them without a timeout). + # Simiarly for threads, but by a different mechanism. This fact, + # together with a few implementation details of multiprocessing, forces + # us to make workers daemonic. All of our problems arise when a + # DataLoader is used in a subprocess, and are caused by multiprocessing + # code which looks more or less like this: + # + # try: + # your_function_using_a_dataloader() + # finally: + # multiprocessing.util._exit_function() + # + # The joining/termination mentioned above happens inside + # `_exit_function()`. Now, if `your_function_using_a_dataloader()` + # throws, the stack trace stored in the exception will prevent the + # frame which uses `DataLoaderIter` to be freed. If the frame has any + # reference to the `DataLoaderIter` (e.g., in a method of the iter), + # its `__del__`, which starts the shutdown procedure, will not be + # called. That, in turn, means that workers aren't notified. Attempting + # to join in `_exit_function` will then result in a hang. + # + # For context, `_exit_function` is also registered as an `atexit` call. + # So it is unclear to me (@ssnl) why this is needed in a finally block. + # The code dates back to 2008 and there is no comment on the original + # PEP 371 or patch https://bugs.python.org/issue3050 (containing both + # the finally block and the `atexit` registration) that explains this. + # + # Another choice is to just shutdown workers with logic in 1 above + # whenever we see an error in `next`. This isn't ideal because + # a. It prevents users from using try-catch to resume data loading. + # b. It doesn't prevent hanging if users have references to the + # iterator. + # + # 3. All processes exit if any of them die unexpectedly by fatal signals. + # + # As shown above, the workers are set as daemonic children of the main + # process. However, automatic cleaning-up of such child processes only + # happens if the parent process exits gracefully (e.g., not via fatal + # signals like SIGKILL). So we must ensure that each process will exit + # even the process that should send/receive data to/from it were + # killed, i.e., + # + # a. A process won't hang when getting from a queue. + # + # Even with carefully designed data dependencies (i.e., a `put()` + # always corresponding to a `get()`), hanging on `get()` can still + # happen when data in queue is corrupted (e.g., due to + # `cancel_join_thread` or unexpected exit). + # + # For child exit, we set a timeout whenever we try to get data + # from `data_queue`, and check the workers' status on each timeout + # and error. + # See `_DataLoaderiter._get_batch()` and + # `_DataLoaderiter._try_get_data()` for details. + # + # Additionally, for child exit on non-Windows platforms, we also + # register a SIGCHLD handler (which is supported on Windows) on + # the main process, which checks if any of the workers fail in the + # (Python) handler. This is more efficient and faster in detecting + # worker failures, compared to only using the above mechanism. + # See `DataLoader.cpp` and `_utils/signal_handling.py` for details. + # + # For `.get()` calls where the sender(s) is not the workers, we + # guard them with timeouts, and check the status of the sender + # when timeout happens: + # + in the workers, the `_utils.worker.ManagerWatchdog` class + # checks the status of the main process. + # + if `pin_memory=True`, when getting from `pin_memory_thread`, + # check `pin_memory_thread` status periodically until `.get()` + # returns or see that `pin_memory_thread` died. + # + # b. A process won't hang when putting into a queue; + # + # We use `mp.Queue` which has a separate background thread to put + # objects from an unbounded buffer array. The background thread is + # daemonic and usually automatically joined when the process + # exits. + # + # However, in case that the receiver has ended abruptly while + # reading from the pipe, the join will hang forever. Therefore, + # for both `worker_result_queue` (worker -> main process/pin_memory_thread) + # and each `index_queue` (main process -> worker), we use + # `q.cancel_join_thread()` in sender process before any `q.put` to + # prevent this automatic join. + # + # Moreover, having all queues called `cancel_join_thread` makes + # implementing graceful shutdown logic in `__del__` much easier. + # It won't need to get from any queue, which would also need to be + # guarded by periodic status checks. + # + # Nonetheless, `cancel_join_thread` must only be called when the + # queue is **not** going to be read from or write into by another + # process, because it may hold onto a lock or leave corrupted data + # in the queue, leading other readers/writers to hang. + # + # `pin_memory_thread`'s `data_queue` is a `queue.Queue` that does + # a blocking `put` if the queue is full. So there is no above + # problem, but we do need to wrap the `put` in a loop that breaks + # not only upon success, but also when the main process stops + # reading, i.e., is shutting down. + # + # + # Now let's get back to 1: + # how we gracefully exit the workers when the last reference to the + # iterator is gone. + # + # To achieve this, we implement the following logic along with the design + # choices mentioned above: + # + # `workers_done_event`: + # A `multiprocessing.Event` shared among the main process and all worker + # processes. This is used to signal the workers that the iterator is + # shutting down. After it is set, they will not send processed data to + # queues anymore, and only wait for the final `None` before exiting. + # `done_event` isn't strictly needed. I.e., we can just check for `None` + # from the input queue, but it allows us to skip wasting resources + # processing data if we are already shutting down. + # + # `pin_memory_thread_done_event`: + # A `threading.Event` for a similar purpose to that of + # `workers_done_event`, but is for the `pin_memory_thread`. The reason + # that separate events are needed is that `pin_memory_thread` reads from + # the output queue of the workers. But the workers, upon seeing that + # `workers_done_event` is set, only wants to see the final `None`, and is + # not required to flush all data in the output queue (e.g., it may call + # `cancel_join_thread` on that queue if its `IterableDataset` iterator + # happens to exhaust coincidentally, which is out of the control of the + # main process). Thus, since we will exit `pin_memory_thread` before the + # workers (see below), two separete events are used. + # + # NOTE: In short, the protocol is that the main process will set these + # `done_event`s and then the corresponding processes/threads a `None`, + # and that they may exit at any time after receiving the `None`. + # + # NOTE: Using `None` as the final signal is valid, since normal data will + # always be a 2-tuple with the 1st element being the index of the data + # transferred (different from dataset index/key), and the 2nd being + # either the dataset key or the data sample (depending on which part + # of the data model the queue is at). + # + # [ worker processes ] + # While loader process is alive: + # Get from `index_queue`. + # If get anything else, + # Check `workers_done_event`. + # If set, continue to next iteration + # i.e., keep getting until see the `None`, then exit. + # Otherwise, process data: + # If is fetching from an `IterableDataset` and the iterator + # is exhausted, send an `_IterableDatasetStopIteration` + # object to signal iteration end. The main process, upon + # receiving such an object, will send `None` to this + # worker and not use the corresponding `index_queue` + # anymore. + # If timed out, + # No matter `workers_done_event` is set (still need to see `None`) + # or not, must continue to next iteration. + # (outside loop) + # If `workers_done_event` is set, (this can be False with `IterableDataset`) + # `data_queue.cancel_join_thread()`. (Everything is ending here: + # main process won't read from it; + # other workers will also call + # `cancel_join_thread`.) + # + # [ pin_memory_thread ] + # # No need to check main thread. If this thread is alive, the main loader + # # thread must be alive, because this thread is set as daemonic. + # While `pin_memory_thread_done_event` is not set: + # Get from `index_queue`. + # If timed out, continue to get in the next iteration. + # Otherwise, process data. + # While `pin_memory_thread_done_event` is not set: + # Put processed data to `data_queue` (a `queue.Queue` with blocking put) + # If timed out, continue to put in the next iteration. + # Otherwise, break, i.e., continuing to the out loop. + # + # NOTE: we don't check the status of the main thread because + # 1. if the process is killed by fatal signal, `pin_memory_thread` + # ends. + # 2. in other cases, either the cleaning-up in __del__ or the + # automatic exit of daemonic thread will take care of it. + # This won't busy-wait either because `.get(timeout)` does not + # busy-wait. + # + # [ main process ] + # In the DataLoader Iter's `__del__` + # b. Exit `pin_memory_thread` + # i. Set `pin_memory_thread_done_event`. + # ii Put `None` in `worker_result_queue`. + # iii. Join the `pin_memory_thread`. + # iv. `worker_result_queue.cancel_join_thread()`. + # + # c. Exit the workers. + # i. Set `workers_done_event`. + # ii. Put `None` in each worker's `index_queue`. + # iii. Join the workers. + # iv. Call `.cancel_join_thread()` on each worker's `index_queue`. + # + # NOTE: (c) is better placed after (b) because it may leave corrupted + # data in `worker_result_queue`, which `pin_memory_thread` + # reads from, in which case the `pin_memory_thread` can only + # happen at timeing out, which is slow. Nonetheless, same thing + # happens if a worker is killed by signal at unfortunate times, + # but in other cases, we are better off having a non-corrupted + # `worker_result_queue` for `pin_memory_thread`. + # + # NOTE: If `pin_memory=False`, there is no `pin_memory_thread` and (b) + # can be omitted + # + # NB: `done_event`s isn't strictly needed. E.g., we can just check for + # `None` from `index_queue`, but it allows us to skip wasting resources + # processing indices already in `index_queue` if we are already shutting + # down. + + def __init__(self, loader): + super(_MultiProcessingDataLoaderIter, self).__init__(loader) + + assert self._num_workers > 0 + + if loader.multiprocessing_context is None: + multiprocessing_context = multiprocessing + else: + multiprocessing_context = loader.multiprocessing_context + + self._worker_init_fn = loader.worker_init_fn + self._worker_queue_idx_cycle = itertools.cycle(range(self._num_workers)) + self._worker_result_queue = multiprocessing_context.Queue() + self._worker_pids_set = False + self._shutdown = False + self._send_idx = 0 # idx of the next task to be sent to workers + self._rcvd_idx = 0 # idx of the next task to be returned in __next__ + # information about data not yet yielded, i.e., tasks w/ indices in range [rcvd_idx, send_idx). + # map: task idx => - (worker_id,) if data isn't fetched (outstanding) + # \ (worker_id, data) if data is already fetched (out-of-order) + self._task_info = {} + self._tasks_outstanding = 0 # always equal to count(v for v in task_info.values() if len(v) == 1) + self._workers_done_event = multiprocessing_context.Event() + + self._index_queues = [] + self._workers = [] + # A list of booleans representing whether each worker still has work to + # do, i.e., not having exhausted its iterable dataset object. It always + # contains all `True`s if not using an iterable-style dataset + # (i.e., if kind != Iterable). + self._workers_status = [] + for i in range(self._num_workers): + index_queue = multiprocessing_context.Queue() + # index_queue.cancel_join_thread() + w = multiprocessing_context.Process( + target=worker_loop, + args=(self._dataset_kind, self._dataset, index_queue, + self._worker_result_queue, self._workers_done_event, + self._auto_collation, self._collate_fn, self._drop_last, + self._base_seed + i, self._worker_init_fn, i, self._num_workers)) + w.daemon = True + # NB: Process.start() actually take some time as it needs to + # start a process and pass the arguments over via a pipe. + # Therefore, we only add a worker to self._workers list after + # it started, so that we do not call .join() if program dies + # before it starts, and __del__ tries to join but will get: + # AssertionError: can only join a started process. + w.start() + self._index_queues.append(index_queue) + self._workers.append(w) + self._workers_status.append(True) + + if self._pin_memory: + self._pin_memory_thread_done_event = threading.Event() + self._data_queue = queue.Queue() + pin_memory_thread = threading.Thread( + target=_utils.pin_memory._pin_memory_loop, + args=(self._worker_result_queue, self._data_queue, + torch.cuda.current_device(), + self._pin_memory_thread_done_event)) + pin_memory_thread.daemon = True + pin_memory_thread.start() + # Similar to workers (see comment above), we only register + # pin_memory_thread once it is started. + self._pin_memory_thread = pin_memory_thread + else: + self._data_queue = self._worker_result_queue + + _utils.signal_handling._set_worker_pids(id(self), tuple(w.pid for w in self._workers)) + _utils.signal_handling._set_SIGCHLD_handler() + self._worker_pids_set = True + + # prime the prefetch loop + for _ in range(2 * self._num_workers): + self._try_put_index() + + def _try_get_data(self, timeout=_utils.MP_STATUS_CHECK_INTERVAL): + # Tries to fetch data from `self._data_queue` once for a given timeout. + # This can also be used as inner loop of fetching without timeout, with + # the sender status as the loop condition. + # + # This raises a `RuntimeError` if any worker died expectedly. This error + # can come from either the SIGCHLD handler in `_utils/signal_handling.py` + # (only for non-Windows platforms), or the manual check below on errors + # and timeouts. + # + # Returns a 2-tuple: + # (bool: whether successfully get data, any: data if successful else None) + try: + data = self._data_queue.get(timeout=timeout) + return (True, data) + except Exception as e: + # At timeout and error, we manually check whether any worker has + # failed. Note that this is the only mechanism for Windows to detect + # worker failures. + failed_workers = [] + for worker_id, w in enumerate(self._workers): + if self._workers_status[worker_id] and not w.is_alive(): + failed_workers.append(w) + self._shutdown_worker(worker_id) + if len(failed_workers) > 0: + pids_str = ', '.join(str(w.pid) for w in failed_workers) + raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) + if isinstance(e, queue.Empty): + return (False, None) + raise + + def _get_data(self): + # Fetches data from `self._data_queue`. + # + # We check workers' status every `MP_STATUS_CHECK_INTERVAL` seconds, + # which we achieve by running `self._try_get_data(timeout=MP_STATUS_CHECK_INTERVAL)` + # in a loop. This is the only mechanism to detect worker failures for + # Windows. For other platforms, a SIGCHLD handler is also used for + # worker failure detection. + # + # If `pin_memory=True`, we also need check if `pin_memory_thread` had + # died at timeouts. + if self._timeout > 0: + success, data = self._try_get_data(self._timeout) + if success: + return data + else: + raise RuntimeError('DataLoader timed out after {} seconds'.format(self._timeout)) + elif self._pin_memory: + while self._pin_memory_thread.is_alive(): + success, data = self._try_get_data() + if success: + return data + else: + # while condition is false, i.e., pin_memory_thread died. + raise RuntimeError('Pin memory thread exited unexpectedly') + # In this case, `self._data_queue` is a `queue.Queue`,. But we don't + # need to call `.task_done()` because we don't use `.join()`. + else: + while True: + success, data = self._try_get_data() + if success: + return data + + def _next_data(self): + while True: + # If the worker responsible for `self._rcvd_idx` has already ended + # and was unable to fulfill this task (due to exhausting an `IterableDataset`), + # we try to advance `self._rcvd_idx` to find the next valid index. + # + # This part needs to run in the loop because both the `self._get_data()` + # call and `_IterableDatasetStopIteration` check below can mark + # extra worker(s) as dead. + while self._rcvd_idx < self._send_idx: + info = self._task_info[self._rcvd_idx] + worker_id = info[0] + if len(info) == 2 or self._workers_status[worker_id]: # has data or is still active + break + del self._task_info[self._rcvd_idx] + self._rcvd_idx += 1 + else: + # no valid `self._rcvd_idx` is found (i.e., didn't break) + self._shutdown_workers() + raise StopIteration + + # Now `self._rcvd_idx` is the batch index we want to fetch + + # Check if the next sample has already been generated + if len(self._task_info[self._rcvd_idx]) == 2: + data = self._task_info.pop(self._rcvd_idx)[1] + return self._process_data(data) + + assert not self._shutdown and self._tasks_outstanding > 0 + idx, data = self._get_data() + self._tasks_outstanding -= 1 + + if self._dataset_kind == _DatasetKind.Iterable: + # Check for _IterableDatasetStopIteration + if isinstance(data, _utils.worker._IterableDatasetStopIteration): + self._shutdown_worker(data.worker_id) + self._try_put_index() + continue + + if idx != self._rcvd_idx: + # store out-of-order samples + self._task_info[idx] += (data,) + else: + del self._task_info[idx] + return self._process_data(data) + + def _try_put_index(self): + assert self._tasks_outstanding < 2 * self._num_workers + try: + index = self._next_index() + except StopIteration: + return + for _ in range(self._num_workers): # find the next active worker, if any + worker_queue_idx = next(self._worker_queue_idx_cycle) + if self._workers_status[worker_queue_idx]: + break + else: + # not found (i.e., didn't break) + return + + self._index_queues[worker_queue_idx].put((self._send_idx, index)) + self._task_info[self._send_idx] = (worker_queue_idx,) + self._tasks_outstanding += 1 + self._send_idx += 1 + + def _process_data(self, data): + self._rcvd_idx += 1 + self._try_put_index() + if isinstance(data, ExceptionWrapper): + data.reraise() + return data + + def _shutdown_worker(self, worker_id): + # Mark a worker as having finished its work and dead, e.g., due to + # exhausting an `IterableDataset`. This should be used only when this + # `_MultiProcessingDataLoaderIter` is going to continue running. + + assert self._workers_status[worker_id] + + # Signal termination to that specific worker. + q = self._index_queues[worker_id] + # Indicate that no more data will be put on this queue by the current + # process. + q.put(None) + + # Note that we don't actually join the worker here, nor do we remove the + # worker's pid from C side struct because (1) joining may be slow, and + # (2) since we don't join, the worker may still raise error, and we + # prefer capturing those, rather than ignoring them, even though they + # are raised after the worker has finished its job. + # Joinning is deferred to `_shutdown_workers`, which it is called when + # all workers finish their jobs (e.g., `IterableDataset` replicas) or + # when this iterator is garbage collected. + self._workers_status[worker_id] = False + + def _shutdown_workers(self): + # Called when shutting down this `_MultiProcessingDataLoaderIter`. + # See NOTE [ Data Loader Multiprocessing Shutdown Logic ] for details on + # the logic of this function. + python_exit_status = _utils.python_exit_status + if python_exit_status is True or python_exit_status is None: + # See (2) of the note. If Python is shutting down, do no-op. + return + # Normal exit when last reference is gone / iterator is depleted. + # See (1) and the second half of the note. + if not self._shutdown: + self._shutdown = True + try: + # Exit `pin_memory_thread` first because exiting workers may leave + # corrupted data in `worker_result_queue` which `pin_memory_thread` + # reads from. + if hasattr(self, '_pin_memory_thread'): + # Use hasattr in case error happens before we set the attribute. + self._pin_memory_thread_done_event.set() + # Send something to pin_memory_thread in case it is waiting + # so that it can wake up and check `pin_memory_thread_done_event` + self._worker_result_queue.put((None, None)) + self._pin_memory_thread.join() + self._worker_result_queue.close() + + # Exit workers now. + self._workers_done_event.set() + for worker_id in range(len(self._workers)): + # Get number of workers from `len(self._workers)` instead of + # `self._num_workers` in case we error before starting all + # workers. + if self._workers_status[worker_id]: + self._shutdown_worker(worker_id) + for w in self._workers: + w.join() + for q in self._index_queues: + q.cancel_join_thread() + q.close() + finally: + # Even though all this function does is putting into queues that + # we have called `cancel_join_thread` on, weird things can + # happen when a worker is killed by a signal, e.g., hanging in + # `Event.set()`. So we need to guard this with SIGCHLD handler, + # and remove pids from the C side data structure only at the + # end. + # + # FIXME: Unfortunately, for Windows, we are missing a worker + # error detection mechanism here in this function, as it + # doesn't provide a SIGCHLD handler. + if self._worker_pids_set: + _utils.signal_handling._remove_worker_pids(id(self)) + self._worker_pids_set = False + + def __del__(self): + self._shutdown_workers() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_data_worker.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_data_worker.py new file mode 100644 index 0000000..fcac280 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_data_worker.py @@ -0,0 +1,207 @@ +r""""Contains definitions of the methods used by the _BaseDataLoaderIter workers. + +These **needs** to be in global scope since Py2 doesn't support serializing +static methods. +""" + +import torch +import random +import os +from collections import namedtuple +from torch._six import queue +from torch._utils import ExceptionWrapper +from torch.utils.data._utils import signal_handling, MP_STATUS_CHECK_INTERVAL, IS_WINDOWS + +from .my_random_resize_crop import MyRandomResizedCrop + +__all__ = ['worker_loop'] + +if IS_WINDOWS: + import ctypes + from ctypes.wintypes import DWORD, BOOL, HANDLE + + + # On Windows, the parent ID of the worker process remains unchanged when the manager process + # is gone, and the only way to check it through OS is to let the worker have a process handle + # of the manager and ask if the process status has changed. + class ManagerWatchdog(object): + def __init__(self): + self.manager_pid = os.getppid() + + self.kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + self.kernel32.OpenProcess.argtypes = (DWORD, BOOL, DWORD) + self.kernel32.OpenProcess.restype = HANDLE + self.kernel32.WaitForSingleObject.argtypes = (HANDLE, DWORD) + self.kernel32.WaitForSingleObject.restype = DWORD + + # Value obtained from https://msdn.microsoft.com/en-us/library/ms684880.aspx + SYNCHRONIZE = 0x00100000 + self.manager_handle = self.kernel32.OpenProcess(SYNCHRONIZE, 0, self.manager_pid) + + if not self.manager_handle: + raise ctypes.WinError(ctypes.get_last_error()) + + self.manager_dead = False + + def is_alive(self): + if not self.manager_dead: + # Value obtained from https://msdn.microsoft.com/en-us/library/windows/desktop/ms687032.aspx + self.manager_dead = self.kernel32.WaitForSingleObject(self.manager_handle, 0) == 0 + return not self.manager_dead +else: + class ManagerWatchdog(object): + def __init__(self): + self.manager_pid = os.getppid() + self.manager_dead = False + + def is_alive(self): + if not self.manager_dead: + self.manager_dead = os.getppid() != self.manager_pid + return not self.manager_dead + +_worker_info = None + + +class WorkerInfo(object): + __initialized = False + + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + self.__initialized = True + + def __setattr__(self, key, val): + if self.__initialized: + raise RuntimeError("Cannot assign attributes to {} objects".format(self.__class__.__name__)) + return super(WorkerInfo, self).__setattr__(key, val) + + +def get_worker_info(): + r"""Returns the information about the current + :class:`~torch.utils.data.DataLoader` iterator worker process. + + When called in a worker, this returns an object guaranteed to have the + following attributes: + + * :attr:`id`: the current worker id. + * :attr:`num_workers`: the total number of workers. + * :attr:`seed`: the random seed set for the current worker. This value is + determined by main process RNG and the worker id. See + :class:`~torch.utils.data.DataLoader`'s documentation for more details. + * :attr:`dataset`: the copy of the dataset object in **this** process. Note + that this will be a different object in a different process than the one + in the main process. + + When called in the main process, this returns ``None``. + + .. note:: + When used in a :attr:`worker_init_fn` passed over to + :class:`~torch.utils.data.DataLoader`, this method can be useful to + set up each worker process differently, for instance, using ``worker_id`` + to configure the ``dataset`` object to only read a specific fraction of a + sharded dataset, or use ``seed`` to seed other libraries used in dataset + code (e.g., NumPy). + """ + return _worker_info + + +r"""Dummy class used to signal the end of an IterableDataset""" +_IterableDatasetStopIteration = namedtuple('_IterableDatasetStopIteration', ['worker_id']) + + +def worker_loop(dataset_kind, dataset, index_queue, data_queue, done_event, + auto_collation, collate_fn, drop_last, seed, init_fn, worker_id, + num_workers): + # See NOTE [ Data Loader Multiprocessing Shutdown Logic ] for details on the + # logic of this function. + + try: + # Intialize C side signal handlers for SIGBUS and SIGSEGV. Python signal + # module's handlers are executed after Python returns from C low-level + # handlers, likely when the same fatal signal had already happened + # again. + # https://docs.python.org/3/library/signal.html#execution-of-python-signal-handlers + signal_handling._set_worker_signal_handlers() + + torch.set_num_threads(1) + random.seed(seed) + torch.manual_seed(seed) + + global _worker_info + _worker_info = WorkerInfo(id=worker_id, num_workers=num_workers, + seed=seed, dataset=dataset) + + from torch.utils.data import _DatasetKind + + init_exception = None + + try: + if init_fn is not None: + init_fn(worker_id) + + fetcher = _DatasetKind.create_fetcher(dataset_kind, dataset, auto_collation, collate_fn, drop_last) + except Exception: + init_exception = ExceptionWrapper( + where="in DataLoader worker process {}".format(worker_id)) + + # When using Iterable mode, some worker can exit earlier than others due + # to the IterableDataset behaving differently for different workers. + # When such things happen, an `_IterableDatasetStopIteration` object is + # sent over to the main process with the ID of this worker, so that the + # main process won't send more tasks to this worker, and will send + # `None` to this worker to properly exit it. + # + # Note that we cannot set `done_event` from a worker as it is shared + # among all processes. Instead, we set the `iteration_end` flag to + # signify that the iterator is exhausted. When either `done_event` or + # `iteration_end` is set, we skip all processing step and just wait for + # `None`. + iteration_end = False + + watchdog = ManagerWatchdog() + + while watchdog.is_alive(): + try: + r = index_queue.get(timeout=MP_STATUS_CHECK_INTERVAL) + except queue.Empty: + continue + if r is None: + # Received the final signal + assert done_event.is_set() or iteration_end + break + elif done_event.is_set() or iteration_end: + # `done_event` is set. But I haven't received the final signal + # (None) yet. I will keep continuing until get it, and skip the + # processing steps. + continue + idx, index = r + """ Added """ + MyRandomResizedCrop.sample_image_size(idx) + """ Added """ + if init_exception is not None: + data = init_exception + init_exception = None + else: + try: + data = fetcher.fetch(index) + except Exception as e: + if isinstance(e, StopIteration) and dataset_kind == _DatasetKind.Iterable: + data = _IterableDatasetStopIteration(worker_id) + # Set `iteration_end` + # (1) to save future `next(...)` calls, and + # (2) to avoid sending multiple `_IterableDatasetStopIteration`s. + iteration_end = True + else: + # It is important that we don't store exc_info in a variable. + # `ExceptionWrapper` does the correct thing. + # See NOTE [ Python Traceback Reference Cycle Problem ] + data = ExceptionWrapper( + where="in DataLoader worker process {}".format(worker_id)) + data_queue.put((idx, data)) + del data, idx, index, r # save memory + except KeyboardInterrupt: + # Main process will raise KeyboardInterrupt anyways. + pass + if done_event.is_set(): + data_queue.cancel_join_thread() + data_queue.close() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_distributed_sampler.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_distributed_sampler.py new file mode 100644 index 0000000..3f4d4f2 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_distributed_sampler.py @@ -0,0 +1,69 @@ +import math +import torch +from torch.utils.data.distributed import DistributedSampler + +__all__ = ['MyDistributedSampler', 'WeightedDistributedSampler'] + + +class MyDistributedSampler(DistributedSampler): + """ Allow Subset Sampler in Distributed Training """ + + def __init__(self, dataset, num_replicas=None, rank=None, shuffle=True, + sub_index_list=None): + super(MyDistributedSampler, self).__init__(dataset, num_replicas, rank, shuffle) + self.sub_index_list = sub_index_list # numpy + + self.num_samples = int(math.ceil(len(self.sub_index_list) * 1.0 / self.num_replicas)) + self.total_size = self.num_samples * self.num_replicas + print('Use MyDistributedSampler: %d, %d' % (self.num_samples, self.total_size)) + + def __iter__(self): + # deterministically shuffle based on epoch + g = torch.Generator() + g.manual_seed(self.epoch) + indices = torch.randperm(len(self.sub_index_list), generator=g).tolist() + + # add extra samples to make it evenly divisible + indices += indices[:(self.total_size - len(indices))] + indices = self.sub_index_list[indices].tolist() + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) + + +class WeightedDistributedSampler(DistributedSampler): + """ Allow Weighted Random Sampling in Distributed Training """ + + def __init__(self, dataset, num_replicas=None, rank=None, shuffle=True, + weights=None, replacement=True): + super(WeightedDistributedSampler, self).__init__(dataset, num_replicas, rank, shuffle) + + self.weights = torch.as_tensor(weights, dtype=torch.double) if weights is not None else None + self.replacement = replacement + print('Use WeightedDistributedSampler') + + def __iter__(self): + if self.weights is None: + return super(WeightedDistributedSampler, self).__iter__() + else: + g = torch.Generator() + g.manual_seed(self.epoch) + if self.shuffle: + # original: indices = torch.randperm(len(self.dataset), generator=g).tolist() + indices = torch.multinomial(self.weights, len(self.dataset), self.replacement, generator=g).tolist() + else: + indices = list(range(len(self.dataset))) + + # add extra samples to make it evenly divisible + indices += indices[:(self.total_size - len(indices))] + assert len(indices) == self.total_size + + # subsample + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_random_resize_crop.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_random_resize_crop.py new file mode 100644 index 0000000..1b48475 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_dataloader/my_random_resize_crop.py @@ -0,0 +1,136 @@ +import time +import random +import math +import os +from PIL import Image + +import torchvision.transforms.functional as F +import torchvision.transforms as transforms + +__all__ = ['MyRandomResizedCrop', 'MyResizeRandomCrop', 'MyResize'] + +_pil_interpolation_to_str = { + Image.NEAREST: 'PIL.Image.NEAREST', + Image.BILINEAR: 'PIL.Image.BILINEAR', + Image.BICUBIC: 'PIL.Image.BICUBIC', + Image.LANCZOS: 'PIL.Image.LANCZOS', + Image.HAMMING: 'PIL.Image.HAMMING', + Image.BOX: 'PIL.Image.BOX', +} + + +class MyRandomResizedCrop(transforms.RandomResizedCrop): + ACTIVE_SIZE = 224 + IMAGE_SIZE_LIST = [224] + IMAGE_SIZE_SEG = 4 + + CONTINUOUS = False + SYNC_DISTRIBUTED = True + + EPOCH = 0 + BATCH = 0 + + def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), interpolation=Image.BILINEAR): + if not isinstance(size, int): + size = size[0] + super(MyRandomResizedCrop, self).__init__(size, scale, ratio, interpolation) + + def __call__(self, img): + i, j, h, w = self.get_params(img, self.scale, self.ratio) + return F.resized_crop( + img, i, j, h, w, (MyRandomResizedCrop.ACTIVE_SIZE, MyRandomResizedCrop.ACTIVE_SIZE), self.interpolation + ) + + @staticmethod + def get_candidate_image_size(): + if MyRandomResizedCrop.CONTINUOUS: + min_size = min(MyRandomResizedCrop.IMAGE_SIZE_LIST) + max_size = max(MyRandomResizedCrop.IMAGE_SIZE_LIST) + candidate_sizes = [] + for i in range(min_size, max_size + 1): + if i % MyRandomResizedCrop.IMAGE_SIZE_SEG == 0: + candidate_sizes.append(i) + else: + candidate_sizes = MyRandomResizedCrop.IMAGE_SIZE_LIST + + relative_probs = None + return candidate_sizes, relative_probs + + @staticmethod + def sample_image_size(batch_id=None): + if batch_id is None: + batch_id = MyRandomResizedCrop.BATCH + if MyRandomResizedCrop.SYNC_DISTRIBUTED: + _seed = int('%d%.3d' % (batch_id, MyRandomResizedCrop.EPOCH)) + else: + _seed = os.getpid() + time.time() + random.seed(_seed) + candidate_sizes, relative_probs = MyRandomResizedCrop.get_candidate_image_size() + MyRandomResizedCrop.ACTIVE_SIZE = random.choices(candidate_sizes, weights=relative_probs)[0] + + def __repr__(self): + interpolate_str = _pil_interpolation_to_str[self.interpolation] + format_string = self.__class__.__name__ + '(size={0}'.format(MyRandomResizedCrop.IMAGE_SIZE_LIST) + if MyRandomResizedCrop.CONTINUOUS: + format_string += '@continuous' + format_string += ', scale={0}'.format(tuple(round(s, 4) for s in self.scale)) + format_string += ', ratio={0}'.format(tuple(round(r, 4) for r in self.ratio)) + format_string += ', interpolation={0})'.format(interpolate_str) + return format_string + + +class MyResizeRandomCrop(object): + + def __init__(self, interpolation=Image.BILINEAR, + use_padding=False, pad_if_needed=False, fill=0, padding_mode='constant'): + # resize + self.interpolation = interpolation + # random crop + self.use_padding = use_padding + self.pad_if_needed = pad_if_needed + self.fill = fill + self.padding_mode = padding_mode + + def __call__(self, img): + crop_size = MyRandomResizedCrop.ACTIVE_SIZE + + if not self.use_padding: + resize_size = int(math.ceil(crop_size / 0.875)) + img = F.resize(img, resize_size, self.interpolation) + else: + img = F.resize(img, crop_size, self.interpolation) + padding_size = crop_size // 8 + img = F.pad(img, padding_size, self.fill, self.padding_mode) + + # pad the width if needed + if self.pad_if_needed and img.size[0] < crop_size: + img = F.pad(img, (crop_size - img.size[0], 0), self.fill, self.padding_mode) + # pad the height if needed + if self.pad_if_needed and img.size[1] < crop_size: + img = F.pad(img, (0, crop_size - img.size[1]), self.fill, self.padding_mode) + + i, j, h, w = transforms.RandomCrop.get_params(img, (crop_size, crop_size)) + return F.crop(img, i, j, h, w) + + def __repr__(self): + return 'MyResizeRandomCrop(size=%s%s, interpolation=%s, use_padding=%s, fill=%s)' % ( + MyRandomResizedCrop.IMAGE_SIZE_LIST, '@continuous' if MyRandomResizedCrop.CONTINUOUS else '', + _pil_interpolation_to_str[self.interpolation], self.use_padding, self.fill, + ) + + +class MyResize(object): + + def __init__(self, interpolation=Image.BILINEAR): + self.interpolation = interpolation + + def __call__(self, img): + target_size = MyRandomResizedCrop.ACTIVE_SIZE + img = F.resize(img, target_size, self.interpolation) + return img + + def __repr__(self): + return 'MyResize(size=%s%s, interpolation=%s)' % ( + MyRandomResizedCrop.IMAGE_SIZE_LIST, '@continuous' if MyRandomResizedCrop.CONTINUOUS else '', + _pil_interpolation_to_str[self.interpolation] + ) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_modules.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_modules.py new file mode 100644 index 0000000..e11dcbe --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/my_modules.py @@ -0,0 +1,238 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import math +import torch.nn as nn +import torch.nn.functional as F + +from .common_tools import min_divisible_value + +__all__ = ['MyModule', 'MyNetwork', 'init_models', 'set_bn_param', 'get_bn_param', 'replace_bn_with_gn', + 'MyConv2d', 'replace_conv2d_with_my_conv2d'] + + +def set_bn_param(net, momentum, eps, gn_channel_per_group=None, ws_eps=None, **kwargs): + replace_bn_with_gn(net, gn_channel_per_group) + + for m in net.modules(): + if type(m) in [nn.BatchNorm1d, nn.BatchNorm2d]: + m.momentum = momentum + m.eps = eps + elif isinstance(m, nn.GroupNorm): + m.eps = eps + + replace_conv2d_with_my_conv2d(net, ws_eps) + return + + +def get_bn_param(net): + ws_eps = None + for m in net.modules(): + if isinstance(m, MyConv2d): + ws_eps = m.WS_EPS + break + for m in net.modules(): + if isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d): + return { + 'momentum': m.momentum, + 'eps': m.eps, + 'ws_eps': ws_eps, + } + elif isinstance(m, nn.GroupNorm): + return { + 'momentum': None, + 'eps': m.eps, + 'gn_channel_per_group': m.num_channels // m.num_groups, + 'ws_eps': ws_eps, + } + return None + + +def replace_bn_with_gn(model, gn_channel_per_group): + if gn_channel_per_group is None: + return + + for m in model.modules(): + to_replace_dict = {} + for name, sub_m in m.named_children(): + if isinstance(sub_m, nn.BatchNorm2d): + num_groups = sub_m.num_features // min_divisible_value(sub_m.num_features, gn_channel_per_group) + gn_m = nn.GroupNorm(num_groups=num_groups, num_channels=sub_m.num_features, eps=sub_m.eps, affine=True) + + # load weight + gn_m.weight.data.copy_(sub_m.weight.data) + gn_m.bias.data.copy_(sub_m.bias.data) + # load requires_grad + gn_m.weight.requires_grad = sub_m.weight.requires_grad + gn_m.bias.requires_grad = sub_m.bias.requires_grad + + to_replace_dict[name] = gn_m + m._modules.update(to_replace_dict) + + +def replace_conv2d_with_my_conv2d(net, ws_eps=None): + if ws_eps is None: + return + + for m in net.modules(): + to_update_dict = {} + for name, sub_module in m.named_children(): + if isinstance(sub_module, nn.Conv2d) and not sub_module.bias: + # only replace conv2d layers that are followed by normalization layers (i.e., no bias) + to_update_dict[name] = sub_module + for name, sub_module in to_update_dict.items(): + m._modules[name] = MyConv2d( + sub_module.in_channels, sub_module.out_channels, sub_module.kernel_size, sub_module.stride, + sub_module.padding, sub_module.dilation, sub_module.groups, sub_module.bias, + ) + # load weight + m._modules[name].load_state_dict(sub_module.state_dict()) + # load requires_grad + m._modules[name].weight.requires_grad = sub_module.weight.requires_grad + if sub_module.bias is not None: + m._modules[name].bias.requires_grad = sub_module.bias.requires_grad + # set ws_eps + for m in net.modules(): + if isinstance(m, MyConv2d): + m.WS_EPS = ws_eps + + +def init_models(net, model_init='he_fout'): + """ + Conv2d, + BatchNorm2d, BatchNorm1d, GroupNorm + Linear, + """ + if isinstance(net, list): + for sub_net in net: + init_models(sub_net, model_init) + return + for m in net.modules(): + if isinstance(m, nn.Conv2d): + if model_init == 'he_fout': + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif model_init == 'he_fin': + n = m.kernel_size[0] * m.kernel_size[1] * m.in_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + else: + raise NotImplementedError + if m.bias is not None: + m.bias.data.zero_() + elif type(m) in [nn.BatchNorm2d, nn.BatchNorm1d, nn.GroupNorm]: + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + stdv = 1. / math.sqrt(m.weight.size(1)) + m.weight.data.uniform_(-stdv, stdv) + if m.bias is not None: + m.bias.data.zero_() + + +class MyConv2d(nn.Conv2d): + """ + Conv2d with Weight Standardization + https://github.com/joe-siyuan-qiao/WeightStandardization + """ + + def __init__(self, in_channels, out_channels, kernel_size, stride=1, + padding=0, dilation=1, groups=1, bias=True): + super(MyConv2d, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, groups, bias) + self.WS_EPS = None + + def weight_standardization(self, weight): + if self.WS_EPS is not None: + weight_mean = weight.mean(dim=1, keepdim=True).mean(dim=2, keepdim=True).mean(dim=3, keepdim=True) + weight = weight - weight_mean + std = weight.view(weight.size(0), -1).std(dim=1).view(-1, 1, 1, 1) + self.WS_EPS + weight = weight / std.expand_as(weight) + return weight + + def forward(self, x): + if self.WS_EPS is None: + return super(MyConv2d, self).forward(x) + else: + return F.conv2d(x, self.weight_standardization(self.weight), self.bias, + self.stride, self.padding, self.dilation, self.groups) + + def __repr__(self): + return super(MyConv2d, self).__repr__()[:-1] + ', ws_eps=%s)' % self.WS_EPS + + +class MyModule(nn.Module): + + def forward(self, x): + raise NotImplementedError + + @property + def module_str(self): + raise NotImplementedError + + @property + def config(self): + raise NotImplementedError + + @staticmethod + def build_from_config(config): + raise NotImplementedError + + +class MyNetwork(MyModule): + CHANNEL_DIVISIBLE = 8 + + def forward(self, x): + raise NotImplementedError + + @property + def module_str(self): + raise NotImplementedError + + @property + def config(self): + raise NotImplementedError + + @staticmethod + def build_from_config(config): + raise NotImplementedError + + def zero_last_gamma(self): + raise NotImplementedError + + @property + def grouped_block_index(self): + raise NotImplementedError + + """ implemented methods """ + + def set_bn_param(self, momentum, eps, gn_channel_per_group=None, **kwargs): + set_bn_param(self, momentum, eps, gn_channel_per_group, **kwargs) + + def get_bn_param(self): + return get_bn_param(self) + + def get_parameters(self, keys=None, mode='include'): + if keys is None: + for name, param in self.named_parameters(): + if param.requires_grad: yield param + elif mode == 'include': + for name, param in self.named_parameters(): + flag = False + for key in keys: + if key in name: + flag = True + break + if flag and param.requires_grad: yield param + elif mode == 'exclude': + for name, param in self.named_parameters(): + flag = True + for key in keys: + if key in name: + flag = False + break + if flag and param.requires_grad: yield param + else: + raise ValueError('do not support: %s' % mode) + + def weight_parameters(self): + return self.get_parameters() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/pytorch_modules.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/pytorch_modules.py new file mode 100644 index 0000000..a88e978 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/pytorch_modules.py @@ -0,0 +1,154 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import torch +import torch.nn as nn +import torch.nn.functional as F +from collections import OrderedDict +from .my_modules import MyNetwork + +__all__ = [ + 'make_divisible', 'build_activation', 'ShuffleLayer', 'MyGlobalAvgPool2d', 'Hswish', 'Hsigmoid', 'SEModule', + 'MultiHeadCrossEntropyLoss' +] + + +def make_divisible(v, divisor, min_val=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_val: + :return: + """ + if min_val is None: + min_val = divisor + new_v = max(min_val, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +def build_activation(act_func, inplace=True): + if act_func == 'relu': + return nn.ReLU(inplace=inplace) + elif act_func == 'relu6': + return nn.ReLU6(inplace=inplace) + elif act_func == 'tanh': + return nn.Tanh() + elif act_func == 'sigmoid': + return nn.Sigmoid() + elif act_func == 'h_swish': + return Hswish(inplace=inplace) + elif act_func == 'h_sigmoid': + return Hsigmoid(inplace=inplace) + elif act_func is None or act_func == 'none': + return None + else: + raise ValueError('do not support: %s' % act_func) + + +class ShuffleLayer(nn.Module): + + def __init__(self, groups): + super(ShuffleLayer, self).__init__() + self.groups = groups + + def forward(self, x): + batch_size, num_channels, height, width = x.size() + channels_per_group = num_channels // self.groups + # reshape + x = x.view(batch_size, self.groups, channels_per_group, height, width) + x = torch.transpose(x, 1, 2).contiguous() + # flatten + x = x.view(batch_size, -1, height, width) + return x + + def __repr__(self): + return 'ShuffleLayer(groups=%d)' % self.groups + + +class MyGlobalAvgPool2d(nn.Module): + + def __init__(self, keep_dim=True): + super(MyGlobalAvgPool2d, self).__init__() + self.keep_dim = keep_dim + + def forward(self, x): + return x.mean(3, keepdim=self.keep_dim).mean(2, keepdim=self.keep_dim) + + def __repr__(self): + return 'MyGlobalAvgPool2d(keep_dim=%s)' % self.keep_dim + + +class Hswish(nn.Module): + + def __init__(self, inplace=True): + super(Hswish, self).__init__() + self.inplace = inplace + + def forward(self, x): + return x * F.relu6(x + 3., inplace=self.inplace) / 6. + + def __repr__(self): + return 'Hswish()' + + +class Hsigmoid(nn.Module): + + def __init__(self, inplace=True): + super(Hsigmoid, self).__init__() + self.inplace = inplace + + def forward(self, x): + return F.relu6(x + 3., inplace=self.inplace) / 6. + + def __repr__(self): + return 'Hsigmoid()' + + +class SEModule(nn.Module): + REDUCTION = 4 + + def __init__(self, channel, reduction=None): + super(SEModule, self).__init__() + + self.channel = channel + self.reduction = SEModule.REDUCTION if reduction is None else reduction + + num_mid = make_divisible(self.channel // self.reduction, divisor=MyNetwork.CHANNEL_DIVISIBLE) + + self.fc = nn.Sequential(OrderedDict([ + ('reduce', nn.Conv2d(self.channel, num_mid, 1, 1, 0, bias=True)), + ('relu', nn.ReLU(inplace=True)), + ('expand', nn.Conv2d(num_mid, self.channel, 1, 1, 0, bias=True)), + ('h_sigmoid', Hsigmoid(inplace=True)), + ])) + + def forward(self, x): + y = x.mean(3, keepdim=True).mean(2, keepdim=True) + y = self.fc(y) + return x * y + + def __repr__(self): + return 'SE(channel=%d, reduction=%d)' % (self.channel, self.reduction) + + +class MultiHeadCrossEntropyLoss(nn.Module): + + def forward(self, outputs, targets): + assert outputs.dim() == 3, outputs + assert targets.dim() == 2, targets + + assert outputs.size(1) == targets.size(1), (outputs, targets) + num_heads = targets.size(1) + + loss = 0 + for k in range(num_heads): + loss += F.cross_entropy(outputs[:, k, :], targets[:, k]) / num_heads + return loss diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/pytorch_utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/pytorch_utils.py new file mode 100644 index 0000000..cc50432 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/ofa_local/utils/pytorch_utils.py @@ -0,0 +1,218 @@ +# Once for All: Train One Network and Specialize it for Efficient Deployment +# Han Cai, Chuang Gan, Tianzhe Wang, Zhekai Zhang, Song Han +# International Conference on Learning Representations (ICLR), 2020. + +import math +import copy +import time +import torch +import torch.nn as nn + +__all__ = [ + 'mix_images', 'mix_labels', + 'label_smooth', 'cross_entropy_loss_with_soft_target', 'cross_entropy_with_label_smoothing', + 'clean_num_batch_tracked', 'rm_bn_from_net', + 'get_net_device', 'count_parameters', 'count_net_flops', 'measure_net_latency', 'get_net_info', + 'build_optimizer', 'calc_learning_rate', +] + + +""" Mixup """ +def mix_images(images, lam): + flipped_images = torch.flip(images, dims=[0]) # flip along the batch dimension + return lam * images + (1 - lam) * flipped_images + + +def mix_labels(target, lam, n_classes, label_smoothing=0.1): + onehot_target = label_smooth(target, n_classes, label_smoothing) + flipped_target = torch.flip(onehot_target, dims=[0]) + return lam * onehot_target + (1 - lam) * flipped_target + + +""" Label smooth """ +def label_smooth(target, n_classes: int, label_smoothing=0.1): + # convert to one-hot + batch_size = target.size(0) + target = torch.unsqueeze(target, 1) + soft_target = torch.zeros((batch_size, n_classes), device=target.device) + soft_target.scatter_(1, target, 1) + # label smoothing + soft_target = soft_target * (1 - label_smoothing) + label_smoothing / n_classes + return soft_target + + +def cross_entropy_loss_with_soft_target(pred, soft_target): + logsoftmax = nn.LogSoftmax() + return torch.mean(torch.sum(- soft_target * logsoftmax(pred), 1)) + + +def cross_entropy_with_label_smoothing(pred, target, label_smoothing=0.1): + soft_target = label_smooth(target, pred.size(1), label_smoothing) + return cross_entropy_loss_with_soft_target(pred, soft_target) + + +""" BN related """ +def clean_num_batch_tracked(net): + for m in net.modules(): + if isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d): + if m.num_batches_tracked is not None: + m.num_batches_tracked.zero_() + + +def rm_bn_from_net(net): + for m in net.modules(): + if isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d): + m.forward = lambda x: x + + +""" Network profiling """ +def get_net_device(net): + return net.parameters().__next__().device + + +def count_parameters(net): + total_params = sum(p.numel() for p in net.parameters() if p.requires_grad) + return total_params + + +def count_net_flops(net, data_shape=(1, 3, 224, 224)): + from .flops_counter import profile + if isinstance(net, nn.DataParallel): + net = net.module + + flop, _ = profile(copy.deepcopy(net), data_shape) + return flop + + +def measure_net_latency(net, l_type='gpu8', fast=True, input_shape=(3, 224, 224), clean=False): + if isinstance(net, nn.DataParallel): + net = net.module + + # remove bn from graph + rm_bn_from_net(net) + + # return `ms` + if 'gpu' in l_type: + l_type, batch_size = l_type[:3], int(l_type[3:]) + else: + batch_size = 1 + + data_shape = [batch_size] + list(input_shape) + if l_type == 'cpu': + if fast: + n_warmup = 5 + n_sample = 10 + else: + n_warmup = 50 + n_sample = 50 + if get_net_device(net) != torch.device('cpu'): + if not clean: + print('move net to cpu for measuring cpu latency') + net = copy.deepcopy(net).cpu() + elif l_type == 'gpu': + if fast: + n_warmup = 5 + n_sample = 10 + else: + n_warmup = 50 + n_sample = 50 + else: + raise NotImplementedError + images = torch.zeros(data_shape, device=get_net_device(net)) + + measured_latency = {'warmup': [], 'sample': []} + net.eval() + with torch.no_grad(): + for i in range(n_warmup): + inner_start_time = time.time() + net(images) + used_time = (time.time() - inner_start_time) * 1e3 # ms + measured_latency['warmup'].append(used_time) + if not clean: + print('Warmup %d: %.3f' % (i, used_time)) + outer_start_time = time.time() + for i in range(n_sample): + net(images) + total_time = (time.time() - outer_start_time) * 1e3 # ms + measured_latency['sample'].append((total_time, n_sample)) + return total_time / n_sample, measured_latency + + +def get_net_info(net, input_shape=(3, 224, 224), measure_latency=None, print_info=True): + net_info = {} + if isinstance(net, nn.DataParallel): + net = net.module + + # parameters + net_info['params'] = count_parameters(net) / 1e6 + + # flops + net_info['flops'] = count_net_flops(net, [1] + list(input_shape)) / 1e6 + + # latencies + latency_types = [] if measure_latency is None else measure_latency.split('#') + for l_type in latency_types: + latency, measured_latency = measure_net_latency(net, l_type, fast=False, input_shape=input_shape) + net_info['%s latency' % l_type] = { + 'val': latency, + 'hist': measured_latency + } + + if print_info: + print(net) + print('Total training params: %.2fM' % (net_info['params'])) + print('Total FLOPs: %.2fM' % (net_info['flops'])) + for l_type in latency_types: + print('Estimated %s latency: %.3fms' % (l_type, net_info['%s latency' % l_type]['val'])) + + return net_info + + +""" optimizer """ +def build_optimizer(net_params, opt_type, opt_param, init_lr, weight_decay, no_decay_keys, seperate=1.0): + # enc_list, dec_list = [], [] + # for name, param in model.named_parameters(): + # if ('setenc' in name) or ('fc1' in name) or ('fc2' in name): + # enc_list.append(param) + # else: + # dec_list.append(param) + #optimizer = optim.Adam([{'params': dec_list, 'lr': args.dec_lr}, + # {'params': enc_list, 'lr': args.enc_lr}], lr=1e-4) + if no_decay_keys is not None: + assert isinstance(net_params, list) and len(net_params) == 2 + net_params = [ + {'params': net_params[0], 'weight_decay': weight_decay}, + {'params': net_params[1], 'weight_decay': 0}, + ] + elif seperate != 1.0: + net_params = [{'params': net_params[0], 'weight_decay': weight_decay, 'lr': init_lr * seperate}, + {'params': net_params[1], 'weight_decay': weight_decay, 'lr': init_lr}] + else: + net_params = [{'params': net_params, 'weight_decay': weight_decay}] + + if opt_type == 'sgd': + opt_param = {} if opt_param is None else opt_param + momentum, nesterov = opt_param.get('momentum', 0.9), opt_param.get('nesterov', True) + optimizer = torch.optim.SGD(net_params, init_lr, momentum=momentum, nesterov=nesterov) + elif opt_type == 'adam': + optimizer = torch.optim.Adam(net_params, init_lr) + else: + raise NotImplementedError + return optimizer + + +""" learning rate schedule """ +def calc_learning_rate(epoch, init_lr, n_epochs, batch=0, + nBatch=None, lr_schedule_type='cosine', optimizer=None): + if lr_schedule_type == 'cosine': + t_total = n_epochs * nBatch + t_cur = epoch * nBatch + batch + lr = 0.5 * init_lr * (1 + math.cos(math.pi * t_cur / t_total)) + elif lr_schedule_type == 'reduce': + for param_group in optimizer.param_groups: + lr = param_group['lr'] + elif lr_schedule_type is None: + lr = init_lr + else: + raise ValueError('do not support: %s' % lr_schedule_type) + return lr diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/parser.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/parser.py new file mode 100644 index 0000000..2d36b32 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/parser.py @@ -0,0 +1,43 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import argparse + +def str2bool(v): + return v.lower() in ['t', 'true', True] + + +def get_parser(): + parser = argparse.ArgumentParser() + # general settings + parser.add_argument('--seed', type=int, default=333) + parser.add_argument('--gpu', type=str, default='0', help='set visible gpus') + parser.add_argument('--model_name', type=str, default=None, choices=['generator', 'predictor', 'train_arch']) + parser.add_argument('--save-path', type=str, default='results', help='the path of save directory') + parser.add_argument('--data-path', type=str, default='data', help='the path of save directory') + parser.add_argument('--save-epoch', type=int, default=20, help='how many epochs to wait each time to save model states') + parser.add_argument('--max-epoch', type=int, default=400, help='number of epochs to train') + parser.add_argument('--batch_size', type=int, default=32, help='batch size for generator') + parser.add_argument('--graph-data-name', default='ofa_mbv3', help='graph dataset name') + parser.add_argument('--nvt', type=int, default=27, help='number of different node types, 21 for ofa_mbv3 without in/out node') + # set encoder + parser.add_argument('--num-sample', type=int, default=20, help='the number of images as input for set encoder') + # graph encoder + parser.add_argument('--hs', type=int, default=56, help='hidden size of GRUs') + parser.add_argument('--nz', type=int, default=56, help='the number of dimensions of latent vectors z') + # test + parser.add_argument('--test', action='store_true', default=False, help='turn on test mode') + parser.add_argument('--load-epoch', type=int, default=20, help='checkpoint epoch loaded for meta-test') + parser.add_argument('--data-name', type=str, default=None, help='meta-test dataset name') + parser.add_argument('--num-class', type=int, default=None, help='the number of class of dataset') + parser.add_argument('--num-gen-arch', type=int, default=200, help='the number of candidate architectures generated by the generator') + parser.add_argument('--train-arch', type=str2bool, default=True, help='whether to train the searched architecture') + # database + parser.add_argument('--index', type=int, default=None, help='the process number when creating DB') + parser.add_argument('--imgnet', type=str, default=None, help='The path of imagenet') + parser.add_argument('--collect', action='store_true', default=False, help='whether to train the searched architecture') + + args = parser.parse_args() + + return args diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/__init__.py new file mode 100644 index 0000000..2e1c31c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/__init__.py @@ -0,0 +1,6 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from .predictor import Predictor +from .predictor_model import PredictorModel diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/predictor.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/predictor.py new file mode 100644 index 0000000..d50ed70 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/predictor.py @@ -0,0 +1,172 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import torch +import os +import random +from tqdm import tqdm +import numpy as np +import time +import os +import shutil + +from torch import nn, optim +from torch.optim.lr_scheduler import ReduceLROnPlateau +from scipy.stats import pearsonr + +from transfer_nag_lib.MetaD2A_mobilenetV3.metad2a_utils import load_graph_config, decode_ofa_mbv3_to_igraph +from transfer_nag_lib.MetaD2A_mobilenetV3.metad2a_utils import Log, get_log +from transfer_nag_lib.MetaD2A_mobilenetV3.metad2a_utils import load_model, save_model + +from transfer_nag_lib.MetaD2A_mobilenetV3.loader import get_meta_train_loader +from .predictor_model import PredictorModel +from all_path import * + + +class Predictor: + def __init__(self, args): + self.args = args + self.batch_size = args.batch_size + self.data_path = args.data_path + self.num_sample = args.num_sample + self.max_epoch = args.max_epoch + self.save_epoch = args.save_epoch + self.model_path = UNNOISE_META_PREDICTOR_CKPT_PATH #MODEL_METAD2A_PATH_OFA + self.save_path = args.save_path + self.model_name = 'predictor' + self.test = args.test + self.device = torch.device("cuda:0") + self.max_corr_dict = {'corr': -1, 'epoch': -1} + self.train_arch = args.train_arch + + graph_config = load_graph_config( + args.graph_data_name, args.nvt, args.data_path) + + self.model = PredictorModel(args, graph_config) + self.model.to(self.device) + + if self.test: + self.data_name = args.data_name + self.num_class = args.num_class + self.load_epoch = args.load_epoch + load_model(self.model, self.model_path, load_max_pt='ckpt_max_corr.pt') + self.model.to(self.device) + else: + self.optimizer = optim.Adam(self.model.parameters(), lr=1e-4) + self.scheduler = ReduceLROnPlateau(self.optimizer, 'min', + factor=0.1, patience=10, verbose=True) + self.mtrloader = get_meta_train_loader( + self.batch_size, self.data_path, self.num_sample, is_pred=True) + + self.acc_mean = self.mtrloader.dataset.mean + self.acc_std = self.mtrloader.dataset.std + + self.mtrlog = Log(self.args, open(os.path.join( + self.save_path, self.model_name, 'meta_train_predictor.log'), 'w')) + self.mtrlog.print_args() + + def forward(self, x, arch): + D_mu = self.model.set_encode(x.unsqueeze(0).to(self.device)).unsqueeze(0) + G_mu = self.model.graph_encode(arch[0]) + y_pred = self.model.predict(D_mu, G_mu) + return y_pred + + def meta_train(self): + sttime = time.time() + for epoch in range(1, self.max_epoch + 1): + self.mtrlog.ep_sttime = time.time() + loss, corr = self.meta_train_epoch(epoch) + self.scheduler.step(loss) + self.mtrlog.print_pred_log(loss, corr, 'train', epoch) + valoss, vacorr = self.meta_validation(epoch) + if self.max_corr_dict['corr'] < vacorr: + self.max_corr_dict['corr'] = vacorr + self.max_corr_dict['epoch'] = epoch + self.max_corr_dict['loss'] = valoss + save_model(epoch, self.model, self.model_path, max_corr=True) + + self.mtrlog.print_pred_log( + valoss, vacorr, 'valid', max_corr_dict=self.max_corr_dict) + + if epoch % self.save_epoch == 0: + save_model(epoch, self.model, self.model_path) + + self.mtrlog.save_time_log() + self.mtrlog.max_corr_log(self.max_corr_dict) + + def meta_train_epoch(self, epoch): + self.model.to(self.device) + self.model.train() + + self.mtrloader.dataset.set_mode('train') + + dlen = len(self.mtrloader.dataset) + trloss = 0 + y_all, y_pred_all = [], [] + pbar = tqdm(self.mtrloader) + + for batch in pbar: + batch_loss = 0 + y_batch, y_pred_batch = [], [] + self.optimizer.zero_grad() + for x, g, acc in batch: + y_pred = self.forward(x, decode_ofa_mbv3_to_igraph(g)) + + y = acc.to(self.device) + batch_loss += self.model.mseloss(y_pred, y) + + y = y.squeeze().tolist() + y_pred = y_pred.squeeze().tolist() + + y_batch.append(y) + y_pred_batch.append(y_pred) + y_all.append(y) + y_pred_all.append(y_pred) + + batch_loss.backward() + trloss += float(batch_loss) + self.optimizer.step() + pbar.set_description(get_log( + epoch, batch_loss, y_pred_batch, y_batch, self.acc_std, self.acc_mean)) + + return trloss / dlen, pearsonr(np.array(y_all), + np.array(y_pred_all))[0] + + + def meta_validation(self, epoch): + self.model.to(self.device) + self.model.eval() + + valoss = 0 + self.mtrloader.dataset.set_mode('valid') + dlen = len(self.mtrloader.dataset) + y_all, y_pred_all = [], [] + pbar = tqdm(self.mtrloader) + + with torch.no_grad(): + for batch in pbar: + batch_loss = 0 + y_batch, y_pred_batch = [], [] + + for x, g, acc in batch: + y_pred = self.forward(x, decode_ofa_mbv3_to_igraph(g)) + + y = acc.to(self.device) + batch_loss += self.model.mseloss(y_pred, y) + + y = y.squeeze().tolist() + y_pred = y_pred.squeeze().tolist() + + y_batch.append(y) + y_pred_batch.append(y_pred) + y_all.append(y) + y_pred_all.append(y_pred) + + valoss += float(batch_loss) + pbar.set_description(get_log( + epoch, batch_loss, y_pred_batch, y_batch, self.acc_std, self.acc_mean, tag='val')) + return valoss / dlen, pearsonr(np.array(y_all), + np.array(y_pred_all))[0] + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/predictor_model.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/predictor_model.py new file mode 100644 index 0000000..28a90a5 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/predictor/predictor_model.py @@ -0,0 +1,241 @@ +###################################################################################### +# Copyright (c) muhanzhang, D-VAE, NeurIPS 2019 [GitHub D-VAE] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +from torch import nn +from transfer_nag_lib.MetaD2A_mobilenetV3.set_encoder.setenc_models import SetPool + + +class PredictorModel(nn.Module): + def __init__(self, args, graph_config): + super(PredictorModel, self).__init__() + self.max_n = graph_config['max_n'] # maximum number of vertices + self.nvt = graph_config['num_vertex_type'] # number of vertex types + self.START_TYPE = graph_config['START_TYPE'] + self.END_TYPE = graph_config['END_TYPE'] + # import pdb; pdb.set_trace() + self.hs = args.hs # hidden state size of each vertex + self.nz = args.nz # size of latent representation z + self.gs = args.hs # size of graph state + self.bidir = True # whether to use bidirectional encoding + self.vid = True + self.device = None + self.input_type = 'DG' + self.num_sample = args.num_sample + + if self.vid: + self.vs = self.hs + self.max_n # vertex state size = hidden state + vid + else: + self.vs = self.hs + + # 0. encoding-related + self.grue_forward = nn.GRUCell(self.nvt, self.hs) # encoder GRU + self.grue_backward = nn.GRUCell(self.nvt, self.hs) # backward encoder GRU + self.fc1 = nn.Linear(self.gs, self.nz) # latent mean + self.fc2 = nn.Linear(self.gs, self.nz) # latent logvar + + # 2. gate-related + self.gate_forward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.gate_backward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.mapper_forward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) # disable bias to ensure padded zeros also mapped to zeros + self.mapper_backward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) + + # 3. bidir-related, to unify sizes + if self.bidir: + self.hv_unify = nn.Sequential( + nn.Linear(self.hs * 2, self.hs), + ) + self.hg_unify = nn.Sequential( + nn.Linear(self.gs * 2, self.gs), + ) + + # 4. other + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + self.tanh = nn.Tanh() + self.logsoftmax1 = nn.LogSoftmax(1) + + # 6. predictor + np = self.gs + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + input_dim = 0 + if 'D' in self.input_type: + input_dim += self.nz + if 'G' in self.input_type: + input_dim += self.nz + + self.pred_fc = nn.Sequential( + nn.Linear(input_dim, self.hs), + nn.Tanh(), + nn.Linear(self.hs, 1) + ) + self.mseloss = nn.MSELoss(reduction='sum') + + + def predict(self, D_mu, G_mu): + input_vec = [] + if 'D' in self.input_type: + input_vec.append(D_mu) + if 'G' in self.input_type: + input_vec.append(G_mu) + input_vec = torch.cat(input_vec, dim=1) + return self.pred_fc(input_vec) + + def get_device(self): + if self.device is None: + self.device = next(self.parameters()).device + return self.device + + def _get_zeros(self, n, length): + return torch.zeros(n, length).to(self.get_device()) # get a zero hidden state + + def _get_zero_hidden(self, n=1): + return self._get_zeros(n, self.hs) # get a zero hidden state + + def _one_hot(self, idx, length): + if type(idx) in [list, range]: + if idx == []: + return None + idx = torch.LongTensor(idx).unsqueeze(0).t() + x = torch.zeros((len(idx), length)).scatter_(1, idx, 1).to(self.get_device()) + else: + idx = torch.LongTensor([idx]).unsqueeze(0) + x = torch.zeros((1, length)).scatter_(1, idx, 1).to(self.get_device()) + return x + + def _gated(self, h, gate, mapper): + return gate(h) * mapper(h) + + def _collate_fn(self, G): + return [g.copy() for g in G] + + def _propagate_to(self, G, v, propagator, H=None, reverse=False, gate=None, mapper=None): + # propagate messages to vertex index v for all graphs in G + # return the new messages (states) at v + G = [g for g in G if g.vcount() > v] + if len(G) == 0: + return + if H is not None: + idx = [i for i, g in enumerate(G) if g.vcount() > v] + H = H[idx] + v_types = [g.vs[v]['type'] for g in G] + X = self._one_hot(v_types, self.nvt) + if reverse: + H_name = 'H_backward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.successors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.successors(v), self.max_n) for g in G] + gate, mapper = self.gate_backward, self.mapper_backward + else: + H_name = 'H_forward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.predecessors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.predecessors(v), self.max_n) for g in G] + if gate is None: + gate, mapper = self.gate_forward, self.mapper_forward + if self.vid: + H_pred = [[torch.cat([x[i], y[i:i + 1]], 1) for i in range(len(x))] for x, y in zip(H_pred, vids)] + # if h is not provided, use gated sum of v's predecessors' states as the input hidden state + if H is None: + max_n_pred = max([len(x) for x in H_pred]) # maximum number of predecessors + if max_n_pred == 0: + H = self._get_zero_hidden(len(G)) + else: + H_pred = [torch.cat(h_pred + + [self._get_zeros(max_n_pred - len(h_pred), self.vs)], 0).unsqueeze(0) + for h_pred in H_pred] # pad all to same length + H_pred = torch.cat(H_pred, 0) # batch * max_n_pred * vs + H = self._gated(H_pred, gate, mapper).sum(1) # batch * hs + Hv = propagator(X, H) + for i, g in enumerate(G): + g.vs[v][H_name] = Hv[i:i + 1] + return Hv + + def _propagate_from(self, G, v, propagator, H0=None, reverse=False): + # perform a series of propagation_to steps starting from v following a topo order + # assume the original vertex indices are in a topological order + if reverse: + prop_order = range(v, -1, -1) + else: + prop_order = range(v, self.max_n) + Hv = self._propagate_to(G, v, propagator, H0, reverse=reverse) # the initial vertex + for v_ in prop_order[1:]: + self._propagate_to(G, v_, propagator, reverse=reverse) + return Hv + + def _get_graph_state(self, G, decode=False): + # get the graph states + # when decoding, use the last generated vertex's state as the graph state + # when encoding, use the ending vertex state or unify the starting and ending vertex states + Hg = [] + for g in G: + hg = g.vs[g.vcount() - 1]['H_forward'] + if self.bidir and not decode: # decoding never uses backward propagation + hg_b = g.vs[0]['H_backward'] + hg = torch.cat([hg, hg_b], 1) + Hg.append(hg) + Hg = torch.cat(Hg, 0) + if self.bidir and not decode: + Hg = self.hg_unify(Hg) + return Hg + + + def set_encode(self, X): + proto_batch = [] + for x in X: + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + return v + + + def graph_encode(self, G): + # encode graphs G into latent vectors + if type(G) != list: + G = [G] + self._propagate_from(G, 0, self.grue_forward, H0=self._get_zero_hidden(len(G)), + reverse=False) + if self.bidir: + self._propagate_from(G, self.max_n - 1, self.grue_backward, + H0=self._get_zero_hidden(len(G)), reverse=True) + Hg = self._get_graph_state(G) + mu = self.fc1(Hg) + #logvar = self.fc2(Hg) + return mu #, logvar + + + def reparameterize(self, mu, logvar, eps_scale=0.01): + # return z ~ N(mu, std) + if self.training: + std = logvar.mul(0.5).exp_() + eps = torch.randn_like(std) * eps_scale + return eps.mul(std).add_(mu) + else: + return mu + \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/process_dataset.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/process_dataset.py new file mode 100644 index 0000000..699ec4f --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/process_dataset.py @@ -0,0 +1,158 @@ +import numpy as np +import torchvision.models as models +import torchvision.datasets as dset +import os +import torch +import argparse +import random +import torchvision.transforms as transforms +import os, sys +if sys.version_info[0] == 2: + import cPickle as pickle +else: + import pickle +from PIL import Image + +parser = argparse.ArgumentParser("sota") +parser.add_argument('--gpu', type=str, default='0', help='set visible gpus') +parser.add_argument('--data-path', type=str, default='data', help='the path of save directory') +parser.add_argument('--dataset', type=str, default='cifar10', help='choose dataset') +parser.add_argument('--seed', type=int, default=-1, help='random seed') +args = parser.parse_args() + +if args.seed is None or args.seed < 0: args.seed = random.randint(1, 100000) + +os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu +np.random.seed(args.seed) +random.seed(args.seed) + +# remove last fully-connected layer +model = models.resnet18(pretrained=True).eval() +feature_extractor = torch.nn.Sequential(*list(model.children())[:-1]) + + +def get_transform(dataset): + if args.dataset == 'mnist': + mean, std = [0.1307, 0.1307, 0.1307], [0.3081, 0.3081, 0.3081] + elif args.dataset == 'svhn': + mean, std = [0.4376821, 0.4437697, 0.47280442], [0.19803012, 0.20101562, 0.19703614] + elif args.dataset == 'cifar10': + mean = [x / 255 for x in [125.3, 123.0, 113.9]] + std = [x / 255 for x in [63.0, 62.1, 66.7]] + elif args.dataset == 'cifar100': + mean = [x / 255 for x in [129.3, 124.1, 112.4]] + std = [x / 255 for x in [68.2, 65.4, 70.4]] + elif args.dataset == 'imagenet32': + mean = [x / 255 for x in [122.68, 116.66, 104.01]] + std = [x / 255 for x in [66.22, 64.20, 67.86]] + + transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + if dataset == 'mnist': + transform.transforms.append(transforms.Lambda(lambda x: x.repeat(3, 1, 1))) + return transform + + +def process(dataset, n_classes): + data_label = {i: [] for i in range(n_classes)} + for x, y in dataset: + data_label[y].append(x) + for i in range(n_classes): + data_label[i] = torch.stack(data_label[i]) + + holder = {i: [] for i in range(n_classes)} + for i in range(n_classes): + with torch.no_grad(): + data = feature_extractor(data_label[i]) + holder[i].append(data.squeeze()) + return holder + + + +class ImageNet32(object): + train_list = [ + ['train_data_batch_1', '27846dcaa50de8e21a7d1a35f30f0e91'], + ['train_data_batch_2', 'c7254a054e0e795c69120a5727050e3f'], + ['train_data_batch_3', '4333d3df2e5ffb114b05d2ffc19b1e87'], + ['train_data_batch_4', '1620cdf193304f4a92677b695d70d10f'], + ['train_data_batch_5', '348b3c2fdbb3940c4e9e834affd3b18d'], + ['train_data_batch_6', '6e765307c242a1b3d7d5ef9139b48945'], + ['train_data_batch_7', '564926d8cbf8fc4818ba23d2faac7564'], + ['train_data_batch_8', 'f4755871f718ccb653440b9dd0ebac66'], + ['train_data_batch_9', 'bb6dd660c38c58552125b1a92f86b5d4'], + ['train_data_batch_10', '8f03f34ac4b42271a294f91bf480f29b'], + ] + valid_list = [ + ['val_data', '3410e3017fdaefba8d5073aaa65e4bd6'], + ] + + def __init__(self, root, n_class, transform): + self.transform = transform + downloaded_list = self.train_list + self.n_class = n_class + self.data_label = {i: [] for i in range(n_class)} + self.data = [] + self.targets = [] + + for i, (file_name, checksum) in enumerate(downloaded_list): + file_path = os.path.join(root, file_name) + with open(file_path, 'rb') as f: + if sys.version_info[0] == 2: + entry = pickle.load(f) + else: + entry = pickle.load(f, encoding='latin1') + for j, k in enumerate(entry['labels']): + self.data_label[k - 1].append(entry['data'][j]) + + for i in range(n_class): + self.data_label[i] = np.vstack(self.data_label[i]).reshape(-1, 3, 32, 32) + self.data_label[i] = self.data_label[i].transpose((0, 2, 3, 1)) # convert to HWC + + def get(self, use_num_cls, max_num=None): + assert isinstance(use_num_cls, list) \ + and len(use_num_cls) > 0 and len(use_num_cls) < self.n_class, \ + 'invalid use_num_cls : {:}'.format(use_num_cls) + new_data, new_targets = [], [] + for i in use_num_cls: + new_data.append(self.data_label[i][:max_num] if max_num is not None else self.data_label[i]) + new_targets.extend([i] * max_num if max_num is not None + else [i] * len(self.data_label[i])) + self.data = np.concatenate(new_data) + self.targets = new_targets + + imgs = [] + for img in self.data: + img = Image.fromarray(img) + img = self.transform(img) + with torch.no_grad(): + imgs.append(feature_extractor(img.unsqueeze(0)).squeeze().unsqueeze(0)) + return torch.cat(imgs) + + +if __name__ == '__main__': + ncls = {'mnist': 10, 'svhn': 10, 'cifar10': 10, 'cifar100': 100, 'imagenet32': 1000} + transform = get_transform(args.dataset) + if args.dataset == 'imagenet32': + imgnet32 = ImageNet32(args.data, ncls[args.dataset], transform) + data_label = {i: [] for i in range(1000)} + for i in range(1000): + m = imgnet32.get([i]) + data_label[i].append(m) + if i % 10 == 0: + print(f'Currently saving features of {i}-th class') + torch.save(data_label, f'{args.save_path}/{args.dataset}bylabel.pt') + else: + if args.dataset == 'mnist': + data = dset.MNIST(args.data_path, train=True, transform=transform, download=True) + elif args.dataset == 'svhn': + data = dset.SVHN(args.data_path, split='train', transform=transform, download=True) + elif args.dataset == 'cifar10': + data = dset.CIFAR10(args.data_path, train=True, transform=transform, download=True) + elif args.dataset == 'cifar100': + data = dset.CIFAR100(args.data_path, train=True, transform=transform, download=True) + dataset = process(data, ncls[args.dataset]) + torch.save(dataset, f'{args.save_path}/{args.dataset}bylabel.pt') + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/set_encoder/setenc_models.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/set_encoder/setenc_models.py new file mode 100644 index 0000000..6acab39 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/set_encoder/setenc_models.py @@ -0,0 +1,37 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from transfer_nag_lib.MetaD2A_mobilenetV3.set_encoder.setenc_modules import * + + +class SetPool(nn.Module): + def __init__(self, dim_input, num_outputs, dim_output, + num_inds=32, dim_hidden=128, num_heads=4, ln=False, mode=None): + super(SetPool, self).__init__() + if 'sab' in mode: # [32, 400, 128] + self.enc = nn.Sequential( + SAB(dim_input, dim_hidden, num_heads, ln=ln), # SAB? + SAB(dim_hidden, dim_hidden, num_heads, ln=ln)) + else: # [32, 400, 128] + self.enc = nn.Sequential( + ISAB(dim_input, dim_hidden, num_heads, num_inds, ln=ln), # SAB? + ISAB(dim_hidden, dim_hidden, num_heads, num_inds, ln=ln)) + if 'PF' in mode: #[32, 1, 501] + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), + nn.Linear(dim_hidden, dim_output)) + elif 'P' in mode: + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln)) + else: #torch.Size([32, 1, 501]) + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), # 32 1 128 + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + nn.Linear(dim_hidden, dim_output)) + # "", sm, sab, sabsm + def forward(self, X): + x1 = self.enc(X) + x2 = self.dec(x1) + return x2 diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/set_encoder/setenc_modules.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/set_encoder/setenc_modules.py new file mode 100644 index 0000000..54fe4d7 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_mobilenetV3/set_encoder/setenc_modules.py @@ -0,0 +1,67 @@ +##################################################################################### +# Copyright (c) Juho Lee SetTransformer, ICML 2019 [GitHub set_transformer] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + +class MAB(nn.Module): + def __init__(self, dim_Q, dim_K, dim_V, num_heads, ln=False): + super(MAB, self).__init__() + self.dim_V = dim_V + self.num_heads = num_heads + self.fc_q = nn.Linear(dim_Q, dim_V) + self.fc_k = nn.Linear(dim_K, dim_V) + self.fc_v = nn.Linear(dim_K, dim_V) + if ln: + self.ln0 = nn.LayerNorm(dim_V) + self.ln1 = nn.LayerNorm(dim_V) + self.fc_o = nn.Linear(dim_V, dim_V) + + def forward(self, Q, K): + Q = self.fc_q(Q) + K, V = self.fc_k(K), self.fc_v(K) + + dim_split = self.dim_V // self.num_heads + Q_ = torch.cat(Q.split(dim_split, 2), 0) + K_ = torch.cat(K.split(dim_split, 2), 0) + V_ = torch.cat(V.split(dim_split, 2), 0) + + A = torch.softmax(Q_.bmm(K_.transpose(1,2))/math.sqrt(self.dim_V), 2) + O = torch.cat((Q_ + A.bmm(V_)).split(Q.size(0), 0), 2) + O = O if getattr(self, 'ln0', None) is None else self.ln0(O) + O = O + F.relu(self.fc_o(O)) + O = O if getattr(self, 'ln1', None) is None else self.ln1(O) + return O + +class SAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, ln=False): + super(SAB, self).__init__() + self.mab = MAB(dim_in, dim_in, dim_out, num_heads, ln=ln) + + def forward(self, X): + return self.mab(X, X) + +class ISAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, num_inds, ln=False): + super(ISAB, self).__init__() + self.I = nn.Parameter(torch.Tensor(1, num_inds, dim_out)) + nn.init.xavier_uniform_(self.I) + self.mab0 = MAB(dim_out, dim_in, dim_out, num_heads, ln=ln) + self.mab1 = MAB(dim_in, dim_out, dim_out, num_heads, ln=ln) + + def forward(self, X): + H = self.mab0(self.I.repeat(X.size(0), 1, 1), X) + return self.mab1(X, H) + +class PMA(nn.Module): + def __init__(self, dim, num_heads, num_seeds, ln=False): + super(PMA, self).__init__() + self.S = nn.Parameter(torch.Tensor(1, num_seeds, dim)) + nn.init.xavier_uniform_(self.S) + self.mab = MAB(dim, dim, dim, num_heads, ln=ln) + + def forward(self, X): + return self.mab(self.S.repeat(X.size(0), 1, 1), X) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/__init__.py new file mode 100644 index 0000000..f2ac959 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/__init__.py @@ -0,0 +1,5 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from .solver import Generator diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/generator_model.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/generator_model.py new file mode 100644 index 0000000..4084328 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/generator_model.py @@ -0,0 +1,406 @@ +###################################################################################### +# Copyright (c) muhanzhang, D-VAE, NeurIPS 2019 [GitHub D-VAE] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import math +import random +import torch +from torch import nn +from torch.nn import functional as F +import numpy as np +import igraph +import pdb +import os +import sys +from transfer_nag_lib.MetaD2A_nas_bench_201.set_encoder.setenc_models import SetPool + + +class GeneratorModel(nn.Module): + def __init__(self, args, graph_config): + super(GeneratorModel, self).__init__() + self.max_n = graph_config['max_n'] # maximum number of vertices + self.nvt = args.nvt # number of vertex types + self.START_TYPE = graph_config['START_TYPE'] + self.END_TYPE = graph_config['END_TYPE'] + self.hs = args.hs # hidden state size of each vertex + self.nz = args.nz # size of latent representation z + self.gs = args.hs # size of graph state + self.bidir = True # whether to use bidirectional encoding + self.vid = True + self.device = None + self.num_sample = args.num_sample + + if self.vid: + self.vs = self.hs + self.max_n # vertex state size = hidden state + vid + else: + self.vs = self.hs + + # 0. encoding-related + self.grue_forward = nn.GRUCell(self.nvt, self.hs) # encoder GRU + self.grue_backward = nn.GRUCell( + self.nvt, self.hs) # backward encoder GRU + self.enc_g_mu = nn.Linear(self.gs, self.nz) # latent mean + self.enc_g_var = nn.Linear(self.gs, self.nz) # latent var + self.fc1 = nn.Linear(self.gs, self.nz) # latent mean + self.fc2 = nn.Linear(self.gs, self.nz) # latent logvar + + # 1. decoding-related + self.grud = nn.GRUCell(self.nvt, self.hs) # decoder GRU + # from latent z to initial hidden state h0 + self.fc3 = nn.Linear(self.nz, self.hs) + self.add_vertex = nn.Sequential( + nn.Linear(self.hs, self.hs * 2), + nn.ReLU(), + nn.Linear(self.hs * 2, self.nvt) + ) # which type of new vertex to add f(h0, hg) + self.add_edge = nn.Sequential( + nn.Linear(self.hs * 2, self.hs * 4), + nn.ReLU(), + nn.Linear(self.hs * 4, 1) + ) # whether to add edge between v_i and v_new, f(hvi, hnew) + self.decoding_gate = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.decoding_mapper = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) # disable bias to ensure padded zeros also mapped to zeros + + # 2. gate-related + self.gate_forward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.gate_backward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.mapper_forward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) # disable bias to ensure padded zeros also mapped to zeros + self.mapper_backward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) + + # 3. bidir-related, to unify sizes + if self.bidir: + self.hv_unify = nn.Sequential( + nn.Linear(self.hs * 2, self.hs), + ) + self.hg_unify = nn.Sequential( + nn.Linear(self.gs * 2, self.gs), + ) + + # 4. other + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + self.tanh = nn.Tanh() + self.logsoftmax1 = nn.LogSoftmax(1) + + # 6. predictor + np = self.gs + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + def get_device(self): + if self.device is None: + self.device = next(self.parameters()).device + return self.device + + def _get_zeros(self, n, length): + # get a zero hidden state + return torch.zeros(n, length).to(self.get_device()) + + def _get_zero_hidden(self, n=1): + return self._get_zeros(n, self.hs) # get a zero hidden state + + def _one_hot(self, idx, length): + if type(idx) in [list, range]: + if idx == []: + return None + idx = torch.LongTensor(idx).unsqueeze(0).t() + x = torch.zeros((len(idx), length) + ).scatter_(1, idx, 1).to(self.get_device()) + else: + idx = torch.LongTensor([idx]).unsqueeze(0) + x = torch.zeros((1, length) + ).scatter_(1, idx, 1).to(self.get_device()) + return x + + def _gated(self, h, gate, mapper): + return gate(h) * mapper(h) + + def _collate_fn(self, G): + return [g.copy() for g in G] + + def _propagate_to(self, G, v, propagator, + H=None, reverse=False, gate=None, mapper=None): + # propagate messages to vertex index v for all graphs in G + # return the new messages (states) at v + G = [g for g in G if g.vcount() > v] + if len(G) == 0: + return + if H is not None: + idx = [i for i, g in enumerate(G) if g.vcount() > v] + H = H[idx] + v_types = [g.vs[v]['type'] for g in G] + X = self._one_hot(v_types, self.nvt) + H_name = 'H_forward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.predecessors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.predecessors(v), self.max_n) for g in G] + if reverse: + H_name = 'H_backward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.successors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.successors(v), self.max_n) for g in G] + gate, mapper = self.gate_backward, self.mapper_backward + else: + H_name = 'H_forward' # name of the hidden states attribute + H_pred = [ + [g.vs[x][H_name] for x in g.predecessors(v)] for g in G] + if self.vid: + vids = [ + self._one_hot(g.predecessors(v), self.max_n) for g in G] + if gate is None: + gate, mapper = self.gate_forward, self.mapper_forward + if self.vid: + H_pred = [[torch.cat( + [x[i], y[i:i + 1]], 1) for i in range(len(x)) + ] for x, y in zip(H_pred, vids)] + # if h is not provided, use gated sum of v's predecessors' states as the input hidden state + if H is None: + # maximum number of predecessors + max_n_pred = max([len(x) for x in H_pred]) + if max_n_pred == 0: + H = self._get_zero_hidden(len(G)) + else: + H_pred = [torch.cat(h_pred + + [self._get_zeros(max_n_pred - len(h_pred), + self.vs)], 0).unsqueeze(0) + for h_pred in H_pred] # pad all to same length + H_pred = torch.cat(H_pred, 0) # batch * max_n_pred * vs + H = self._gated(H_pred, gate, mapper).sum(1) # batch * hs + Hv = propagator(X, H) + for i, g in enumerate(G): + g.vs[v][H_name] = Hv[i:i + 1] + return Hv + + def _propagate_from(self, G, v, propagator, H0=None, reverse=False): + # perform a series of propagation_to steps starting from v following a topo order + # assume the original vertex indices are in a topological order + if reverse: + prop_order = range(v, -1, -1) + else: + prop_order = range(v, self.max_n) + Hv = self._propagate_to(G, v, propagator, H0, + reverse=reverse) # the initial vertex + for v_ in prop_order[1:]: + self._propagate_to(G, v_, propagator, reverse=reverse) + return Hv + + def _update_v(self, G, v, H0=None): + # perform a forward propagation step at v when decoding to update v's state + # self._propagate_to(G, v, self.grud, H0, reverse=False) + self._propagate_to(G, v, self.grud, H0, + reverse=False, gate=self.decoding_gate, + mapper=self.decoding_mapper) + return + + def _get_vertex_state(self, G, v): + # get the vertex states at v + Hv = [] + for g in G: + if v >= g.vcount(): + hv = self._get_zero_hidden() + else: + hv = g.vs[v]['H_forward'] + Hv.append(hv) + Hv = torch.cat(Hv, 0) + return Hv + + def _get_graph_state(self, G, decode=False): + # get the graph states + # when decoding, use the last generated vertex's state as the graph state + # when encoding, use the ending vertex state or unify the starting and ending vertex states + Hg = [] + for g in G: + hg = g.vs[g.vcount() - 1]['H_forward'] + if self.bidir and not decode: # decoding never uses backward propagation + hg_b = g.vs[0]['H_backward'] + hg = torch.cat([hg, hg_b], 1) + Hg.append(hg) + Hg = torch.cat(Hg, 0) + if self.bidir and not decode: + Hg = self.hg_unify(Hg) + return Hg + + def graph_encode(self, G): + # encode graphs G into latent vectors + if type(G) != list: + G = [G] + self._propagate_from(G, 0, self.grue_forward, + H0=self._get_zero_hidden(len(G)), reverse=False) + if self.bidir: + self._propagate_from(G, self.max_n - 1, self.grue_backward, + H0=self._get_zero_hidden(len(G)), reverse=True) + Hg = self._get_graph_state(G) + mu, logvar = self.enc_g_mu(Hg), self.enc_g_var(Hg) + return mu, logvar + + def set_encode(self, X): + proto_batch = [] + for x in X: # X.shape: [32, 400, 512] + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + mu, logvar = self.fc1(v), self.fc2(v) + return mu, logvar + + def reparameterize(self, mu, logvar, eps_scale=0.01): + # return z ~ N(mu, std) + if self.training: + std = logvar.mul(0.5).exp_() + eps = torch.randn_like(std) * eps_scale + return eps.mul(std).add_(mu) + else: + return mu + + def _get_edge_score(self, Hvi, H, H0): + # compute scores for edges from vi based on Hvi, H (current vertex) and H0 + # in most cases, H0 need not be explicitly included since Hvi and H contain its information + return self.sigmoid(self.add_edge(torch.cat([Hvi, H], -1))) + + def graph_decode(self, z, stochastic=True): + # decode latent vectors z back to graphs + # if stochastic=True, stochastically sample each action from the predicted distribution; + # otherwise, select argmax action deterministically. + H0 = self.tanh(self.fc3(z)) # or relu activation, similar performance + G = [igraph.Graph(directed=True) for _ in range(len(z))] + for g in G: + g.add_vertex(type=self.START_TYPE) + self._update_v(G, 0, H0) + finished = [False] * len(G) + for idx in range(1, self.max_n): + # decide the type of the next added vertex + if idx == self.max_n - 1: # force the last node to be end_type + new_types = [self.END_TYPE] * len(G) + else: + Hg = self._get_graph_state(G, decode=True) + type_scores = self.add_vertex(Hg) + if stochastic: + type_probs = F.softmax(type_scores, 1 + ).cpu().detach().numpy() + new_types = [np.random.choice(range(self.nvt), + p=type_probs[i]) for i in range(len(G))] + else: + new_types = torch.argmax(type_scores, 1) + new_types = new_types.flatten().tolist() + for i, g in enumerate(G): + if not finished[i]: + g.add_vertex(type=new_types[i]) + self._update_v(G, idx) + + # decide connections + edge_scores = [] + for vi in range(idx - 1, -1, -1): + Hvi = self._get_vertex_state(G, vi) + H = self._get_vertex_state(G, idx) + ei_score = self._get_edge_score(Hvi, H, H0) + if stochastic: + random_score = torch.rand_like(ei_score) + decisions = random_score < ei_score + else: + decisions = ei_score > 0.5 + for i, g in enumerate(G): + if finished[i]: + continue + if new_types[i] == self.END_TYPE: + # if new node is end_type, connect it to all loose-end vertices (out_degree==0) + end_vertices = set([ + v.index for v in g.vs.select(_outdegree_eq=0) + if v.index != g.vcount() - 1]) + for v in end_vertices: + g.add_edge(v, g.vcount() - 1) + finished[i] = True + continue + if decisions[i, 0]: + g.add_edge(vi, g.vcount() - 1) + self._update_v(G, idx) + + for g in G: + del g.vs['H_forward'] # delete hidden states to save GPU memory + return G + + def loss(self, mu, logvar, G_true, beta=0.005): + # compute the loss of decoding mu and logvar to true graphs using teacher forcing + # ensure when computing the loss of step i, steps 0 to i-1 are correct + z = self.reparameterize(mu, logvar) + H0 = self.tanh(self.fc3(z)) # or relu activation, similar performance + G = [igraph.Graph(directed=True) for _ in range(len(z))] + for g in G: + g.add_vertex(type=self.START_TYPE) + self._update_v(G, 0, H0) + res = 0 # log likelihood + for v_true in range(1, self.max_n): + # calculate the likelihood of adding true types of nodes + # use start type to denote padding vertices since start type only appears for vertex 0 + # and will never be a true type for later vertices, thus it's free to use + true_types = [g_true.vs[v_true]['type'] + if v_true < g_true.vcount() + else self.START_TYPE for g_true in G_true] + Hg = self._get_graph_state(G, decode=True) + type_scores = self.add_vertex(Hg) + # vertex log likelihood + vll = self.logsoftmax1(type_scores)[ + np.arange(len(G)), true_types].sum() + res = res + vll + for i, g in enumerate(G): + if true_types[i] != self.START_TYPE: + g.add_vertex(type=true_types[i]) + self._update_v(G, v_true) + + # calculate the likelihood of adding true edges + true_edges = [] + for i, g_true in enumerate(G_true): + true_edges.append(g_true.get_adjlist(igraph.IN)[v_true] + if v_true < g_true.vcount() else []) + edge_scores = [] + for vi in range(v_true - 1, -1, -1): + Hvi = self._get_vertex_state(G, vi) + H = self._get_vertex_state(G, v_true) + ei_score = self._get_edge_score(Hvi, H, H0) + edge_scores.append(ei_score) + for i, g in enumerate(G): + if vi in true_edges[i]: + g.add_edge(vi, v_true) + self._update_v(G, v_true) + edge_scores = torch.cat(edge_scores[::-1], 1) + + ground_truth = torch.zeros_like(edge_scores) + idx1 = [i for i, x in enumerate(true_edges) + for _ in range(len(x))] + idx2 = [xx for x in true_edges for xx in x] + ground_truth[idx1, idx2] = 1.0 + + # edges log-likelihood + ell = - F.binary_cross_entropy( + edge_scores, ground_truth, reduction='sum') + res = res + ell + + res = -res # convert likelihood to loss + kld = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) + return res + beta * kld, res, kld diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/solver.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/solver.py new file mode 100644 index 0000000..d40b1ba --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/generator/solver.py @@ -0,0 +1,314 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import os +import sys +import random +from tqdm import tqdm +import numpy as np +import time + +import torch +from torch import optim +from torch.optim.lr_scheduler import ReduceLROnPlateau + +from .generator_model import GeneratorModel +from transfer_nag_lib.MetaD2A_nas_bench_201.loader import get_meta_train_loader, get_meta_test_loader +from transfer_nag_lib.MetaD2A_nas_bench_201.metad2a_utils import load_model, save_model +from transfer_nag_lib.MetaD2A_nas_bench_201.metad2a_utils import Accumulator, Log +from transfer_nag_lib.MetaD2A_nas_bench_201.metad2a_utils import load_graph_config, decode_igraph_to_NAS_BENCH_201_string, decode_igraph_to_NAS201_matrix, \ + decode_NAS_BENCH_201_8_to_igraph + + +class Generator: + def __init__(self, args): + self.args = args + self.batch_size = args.batch_size + self.data_path = args.data_path + self.num_sample = args.num_sample + self.max_epoch = args.max_epoch + self.save_epoch = args.save_epoch + self.model_path = args.model_path + self.save_path = args.save_path + self.model_name = args.model_name + self.test = args.test + self.device = torch.device("cpu") + + graph_config = load_graph_config( + args.graph_data_name, args.nvt, args.data_path) + self.model = GeneratorModel(args, graph_config) + self.nasbench201 = None + self.model.to(self.device) + + if self.test: + self.data_name = args.data_name + self.num_class = args.num_class + self.load_epoch = args.load_epoch + self.num_gen_arch = 10 # args.num_gen_arch + self.mtrloader = get_meta_train_loader( + self.batch_size, self.data_path, self.num_sample, True) + load_model(self.model, self.model_path, self.load_epoch) + + else: + self.optimizer = optim.Adam(self.model.parameters(), lr=1e-4) + self.scheduler = ReduceLROnPlateau(self.optimizer, 'min', + factor=0.1, patience=10, verbose=True) + self.mtrloader = get_meta_train_loader( + self.batch_size, self.data_path, self.num_sample) + self.mtrlog = Log(self.args, open(os.path.join( + self.save_path, self.model_name, 'meta_train_generator.log'), 'w')) + self.mtrlog.print_args() + self.mtrlogger = Accumulator('loss', 'recon_loss', 'kld') + self.mvallogger = Accumulator('loss', 'recon_loss', 'kld') + + def meta_train(self): + sttime = time.time() + for epoch in range(1, self.max_epoch + 1): + self.mtrlog.ep_sttime = time.time() + loss = self.meta_train_epoch(epoch) + self.scheduler.step(loss) + self.mtrlog.print(self.mtrlogger, epoch, tag='train') + + self.meta_validation() + self.mtrlog.print(self.mvallogger, epoch, tag='valid') + + if epoch % self.save_epoch == 0: + save_model(epoch, self.model, self.model_path) + + self.mtrlog.save_time_log() + + def meta_train_epoch(self, epoch): + self.model.to(self.device) + self.model.train() + train_loss, recon_loss, kld_loss = 0, 0, 0 + + self.mtrloader.dataset.set_mode('train') + for x, g, acc in tqdm(self.mtrloader): + self.optimizer.zero_grad() + mu, logvar = self.model.set_encode(x.to(self.device)) + loss, recon, kld = self.model.loss(mu, logvar, g) + loss.backward() + self.optimizer.step() + + cnt = len(x) + self.mtrlogger.accum([loss.item()/cnt, + recon.item()/cnt, + kld.item()/cnt]) + return self.mtrlogger.get('loss') + + def meta_validation(self): + self.model.to(self.device) + self.model.eval() + train_loss, recon_loss, kld_loss = 0, 0, 0 + + self.mtrloader.dataset.set_mode('valid') + for x, g, acc in tqdm(self.mtrloader): + with torch.no_grad(): + mu, logvar = self.model.set_encode(x.to(self.device)) + loss, recon, kld = self.model.loss(mu, logvar, g) + + cnt = len(x) + self.mvallogger.accum([loss.item()/cnt, + recon.item()/cnt, + kld.item()/cnt]) + return self.mvallogger.get('loss') + + def meta_test(self): + if self.data_name == 'all': + for data_name in ['cifar100', 'cifar10', 'mnist', 'svhn', 'aircraft', 'pets']: + self.meta_test_per_dataset(data_name) + else: + self.meta_test_per_dataset(self.data_name) + + def get_topk_idx(self, topk=1): + self.mtrloader.dataset.set_mode('train') + if self.nasbench201 is None: + self.nasbench201 = torch.load( + os.path.join(self.data_path, 'nasbench201.pt')) + z_repr = [] + g_repr = [] + acc_repr = [] + for x, g, acc in tqdm(self.mtrloader): + str = decode_igraph_to_NAS_BENCH_201_string(g[0]) + arch_idx = -1 + for idx, arch_str in enumerate(self.nasbench201['arch']['str']): + if arch_str == str: + arch_idx = idx + break + g_repr.append(arch_idx) + acc_repr.append(acc.detach().cpu().numpy()[0]) + best = np.argsort(-1*np.array(acc_repr))[:topk] + return np.array(g_repr)[best], np.array(acc_repr)[best] + + def topk_train(self, topk=1): + self.mtrloader.dataset.set_mode('train') + z_repr = [] + g_repr = [] + acc_repr = [] + for x, g, acc in tqdm(self.mtrloader): + str = decode_igraph_to_NAS_BENCH_201_string(g[0]) + g_repr.append(str) + acc_repr.append(acc.detach().cpu().numpy()[0]) + best = np.argsort(-1*np.array(acc_repr))[:topk] + return np.array(g_repr)[best], np.array(acc_repr)[best] + + # def decode_igraph_to_NAS201(self, graph, encoding): + # str = decode_igraph_to_NAS_BENCH_201_string(graph) + # op_indices = convert_str_to_op_indices(str) + # naslib_object = NasBench201SearchSpace() + # convert_op_indices_to_naslib(op_indices=op_indices, naslib_object=naslib_object) + # enc = encode_201(arch=naslib_object, encoding_type=encoding) + # return enc + + # def decode_str_to_NAS201(self, str, encoding): + # op_indices = convert_str_to_op_indices(str) + # naslib_object = NasBench201SearchSpace() + # convert_op_indices_to_naslib(op_indices=op_indices, naslib_object=naslib_object) + # enc = encode_201(arch=naslib_object, encoding_type=encoding) + # return enc + + def train_dgp(self, encoding='path', encode=False): + self.model.to(self.device) + self.model.eval() + + self.mtrloader.dataset.set_mode('train') + z_repr = [] + g_repr = [] + acc_repr = [] + for x, g, acc in tqdm(self.mtrloader): + sys.stdout.flush() + mu, logvar = self.model.set_encode(x.to(self.device)) + z = self.model.reparameterize( + mu, logvar).cpu().detach().numpy().flatten() + if encode: + graph_matrix = self.decode_igraph_to_NAS201( + g[0], encoding=encoding) + else: + graph_matrix = decode_igraph_to_NAS201_matrix(g[0]).flatten() + z_repr.append(np.concatenate((z, graph_matrix))) + g_repr.append(graph_matrix) + acc_repr.append(acc.detach().cpu().numpy()[0]) + return z_repr, g_repr, acc_repr + + def test_dgp(self, data_name='cifar10', encoding='path', encode=False): + meta_test_path = os.path.join( + self.save_path, 'meta_test', data_name, 'generated_arch') + if not os.path.exists(meta_test_path): + os.makedirs(meta_test_path) + + meta_test_loader = get_meta_test_loader( + self.data_path, data_name, self.num_sample, self.num_class) + + print(f'==> generate architectures for {data_name}') + inputs = [] + accs = [] + inputs_, accs_ = self.generate_architectures_dgp( + meta_test_loader, data_name, + meta_test_path, self.num_gen_arch, encoding=encoding, encode=encode) + inputs.extend(inputs_) + accs.extend(accs_) + print(f'==> done\n') + return np.array(inputs), np.array(accs) + + def generate_architectures_dgp(self, + meta_test_loader, data_name, meta_test_path, num_gen_arch, encoding='path', encode=False): + self.nasbench201 = torch.load( + os.path.join(self.data_path, 'nasbench201.pt')) + overall_arch_num = len(self.nasbench201['arch']['str']) + self.model.eval() + self.model.to(self.device) + + dataset_arch_repr = [] + acc_repr = [] + for x in meta_test_loader: + mu, logvar = self.model.set_encode(x.to(self.device)) + z = self.model.reparameterize(mu, logvar).cpu().detach().numpy()[0] + break + with torch.no_grad(): + for i in range(overall_arch_num): + if encode: + arch_str = self.nasbench201['arch']['str'][i] + arch = self.decode_str_to_NAS201( + arch_str, encoding=encoding) + else: + arch_str = self.nasbench201['arch']['matrix'][i] + igraph, n = decode_NAS_BENCH_201_8_to_igraph(arch_str) + arch = decode_igraph_to_NAS201_matrix(igraph).flatten() + dataset_arch_repr.append(np.concatenate((z, arch))) + acc_repr.append(self.nasbench201['test-acc'][data_name][i]) + if i % 1000 == 0: + print(i) + + return dataset_arch_repr, acc_repr + + def get_items(self, full_target, full_source, source): + return [full_target[full_source.index(_)] for _ in source] + + def meta_test_per_dataset(self, data_name): + meta_test_path = os.path.join( + self.save_path, 'meta_test', data_name, 'generated_arch') + if not os.path.exists(meta_test_path): + os.makedirs(meta_test_path) + + meta_test_loader = get_meta_test_loader( + self.data_path, data_name, self.num_sample, self.num_class) + + print(f'==> generate architectures for {data_name}') + runs = 10 if data_name in ['cifar10', 'cifar100'] else 1 + elasped_time = [] + for run in range(1, runs+1): + print(f'==> run {run}/{runs}') + elasped_time.append(self.generate_architectures( + meta_test_loader, data_name, + meta_test_path, run, self.num_gen_arch)) + print(f'==> done\n') + + time_path = os.path.join( + self.save_path, 'meta_test', data_name, 'time.txt') + with open(time_path, 'w') as f_time: + msg = f'generator elasped time {np.mean(elasped_time):.2f}s' + print(f'==> save time in {time_path}') + f_time.write(msg+'\n') + print(msg) + + def generate_architectures(self, + meta_test_loader, data_name, meta_test_path, run, num_gen_arch): + self.model.eval() + self.model.to(self.device) + + architecture_string_lst = [] + total_cnt, valid_cnt = 0, 0 + flag = False + + start = time.time() + with torch.no_grad(): + for x in meta_test_loader: + mu, logvar = self.model.set_encode(x.to(self.device)) + z = self.model.reparameterize(mu, logvar) + generated_graph_lst = self.model.graph_decode(z) + for g in generated_graph_lst: + architecture_string = decode_igraph_to_NAS_BENCH_201_string( + g) + total_cnt += 1 + if architecture_string is not None: + if not architecture_string in architecture_string_lst: + valid_cnt += 1 + architecture_string_lst.append(architecture_string) + if valid_cnt == num_gen_arch: + flag = True + break + if flag: + break + elapsed = time.time()-start + + spath = os.path.join(meta_test_path, f"run_{run}.txt") + with open(spath, 'w') as f: + print(f'==> save generated architectures in {spath}') + msg = f'elapsed time: {elapsed:6.2f}s ' + print(msg) + f.write(msg+'\n') + for i, architecture_string in enumerate(architecture_string_lst): + f.write(f"{architecture_string}\n") + return elapsed diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_aircraft.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_aircraft.py new file mode 100644 index 0000000..bf75a21 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_aircraft.py @@ -0,0 +1,64 @@ +""" +@author: Hayeon Lee +2020/02/19 +Script for downloading, and reorganizing aircraft +for few shot classification +Run this file as follows: + python get_data.py +""" + +import pickle +import os +import numpy as np +from tqdm import tqdm +import requests +import tarfile +from PIL import Image +import glob +import shutil +import pickle +import collections +import sys +sys.path.append(os.path.join(os.getcwd(), 'meta_nas')) +from all_path import RAW_DATA_PATH + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +# dir_path = 'data/raw-data/' +dir_path = RAW_DATA_PATH +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, 'fgvc-aircraft-2013b.tar.gz') + +if not os.path.exists(file_name): + print(f"Downloading {file_name}\n") + download_file( + 'http://www.robots.ox.ac.uk/~vgg/data/fgvc-aircraft/archives/fgvc-aircraft-2013b.tar.gz', + file_name) + print("\nDownloading done.\n") +else: + print("fgvc-aircraft-2013b.tar.gz has already been downloaded. Did not download twice.\n") + +untar_file_name = os.path.join(dir_path, 'aircraft') +if not os.path.exists(untar_file_name): + tarname = file_name + print("Untarring: {}".format(tarname)) + tar = tarfile.open(tarname) + tar.extractall(untar_file_name) + tar.close() +else: + print(f"{untar_file_name} folder already exists. Did not untarring twice\n") +os.remove(file_name) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_checkpoint.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_checkpoint.py new file mode 100644 index 0000000..6d8f747 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_checkpoint.py @@ -0,0 +1,37 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +file_name = 'ckpt_400.pt' +dir_path = 'results/generator/model' +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, file_name) +if not os.path.exists(file_name): + print(f"Downloading {file_name}\n") + download_file('https://www.dropbox.com/s/8sh04wxk1t43xtg/ckpt_400.pt?dl=1', file_name) + print("Downloading done.\n") +else: + print(f"{file_name} has already been downloaded. Did not download twice.\n") + + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_mnist.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_mnist.py new file mode 100644 index 0000000..1e00df1 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_mnist.py @@ -0,0 +1,65 @@ +""" +@author: Hayeon Lee +2020/02/19 +Script for downloading, and reorganizing aircraft +for few shot classification +Run this file as follows: + python get_data.py +""" + +import pickle +import os +import numpy as np +from tqdm import tqdm +import requests +import tarfile +from PIL import Image +import glob +import shutil +import pickle +import collections +import sys +sys.path.append(os.path.join(os.getcwd(), 'meta_nas')) +from all_path import RAW_DATA_PATH + + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +# dir_path = 'data/raw-data/mnist' +dir_path = os.path.join(RAW_DATA_PATH, 'mnist') +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, 'mnist.tar.gz') + +if not os.path.exists(file_name): + print(f"Downloading {file_name}\n") + download_file( + 'https://www.dropbox.com/s/fdx2ws1zh5rc9ae/mnist.tar.gz?dl=1', + file_name) + print("\nDownloading done.\n") +else: + print(f"{file_name} has already been downloaded. Did not download twice.\n") + +untar_folder = os.path.join(dir_path, "MNIST") +if not os.path.exists(untar_folder): + tarname = file_name + print("Untarring: {}".format(tarname)) + tar = tarfile.open(tarname) + tar.extractall(dir_path) + tar.close() +else: + print(f"{untar_folder} folder already exists. Did not untarring twice\n") +os.remove(file_name) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_pets.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_pets.py new file mode 100644 index 0000000..2f2e02f --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_pets.py @@ -0,0 +1,47 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile +import sys +sys.path.append(os.path.join(os.getcwd(), 'meta_nas')) +from all_path import RAW_DATA_PATH + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +# dir_path = 'data/raw-data/pets' +dir_path = os.path.join(RAW_DATA_PATH, 'pets') +if not os.path.exists(dir_path): + os.makedirs(dir_path) + +full_name = os.path.join(dir_path, 'test15.pth') +if not os.path.exists(full_name): + print(f"Downloading {full_name}\n") + download_file('https://www.dropbox.com/s/kzmrwyyk5iaugv0/test15.pth?dl=1', full_name) + print("Downloading done.\n") +else: + print(f"{full_name} has already been downloaded. Did not download twice.\n") + +full_name = os.path.join(dir_path, 'train85.pth') +if not os.path.exists(full_name): + print(f"Downloading {full_name}\n") + download_file('https://www.dropbox.com/s/w7mikpztkamnw9s/train85.pth?dl=1', full_name) + print("Downloading done.\n") +else: + print(f"{full_name} has already been downloaded. Did not download twice.\n") diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_predictor_checkpoint.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_predictor_checkpoint.py new file mode 100644 index 0000000..041edca --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_predictor_checkpoint.py @@ -0,0 +1,35 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +file_name = 'ckpt_max_corr.pt' +dir_path = 'results/predictor/model' +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, file_name) +if not os.path.exists(file_name): + print(f"Downloading {file_name}\n") + download_file('https://www.dropbox.com/s/1l73vq2orv0chso/ckpt_max_corr.pt?dl=1', file_name) + print("Downloading done.\n") +else: + print(f"{file_name} has already been downloaded. Did not download twice.\n") diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_preprocessed_data.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_preprocessed_data.py new file mode 100644 index 0000000..c34babf --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/get_files/get_preprocessed_data.py @@ -0,0 +1,56 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile +import sys +sys.path.append(os.path.join(os.getcwd(), 'meta_nas')) +from all_path import DATA_PATH + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +# dir_path = 'data' +dir_path = DATA_PATH +if not os.path.exists(dir_path): + os.makedirs(dir_path) + +def get_preprocessed_data(file_name, url): + print(f"Downloading {file_name} datasets\n") + full_name = os.path.join(dir_path, file_name) + download_file(url, full_name) + print("Downloading done.\n") + + +for file_name, url in [ + ('aircraftbylabel.pt', 'https://www.dropbox.com/s/mb66kitv20ykctp/aircraftbylabel.pt?dl=1'), + ('cifar100bylabel.pt', 'https://www.dropbox.com/s/y0xahxgzj29kffk/cifar100bylabel.pt?dl=1'), + ('cifar10bylabel.pt', 'https://www.dropbox.com/s/wt1pcwi991xyhwr/cifar10bylabel.pt?dl=1'), + ('imgnet32bylabel.pt', 'https://www.dropbox.com/s/7r3hpugql8qgi9d/imgnet32bylabel.pt?dl=1'), + ('meta_train_task_lst.pt', 'https://www.dropbox.com/s/0eu01gig3gnxvk4/meta_train_task_lst.pt?dl=1'), + ('meta_train_tasks_generator_idx.pt', 'https://www.dropbox.com/s/reqtqut3eiyeut4/meta_train_tasks_generator_idx.pt?dl=1'), + ('meta_train_tasks_generator.pt', 'https://www.dropbox.com/s/2qjjtfldw99sqx0/meta_train_tasks_generator.pt?dl=1'), + ('meta_train_tasks_predictor_idx.pt', 'https://www.dropbox.com/s/ziwckbuqdokmgo7/meta_train_tasks_predictor_idx.pt?dl=1'), + ('meta_train_tasks_predictor.pt', 'https://www.dropbox.com/s/wc6kylzo5ehqlem/meta_train_tasks_predictor.pt?dl=1'), + ('petsbylabel.pt', 'https://www.dropbox.com/s/mxh6qz3grhy7wcn/petsbylabel.pt?dl=1'), + ('mnistbylabel.pt', 'https://www.dropbox.com/s/86rbuic7a7y34e4/mnistbylabel.pt?dl=1'), + ('nasbench201.pt', 'https://www.dropbox.com/s/qhyhdfc9l5nborq/nasbench201.pt?dl=1'), + ('svhnbylabel.pt', 'https://www.dropbox.com/s/yywaelhrsl6egvd/svhnbylabel.pt?dl=1'), + ]: + + get_preprocessed_data(file_name, url) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/loader.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/loader.py new file mode 100644 index 0000000..bdc8d2a --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/loader.py @@ -0,0 +1,133 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import os +import torch +from torch.utils.data import Dataset +from torch.utils.data import DataLoader + + +def get_meta_train_loader(batch_size, data_path, num_sample, is_pred=True): + dataset = MetaTrainDatabase(data_path, num_sample, is_pred) + print(f'==> The number of tasks for meta-training: {len(dataset)}') + + loader = DataLoader(dataset=dataset, + batch_size=batch_size, + shuffle=True, + num_workers=0, + collate_fn=collate_fn) + return loader + + +def get_meta_test_loader(data_path, data_name, num_class=None, is_pred=False): + dataset = MetaTestDataset(data_path, data_name, num_class) + print(f'==> Meta-Test dataset {data_name}') + + loader = DataLoader(dataset=dataset, + batch_size=100, + shuffle=False, + num_workers=0) + return loader + + +class MetaTrainDatabase(Dataset): + def __init__(self, data_path, num_sample, is_pred=False): + self.mode = 'train' + self.acc_norm = True + self.num_sample = num_sample + self.x = torch.load(os.path.join(data_path, 'imgnet32bylabel.pt')) + + if is_pred: + mtr_data_path = os.path.join( + data_path, 'meta_train_tasks_predictor.pt') + idx_path = os.path.join( + data_path, 'meta_train_tasks_predictor_idx.pt') + else: + mtr_data_path = os.path.join( + data_path, 'meta_train_tasks_generator.pt') + idx_path = os.path.join( + data_path, 'meta_train_tasks_generator_idx.pt') + + data = torch.load(mtr_data_path) + self.acc = data['acc'] + self.task = data['task'] + self.graph = data['g'] + # self.graph = data['graph'] + + random_idx_lst = torch.load(idx_path) + self.idx_lst = {} + self.idx_lst['valid'] = random_idx_lst[:400] + self.idx_lst['train'] = random_idx_lst[400:] + self.acc = torch.tensor(self.acc) + self.mean = torch.mean(self.acc[self.idx_lst['train']]).item() + self.std = torch.std(self.acc[self.idx_lst['train']]).item() + self.task_lst = torch.load(os.path.join( + data_path, 'meta_train_task_lst.pt')) + + def set_mode(self, mode): + self.mode = mode + + def __len__(self): + return len(self.idx_lst[self.mode]) + + def __getitem__(self, index): + data = [] + ridx = self.idx_lst[self.mode] + tidx = self.task[ridx[index]] + classes = self.task_lst[tidx] + graph = self.graph[ridx[index]] + acc = self.acc[ridx[index]] + for cls in classes: + cx = self.x[cls-1][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + x = torch.cat(data) + if self.acc_norm: + acc = ((acc - self.mean) / self.std) / 100.0 + else: + acc = acc / 100.0 + return x, graph, acc + + +class MetaTestDataset(Dataset): + def __init__(self, data_path, data_name, num_sample, num_class=None): + self.num_sample = num_sample + self.data_name = data_name + + num_class_dict = { + 'cifar100': 100, + 'cifar10': 10, + 'mnist': 10, + 'svhn': 10, + 'aircraft': 30, + 'pets': 37 + } + + if num_class is not None: + self.num_class = num_class + else: + self.num_class = num_class_dict[data_name] + + self.x = torch.load(os.path.join(data_path, f'{data_name}bylabel.pt')) + + def __len__(self): + return 1000000 + + def __getitem__(self, index): + data = [] + classes = list(range(self.num_class)) + for cls in classes: + cx = self.x[cls][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + x = torch.cat(data) + return x + + +def collate_fn(batch): + x = torch.stack([item[0] for item in batch]) + graph = [item[1] for item in batch] + acc = torch.stack([item[2] for item in batch]) + return [x, graph, acc] diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/main.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/main.py new file mode 100644 index 0000000..240657b --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/main.py @@ -0,0 +1,105 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +import sys +import random +import numpy as np +import argparse +import torch +from torch import optim +from torch.optim.lr_scheduler import ReduceLROnPlateau + +# from parser import get_parser +from .generator import Generator +from .predictor import Predictor +sys.path.append(os.getcwd()) + + +def str2bool(v): + return v.lower() in ['t', 'true', True] + + +def get_parser(): + parser = argparse.ArgumentParser() + # general settings + parser.add_argument('--seed', type=int, default=333) + parser.add_argument('--gpu', type=str, default='0', + help='set visible gpus') + parser.add_argument('--model_name', type=str, default='generator', + help='select model [generator|predictor]') + parser.add_argument('--save-path', type=str, + default='C:\\Users\\gress\\OneDrive\\Documents\\Gresa\\DeepKernelGP\\MetaD2A_nas_bench_201\\results', help='the path of save directory') + parser.add_argument('--data-path', type=str, + default='C:\\Users\\gress\\OneDrive\\Documents\\Gresa\\DeepKernelGP\\MetaD2A_nas_bench_201\\data', help='the path of save directory') + parser.add_argument('--save-epoch', type=int, default=400, + help='how many epochs to wait each time to save model states') + parser.add_argument('--max-epoch', type=int, default=400, + help='number of epochs to train') + parser.add_argument('--batch_size', type=int, default=1, + help='batch size for generator') + parser.add_argument('--graph-data-name', + default='nasbench201', help='graph dataset name') + parser.add_argument('--nvt', type=int, default=7, + help='number of different node types, 7: NAS-Bench-201 including in/out node') + # set encoder + parser.add_argument('--num-sample', type=int, default=20, + help='the number of images as input for set encoder') + # graph encoder + parser.add_argument('--hs', type=int, default=56, + help='hidden size of GRUs') + parser.add_argument('--nz', type=int, default=56, + help='the number of dimensions of latent vectors z') + # test + parser.add_argument('--test', action='store_true', + default=True, help='turn on test mode') + parser.add_argument('--load-epoch', type=int, default=400, + help='checkpoint epoch loaded for meta-test') + parser.add_argument('--data-name', type=str, + default=None, help='meta-test dataset name') + parser.add_argument('--num-class', type=int, default=None, + help='the number of class of dataset') + parser.add_argument('--num-gen-arch', type=int, default=800, + help='the number of candidate architectures generated by the generator') + parser.add_argument('--train-arch', type=str2bool, default=True, + help='whether to train the searched architecture') + + args = parser.parse_args() + + return args + + +def main(): + args = get_parser() + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu + device = torch.device("cuda:0") + torch.cuda.manual_seed(args.seed) + torch.manual_seed(args.seed) + np.random.seed(args.seed) + random.seed(args.seed) + + if not os.path.exists(args.save_path): + os.makedirs(args.save_path) + args.model_path = os.path.join(args.save_path, args.model_name, 'model') + if not os.path.exists(args.model_path): + os.makedirs(args.model_path) + + if args.model_name == 'generator': + g = Generator(args) + if args.test: + g.meta_test() + else: + g.meta_train() + elif args.model_name == 'predictor': + p = Predictor(args) + if args.test: + p.meta_test() + else: + p.meta_train() + else: + raise ValueError('You should select generator|predictor|train_arch') + + +if __name__ == '__main__': + main() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/metad2a_utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/metad2a_utils.py new file mode 100644 index 0000000..a203032 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/metad2a_utils.py @@ -0,0 +1,315 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import os +import time +import igraph +import random +import numpy as np +import scipy.stats +import argparse +import torch +import logging + + +def reset_seed(seed): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + random.seed(seed) + torch.backends.cudnn.deterministic = True + +def restore_checkpoint(ckpt_dir, state, device, resume=False): + if not resume: + os.makedirs(os.path.dirname(ckpt_dir), exist_ok=True) + return state + elif not os.path.exists(ckpt_dir): + if not os.path.exists(os.path.dirname(ckpt_dir)): + os.makedirs(os.path.dirname(ckpt_dir)) + logging.warning(f"No checkpoint found at {ckpt_dir}. " + f"Returned the same state as input") + return state + else: + loaded_state = torch.load(ckpt_dir, map_location=device) + for k in state: + if k in ['optimizer', 'model', 'ema']: + state[k].load_state_dict(loaded_state[k]) + else: + state[k] = loaded_state[k] + return state + + +def load_graph_config(graph_data_name, nvt, data_path): + if graph_data_name is 'nasbench201': + g_list = [] + max_n = 0 # maximum number of nodes + ms = torch.load(os.path.join( + data_path, f'{graph_data_name}.pt'))['arch']['matrix'] + for i in range(len(ms)): + g, n = decode_NAS_BENCH_201_8_to_igraph(ms[i]) + max_n = max(max_n, n) + g_list.append((g, 0)) + # number of different node types including in/out node + graph_config = {} + graph_config['num_vertex_type'] = nvt # original types + start/end types + graph_config['max_n'] = max_n # maximum number of nodes + graph_config['START_TYPE'] = 0 # predefined start vertex type + graph_config['END_TYPE'] = 1 # predefined end vertex type + elif graph_data_name is 'ofa': + max_n = 20 + # nvt = 27 + graph_config = {} + graph_config['num_vertex_type'] = nvt + 2 # original types + start/end types + graph_config['max_n'] = max_n + 2 # maximum number of nodes + graph_config['START_TYPE'] = 0 # predefined start vertex type + graph_config['END_TYPE'] = 1 # predefined end vertex type + else: + raise NotImplementedError(graph_data_name) + return graph_config + + +def decode_NAS_BENCH_201_8_to_igraph(row): + if type(row) == str: + row = eval(row) # convert string to list of lists + n = len(row) + g = igraph.Graph(directed=True) + g.add_vertices(n) + for i, node in enumerate(row): + g.vs[i]['type'] = node[0] + if i < (n - 2) and i > 0: + g.add_edge(i, i + 1) # always connect from last node + for j, edge in enumerate(node[1:]): + if edge == 1: + g.add_edge(j, i) + return g, n + + +def is_valid_NAS201(g, START_TYPE=0, END_TYPE=1): + # first need to be a valid DAG computation graph + res = is_valid_DAG(g, START_TYPE, END_TYPE) + # in addition, node i must connect to node i+1 + res = res and len(g.vs['type']) == 8 + res = res and not (0 in g.vs['type'][1:-1]) + res = res and not (1 in g.vs['type'][1:-1]) + return res + + +def decode_igraph_to_NAS201_matrix(g): + m = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]] + xys = [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] + for i, xy in enumerate(xys): + m[xy[0]][xy[1]] = float(g.vs[i + 1]['type']) - 2 + import numpy + return numpy.array(m) + + +def decode_igraph_to_NAS_BENCH_201_string(g): + if not is_valid_NAS201(g): + return None + m = decode_igraph_to_NAS201_matrix(g) + types = ['none', 'skip_connect', 'nor_conv_1x1', + 'nor_conv_3x3', 'avg_pool_3x3'] + return '|{}~0|+|{}~0|{}~1|+|{}~0|{}~1|{}~2|'.\ + format(types[int(m[1][0])], + types[int(m[2][0])], types[int(m[2][1])], + types[int(m[3][0])], types[int(m[3][1])], types[int(m[3][2])]) + + +def is_valid_DAG(g, START_TYPE=0, END_TYPE=1): + res = g.is_dag() + n_start, n_end = 0, 0 + for v in g.vs: + if v['type'] == START_TYPE: + n_start += 1 + elif v['type'] == END_TYPE: + n_end += 1 + if v.indegree() == 0 and v['type'] != START_TYPE: + return False + if v.outdegree() == 0 and v['type'] != END_TYPE: + return False + return res and n_start == 1 and n_end == 1 + + +class Accumulator(): + def __init__(self, *args): + self.args = args + self.argdict = {} + for i, arg in enumerate(args): + self.argdict[arg] = i + self.sums = [0] * len(args) + self.cnt = 0 + + def accum(self, val): + val = [val] if type(val) is not list else val + val = [v for v in val if v is not None] + assert (len(val) == len(self.args)) + for i in range(len(val)): + if torch.is_tensor(val[i]): + val[i] = val[i].item() + self.sums[i] += val[i] + self.cnt += 1 + + def clear(self): + self.sums = [0] * len(self.args) + self.cnt = 0 + + def get(self, arg, avg=True): + i = self.argdict.get(arg, -1) + assert (i is not -1) + if avg: + return self.sums[i] / (self.cnt + 1e-8) + else: + return self.sums[i] + + def print_(self, header=None, time=None, + logfile=None, do_not_print=[], as_int=[], + avg=True): + msg = '' if header is None else header + ': ' + if time is not None: + msg += ('(%.3f secs), ' % time) + + args = [arg for arg in self.args if arg not in do_not_print] + arg = [] + for arg in args: + val = self.sums[self.argdict[arg]] + if avg: + val /= (self.cnt + 1e-8) + if arg in as_int: + msg += ('%s %d, ' % (arg, int(val))) + else: + msg += ('%s %.4f, ' % (arg, val)) + print(msg) + + if logfile is not None: + logfile.write(msg + '\n') + logfile.flush() + + def add_scalars(self, summary, header=None, tag_scalar=None, + step=None, avg=True, args=None): + for arg in self.args: + val = self.sums[self.argdict[arg]] + if avg: + val /= (self.cnt + 1e-8) + else: + val = val + tag = f'{header}/{arg}' if header is not None else arg + if tag_scalar is not None: + summary.add_scalars(main_tag=tag, + tag_scalar_dict={tag_scalar: val}, + global_step=step) + else: + summary.add_scalar(tag=tag, + scalar_value=val, + global_step=step) + + +class Log: + def __init__(self, args, logf, summary=None): + self.args = args + self.logf = logf + self.summary = summary + self.stime = time.time() + self.ep_sttime = None + + def print(self, logger, epoch, tag=None, avg=True): + if tag == 'train': + ct = time.time() - self.ep_sttime + tt = time.time() - self.stime + msg = f'[total {tt:6.2f}s (ep {ct:6.2f}s)] epoch {epoch:3d}' + print(msg) + self.logf.write(msg+'\n') + logger.print_(header=tag, logfile=self.logf, avg=avg) + + if self.summary is not None: + logger.add_scalars( + self.summary, header=tag, step=epoch, avg=avg) + logger.clear() + + def print_args(self): + argdict = vars(self.args) + print(argdict) + for k, v in argdict.items(): + self.logf.write(k + ': ' + str(v) + '\n') + self.logf.write('\n') + + def set_time(self): + self.stime = time.time() + + def save_time_log(self): + ct = time.time() - self.stime + msg = f'({ct:6.2f}s) meta-training phase done' + print(msg) + self.logf.write(msg+'\n') + + def print_pred_log(self, loss, corr, tag, epoch=None, max_corr_dict=None): + if tag == 'train': + ct = time.time() - self.ep_sttime + tt = time.time() - self.stime + msg = f'[total {tt:6.2f}s (ep {ct:6.2f}s)] epoch {epoch:3d}' + self.logf.write(msg+'\n') + print(msg) + self.logf.flush() + # msg = f'ep {epoch:3d} ep time {time.time() - ep_sttime:8.2f} ' + # msg += f'time {time.time() - sttime:6.2f} ' + if max_corr_dict is not None: + max_corr = max_corr_dict['corr'] + max_loss = max_corr_dict['loss'] + msg = f'{tag}: loss {loss:.6f} ({max_loss:.6f}) ' + msg += f'corr {corr:.4f} ({max_corr:.4f})' + else: + msg = f'{tag}: loss {loss:.6f} corr {corr:.4f}' + self.logf.write(msg+'\n') + print(msg) + self.logf.flush() + + def max_corr_log(self, max_corr_dict): + corr = max_corr_dict['corr'] + loss = max_corr_dict['loss'] + epoch = max_corr_dict['epoch'] + msg = f'[epoch {epoch}] max correlation: {corr:.4f}, loss: {loss:.6f}' + self.logf.write(msg+'\n') + print(msg) + self.logf.flush() + + +def get_log(epoch, loss, y_pred, y, acc_std, acc_mean, tag='train'): + msg = f'[{tag}] Ep {epoch} loss {loss.item()/len(y):0.4f} ' + if type(y_pred) == list: + msg += f'pacc {y_pred[0]:0.4f}' + msg += f'({y_pred[0]*100.0*acc_std+acc_mean:0.4f}) ' + else: + msg += f'pacc {y_pred:0.4f}' + msg += f'({y_pred*100.0*acc_std+acc_mean:0.4f}) ' + msg += f'acc {y[0]:0.4f}({y[0]*100*acc_std+acc_mean:0.4f})' + return msg + + +def load_model(model, model_path, load_epoch=None, load_max_pt=None): + if load_max_pt is not None: + ckpt_path = os.path.join(model_path, load_max_pt) + else: + ckpt_path = os.path.join(model_path, f'ckpt_{load_epoch}.pt') + print(f"==> load model from {ckpt_path} ...") + model.cpu() + model.load_state_dict(torch.load(ckpt_path)) + + +def save_model(epoch, model, model_path, max_corr=None): + print("==> save current model...") + if max_corr is not None: + torch.save(model.cpu().state_dict(), + os.path.join(model_path, 'ckpt_max_corr.pt')) + else: + torch.save(model.cpu().state_dict(), + os.path.join(model_path, f'ckpt_{epoch}.pt')) + + +def mean_confidence_interval(data, confidence=0.95): + a = 1.0 * np.array(data) + n = len(a) + m, se = np.mean(a), scipy.stats.sem(a) + h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1) + return m, h diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/__init__.py new file mode 100644 index 0000000..f76c2e0 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/__init__.py @@ -0,0 +1,6 @@ +from pathlib import Path +import sys +dir_path = (Path(__file__).parent).resolve() +if str(dir_path) not in sys.path: sys.path.insert(0, str(dir_path)) + +from .architecture import train_single_model \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/architecture.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/architecture.py new file mode 100644 index 0000000..2a890a9 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/architecture.py @@ -0,0 +1,173 @@ +############################################################### +# NAS-Bench-201, ICLR 2020 (https://arxiv.org/abs/2001.00326) # +############################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +############################################################### +from functions import evaluate_for_seed +from nas_bench_201_models import CellStructure, CellArchitectures, get_search_spaces +from log_utils import Logger, AverageMeter, time_string, convert_secs2time +from nas_bench_201_datasets import get_datasets +from procedures import get_machine_info +from procedures import save_checkpoint, copy_checkpoint +from config_utils import load_config +from pathlib import Path +from copy import deepcopy +import os +import sys +import time +import torch +import random +import argparse +from PIL import ImageFile + +ImageFile.LOAD_TRUNCATED_IMAGES = True + + +NASBENCH201_CONFIG_PATH = os.path.join( + os.getcwd(), 'meta_nas', 'TNAS-DCS', 'MetaD2A_nas_bench_201') + + +def evaluate_all_datasets(arch, datasets, xpaths, splits, use_less, seed, + arch_config, workers, logger): + machine_info, arch_config = get_machine_info(), deepcopy(arch_config) + all_infos = {'info': machine_info} + all_dataset_keys = [] + # look all the datasets + for dataset, xpath, split in zip(datasets, xpaths, splits): + # train valid data + task = None + train_data, valid_data, xshape, class_num = get_datasets( + dataset, xpath, -1, task) + + # load the configuration + if dataset in ['mnist', 'svhn', 'aircraft', 'pets']: + if use_less: + config_path = os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/LESS.config') + else: + config_path = os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/{}.config'.format(dataset)) + + p = os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/{:}-split.txt'.format(dataset)) + if not os.path.exists(p): + import json + label_list = list(range(len(train_data))) + random.shuffle(label_list) + strlist = [str(label_list[i]) for i in range(len(label_list))] + splited = {'train': ["int", strlist[:len(train_data) // 2]], + 'valid': ["int", strlist[len(train_data) // 2:]]} + with open(p, 'w') as f: + f.write(json.dumps(splited)) + split_info = load_config(os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/{:}-split.txt'.format(dataset)), None, None) + else: + raise ValueError('invalid dataset : {:}'.format(dataset)) + + config = load_config( + config_path, {'class_num': class_num, 'xshape': xshape}, logger) + # data loader + train_loader = torch.utils.data.DataLoader(train_data, batch_size=config.batch_size, + shuffle=True, num_workers=workers, pin_memory=True) + valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=config.batch_size, + shuffle=False, num_workers=workers, pin_memory=True) + splits = load_config(os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/{}-test-split.txt'.format(dataset)), None, None) + ValLoaders = {'ori-test': valid_loader, + 'x-valid': torch.utils.data.DataLoader(valid_data, batch_size=config.batch_size, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + splits.xvalid), + num_workers=workers, pin_memory=True), + 'x-test': torch.utils.data.DataLoader(valid_data, batch_size=config.batch_size, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + splits.xtest), + num_workers=workers, pin_memory=True) + } + dataset_key = '{:}'.format(dataset) + if bool(split): + dataset_key = dataset_key + '-valid' + logger.log( + 'Evaluate ||||||| {:10s} ||||||| Train-Num={:}, Valid-Num={:}, Train-Loader-Num={:}, Valid-Loader-Num={:}, batch size={:}'. + format(dataset_key, len(train_data), len(valid_data), len(train_loader), len(valid_loader), config.batch_size)) + logger.log('Evaluate ||||||| {:10s} ||||||| Config={:}'.format( + dataset_key, config)) + for key, value in ValLoaders.items(): + logger.log( + 'Evaluate ---->>>> {:10s} with {:} batchs'.format(key, len(value))) + + results = evaluate_for_seed( + arch_config, config, arch, train_loader, ValLoaders, seed, logger) + all_infos[dataset_key] = results + all_dataset_keys.append(dataset_key) + all_infos['all_dataset_keys'] = all_dataset_keys + return all_infos + + +def train_single_model(save_dir, workers, datasets, xpaths, splits, use_less, + seeds, model_str, arch_config): + assert torch.cuda.is_available(), 'CUDA is not available.' + torch.backends.cudnn.enabled = True + torch.backends.cudnn.deterministic = True + torch.set_num_threads(workers) + + save_dir = Path(save_dir) + logger = Logger(str(save_dir), 0, False) + + if model_str in CellArchitectures: + arch = CellArchitectures[model_str] + logger.log( + 'The model string is found in pre-defined architecture dict : {:}'.format(model_str)) + else: + try: + arch = CellStructure.str2structure(model_str) + except: + raise ValueError( + 'Invalid model string : {:}. It can not be found or parsed.'.format(model_str)) + + assert arch.check_valid_op(get_search_spaces( + 'cell', 'nas-bench-201')), '{:} has the invalid op.'.format(arch) + # assert arch.check_valid_op(get_search_spaces('cell', 'full')), '{:} has the invalid op.'.format(arch) + logger.log('Start train-evaluate {:}'.format(arch.tostr())) + logger.log('arch_config : {:}'.format(arch_config)) + + start_time, seed_time = time.time(), AverageMeter() + for _is, seed in enumerate(seeds): + logger.log( + '\nThe {:02d}/{:02d}-th seed is {:} ----------------------<.>----------------------'.format(_is, len(seeds), + seed)) + to_save_name = save_dir / 'seed-{:04d}.pth'.format(seed) + if to_save_name.exists(): + logger.log( + 'Find the existing file {:}, directly load!'.format(to_save_name)) + checkpoint = torch.load(to_save_name) + else: + logger.log( + 'Does not find the existing file {:}, train and evaluate!'.format(to_save_name)) + checkpoint = evaluate_all_datasets(arch, datasets, xpaths, splits, use_less, + seed, arch_config, workers, logger) + torch.save(checkpoint, to_save_name) + # log information + logger.log('{:}'.format(checkpoint['info'])) + all_dataset_keys = checkpoint['all_dataset_keys'] + for dataset_key in all_dataset_keys: + logger.log('\n{:} dataset : {:} {:}'.format( + '-' * 15, dataset_key, '-' * 15)) + dataset_info = checkpoint[dataset_key] + # logger.log('Network ==>\n{:}'.format( dataset_info['net_string'] )) + logger.log('Flops = {:} MB, Params = {:} MB'.format( + dataset_info['flop'], dataset_info['param'])) + logger.log('config : {:}'.format(dataset_info['config'])) + logger.log('Training State (finish) = {:}'.format( + dataset_info['finish-train'])) + last_epoch = dataset_info['total_epoch'] - 1 + train_acc1es, train_acc5es = dataset_info['train_acc1es'], dataset_info['train_acc5es'] + valid_acc1es, valid_acc5es = dataset_info['valid_acc1es'], dataset_info['valid_acc5es'] + # measure elapsed time + seed_time.update(time.time() - start_time) + start_time = time.time() + need_time = 'Time Left: {:}'.format(convert_secs2time( + seed_time.avg * (len(seeds) - _is - 1), True)) + logger.log( + '\n<<<***>>> The {:02d}/{:02d}-th seed is {:} other procedures need {:}'.format(_is, len(seeds), seed, + need_time)) + logger.close() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/config_utils/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/config_utils/__init__.py new file mode 100644 index 0000000..2d57bbd --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/config_utils/__init__.py @@ -0,0 +1,13 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from .configure_utils import load_config, dict2config#, configure2str +#from .basic_args import obtain_basic_args +#from .attention_args import obtain_attention_args +#from .random_baseline import obtain_RandomSearch_args +#from .cls_kd_args import obtain_cls_kd_args +#from .cls_init_args import obtain_cls_init_args +#from .search_single_args import obtain_search_single_args +#from .search_args import obtain_search_args +# for network pruning +#from .pruning_args import obtain_pruning_args diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/config_utils/configure_utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/config_utils/configure_utils.py new file mode 100644 index 0000000..125e68e --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/config_utils/configure_utils.py @@ -0,0 +1,106 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# +import os, json +from os import path as osp +from pathlib import Path +from collections import namedtuple + +support_types = ('str', 'int', 'bool', 'float', 'none') + + +def convert_param(original_lists): + assert isinstance(original_lists, list), 'The type is not right : {:}'.format(original_lists) + ctype, value = original_lists[0], original_lists[1] + assert ctype in support_types, 'Ctype={:}, support={:}'.format(ctype, support_types) + is_list = isinstance(value, list) + if not is_list: value = [value] + outs = [] + for x in value: + if ctype == 'int': + x = int(x) + elif ctype == 'str': + x = str(x) + elif ctype == 'bool': + x = bool(int(x)) + elif ctype == 'float': + x = float(x) + elif ctype == 'none': + if x.lower() != 'none': + raise ValueError('For the none type, the value must be none instead of {:}'.format(x)) + x = None + else: + raise TypeError('Does not know this type : {:}'.format(ctype)) + outs.append(x) + if not is_list: outs = outs[0] + return outs + + +def load_config(path, extra, logger): + path = str(path) + if hasattr(logger, 'log'): logger.log(path) + assert os.path.exists(path), 'Can not find {:}'.format(path) + # Reading data back + with open(path, 'r') as f: + data = json.load(f) + content = { k: convert_param(v) for k,v in data.items()} + assert extra is None or isinstance(extra, dict), 'invalid type of extra : {:}'.format(extra) + if isinstance(extra, dict): content = {**content, **extra} + Arguments = namedtuple('Configure', ' '.join(content.keys())) + content = Arguments(**content) + if hasattr(logger, 'log'): logger.log('{:}'.format(content)) + return content + + +def configure2str(config, xpath=None): + if not isinstance(config, dict): + config = config._asdict() + def cstring(x): + return "\"{:}\"".format(x) + def gtype(x): + if isinstance(x, list): x = x[0] + if isinstance(x, str) : return 'str' + elif isinstance(x, bool) : return 'bool' + elif isinstance(x, int): return 'int' + elif isinstance(x, float): return 'float' + elif x is None : return 'none' + else: raise ValueError('invalid : {:}'.format(x)) + def cvalue(x, xtype): + if isinstance(x, list): is_list = True + else: + is_list, x = False, [x] + temps = [] + for temp in x: + if xtype == 'bool' : temp = cstring(int(temp)) + elif xtype == 'none': temp = cstring('None') + else : temp = cstring(temp) + temps.append( temp ) + if is_list: + return "[{:}]".format( ', '.join( temps ) ) + else: + return temps[0] + + xstrings = [] + for key, value in config.items(): + xtype = gtype(value) + string = ' {:20s} : [{:8s}, {:}]'.format(cstring(key), cstring(xtype), cvalue(value, xtype)) + xstrings.append(string) + Fstring = '{\n' + ',\n'.join(xstrings) + '\n}' + if xpath is not None: + parent = Path(xpath).resolve().parent + parent.mkdir(parents=True, exist_ok=True) + if osp.isfile(xpath): os.remove(xpath) + with open(xpath, "w") as text_file: + text_file.write('{:}'.format(Fstring)) + return Fstring + + +def dict2config(xdict, logger): + assert isinstance(xdict, dict), 'invalid type : {:}'.format( type(xdict) ) + Arguments = namedtuple('Configure', ' '.join(xdict.keys())) + content = Arguments(**xdict) + if hasattr(logger, 'log'): logger.log('{:}'.format(content)) + return content diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft-split.txt b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft-split.txt new file mode 100644 index 0000000..420ab52 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft-split.txt @@ -0,0 +1 @@ +{"train": ["int", ["353", "297", "1508", "3700", "1221", "4489", "1279", "1420", "2306", "3538", "4301", "6301", "3437", "2175", "3779", "2024", "1036", "3696", "2544", "183", "129", "2917", "5420", "3094", "448", "4018", "4037", "1639", "6070", "1308", "1385", "159", "1632", "2845", "1282", "1041", "4112", "1096", "5893", "4918", "4307", "947", "2214", "2432", "1428", "2792", "827", "3922", "163", "2545", "5992", "2226", "6196", "4349", "1959", "1287", "4743", "529", "2642", "1269", "169", "1101", "2806", "1289", "2339", "5739", "5974", "616", "641", "5863", "6401", "138", "853", "1920", "1579", "1018", "4631", "644", "6030", "6285", "4359", "787", "50", "2948", "2475", "1651", "749", "5652", "4145", "4910", "3281", "3008", "5124", "1190", "5499", "3651", "1753", "5908", "3501", "5762", "2639", "2136", "1068", "2018", "6054", "5422", "319", "6209", "2184", "4833", "4001", "848", "2454", "1625", "4028", "3309", "1780", "4041", "3275", "236", "146", "4903", "5123", "2089", "383", "1233", "6493", "4646", "1186", "282", "1706", "1586", "3968", "4450", "3217", "4731", "2983", "1871", "6332", "6528", "3680", "6323", "1459", "3173", "1846", "3916", "6001", "4197", "181", "5521", "2198", "540", "881", "2891", "4211", "5433", "502", "2627", "3264", "3427", "2274", "6116", "3608", "3761", "5052", "2686", "815", "517", "3679", "2511", "1359", "2918", "5876", "3688", "417", "4560", "3304", "1461", "1567", "6617", "1636", "725", "5125", "2495", "1343", "3792", "3631", "4105", "6313", "4084", "4474", "1141", "4070", "3739", "3117", "2446", "1763", "3092", "3382", "6056", "6282", "4959", "3808", "1734", "2912", "2543", "4870", "416", "4510", "3611", "4233", "5221", "5791", "3978", "4065", "741", "4315", "2700", "2922", "6057", "2085", "4064", "70", "5444", "5373", "4171", "2162", "2393", "793", "3570", "3328", "2381", "4430", "273", "5891", "4865", "1032", "978", "5960", "6290", "6601", "4935", "2611", "2573", "186", "2257", "4680", "556", "922", "4292", "2091", "4251", "3153", "355", "1656", "4284", "5353", "3604", "1374", "6067", "4773", "2848", "263", "3620", "3492", "786", "1777", "6291", "1611", "2582", "6328", "6300", "2844", "1000", "1996", "2603", "3389", "2646", "6106", "2488", "3440", "3592", "2326", "5136", "5849", "2414", "2205", "4906", "4297", "1779", "2609", "1881", "3012", "5393", "4864", "2663", "5379", "3154", "6366", "5180", "3388", "974", "3099", "705", "1425", "2797", "241", "5256", "5906", "6337", "3637", "3719", "4242", "477", "5912", "4883", "5979", "102", "1979", "5182", "5248", "5179", "5654", "5068", "6038", "1180", "5015", "1213", "2579", "5178", "1836", "5878", "2493", "3692", "2104", "5619", "4837", "6214", "1346", "2787", "5846", "2285", "5498", "1371", "2254", "5252", "3957", "5724", "5304", "4869", "3880", "215", "3358", "2476", "6269", "109", "1467", "3004", "6092", "5228", "3401", "5594", "4772", "3757", "5291", "4702", "729", "4893", "145", "5094", "6403", "1223", "2608", "2741", "4729", "4767", "4508", "717", "3764", "3548", "1446", "2980", "3959", "6452", "5454", "1315", "2228", "2439", "711", "1892", "4795", "132", "3227", "2408", "268", "4787", "6161", "4710", "3073", "4535", "5750", "4716", "470", "2378", "3799", "6115", "6397", "2155", "2385", "2702", "4106", "731", "1382", "4309", "5768", "85", "6251", "433", "5045", "1342", "2813", "6554", "269", "3954", "5066", "645", "1193", "5613", "158", "3614", "3736", "2139", "1578", "1761", "4180", "630", "3036", "4139", "3508", "6651", "5369", "1702", "5164", "4032", "667", "2643", "4638", "712", "2443", "5410", "1793", "4666", "1873", "1434", "2805", "618", "4920", "170", "949", "4066", "3018", "5133", "6274", "2824", "850", "1898", "1230", "4520", "3194", "4518", "2688", "4400", "120", "5501", "2245", "5735", "689", "1367", "4834", "2547", "2991", "5036", "3913", "3840", "182", "272", "3741", "4498", "3030", "3331", "5163", "2087", "4867", "3067", "3778", "184", "4790", "4281", "5797", "3949", "2769", "6469", "561", "500", "6433", "4868", "5772", "5343", "1037", "6218", "3259", "5612", "4368", "1715", "3556", "6232", "4687", "2862", "5112", "5222", "1176", "4945", "1712", "5610", "5358", "3726", "2019", "3533", "4338", "117", "728", "4998", "2357", "3587", "5671", "785", "6280", "6655", "5289", "2788", "6281", "742", "2667", "4219", "5617", "4671", "5909", "1545", "5577", "5371", "167", "991", "1004", "6024", "5894", "3393", "2153", "3576", "609", "1549", "5914", "1940", "1680", "1758", "5208", "4046", "5564", "699", "3539", "3263", "4625", "162", "2768", "6372", "1162", "2218", "61", "5811", "6358", "4609", "5493", "5645", "1500", "1302", "4569", "2206", "190", "6191", "3509", "4383", "4079", "5709", "2516", "1408", "3303", "1803", "2499", "5737", "6253", "1513", "1095", "2850", "2561", "5349", "3821", "1510", "5022", "1456", "880", "4504", "2456", "5720", "2647", "6224", "1557", "6576", "4433", "1022", "4851", "15", "3253", "6559", "1398", "3702", "1016", "990", "5098", "6414", "1654", "6564", "2574", "876", "4916", "5384", "2719", "1368", "4996", "549", "5131", "5440", "3781", "6456", "3837", "5438", "1054", "4187", "5421", "2956", "3469", "3052", "1896", "5717", "3948", "6471", "134", "4469", "3131", "3554", "3989", "1890", "5245", "4696", "5952", "1334", "2635", "3586", "5079", "6396", "1575", "6474", "6625", "4614", "5559", "3392", "463", "2617", "3877", "1159", "6027", "4823", "3262", "6222", "3879", "3288", "342", "833", "1730", "1865", "395", "5598", "3885", "6558", "6168", "2857", "954", "6249", "4794", "1478", "1143", "6388", "5403", "3462", "4546", "5188", "1745", "25", "2597", "4882", "147", "4363", "3476", "5360", "295", "3182", "5192", "6006", "426", "2133", "2503", "4708", "3488", "4624", "579", "969", "4892", "620", "5941", "1818", "2380", "2933", "3616", "2654", "1164", "6177", "2262", "3057", "6387", "6354", "4375", "4688", "2337", "4650", "4565", "2035", "6446", "953", "4423", "6480", "2324", "4779", "5780", "5110", "2340", "5159", "6357", "4013", "3813", "1447", "1533", "4086", "4470", "567", "2708", "2802", "6666", "3843", "710", "5665", "3633", "2426", "4177", "3338", "1662", "814", "3046", "2296", "6049", "2697", "1133", "4927", "4291", "5773", "3685", "5745", "3349", "4580", "692", "671", "798", "713", "1417", "3863", "3138", "2689", "4445", "5126", "908", "5799", "4038", "1108", "5731", "1307", "4373", "359", "6499", "1377", "3907", "6434", "605", "6202", "2739", "2197", "944", "5474", "6551", "3710", "2103", "1453", "2410", "4681", "5678", "6594", "886", "2086", "4311", "315", "6100", "367", "3322", "1337", "5365", "6190", "3617", "6490", "3672", "3355", "921", "933", "1633", "4044", "2520", "6662", "2669", "4228", "1250", "5286", "4352", "2204", "4481", "3325", "2297", "1084", "114", "352", "1700", "2926", "2533", "1266", "5447", "4290", "3222", "2243", "3901", "1294", "402", "5058", "4815", "1331", "5530", "6592", "3225", "2100", "1201", "1415", "1214", "4941", "4353", "334", "5861", "6255", "3215", "1433", "5806", "3662", "5176", "92", "4796", "677", "1986", "3932", "5234", "1594", "4922", "153", "960", "6364", "5989", "4119", "3861", "2990", "2776", "1551", "2429", "5943", "5888", "5653", "5088", "4250", "1197", "1457", "2468", "5023", "4357", "4673", "3870", "5885", "2195", "2773", "5507", "4384", "6533", "541", "3001", "5647", "2812", "6308", "1659", "4751", "4765", "2981", "1392", "2679", "368", "4100", "1495", "5477", "5102", "6318", "5784", "4750", "2584", "2693", "2430", "532", "2444", "1167", "3876", "5971", "3411", "370", "6521", "4806", "365", "4525", "5032", "4928", "5657", "3290", "1318", "2856", "3632", "5169", "6296", "6504", "2368", "3039", "3814", "591", "5322", "640", "2463", "394", "4737", "1637", "819", "1861", "2487", "1320", "3144", "494", "6611", "121", "2025", "1587", "530", "6311", "1484", "4253", "3640", "4402", "2477", "3795", "3177", "4122", "1189", "1869", "2765", "4499", "6152", "1254", "2709", "6110", "5316", "3838", "5378", "1825", "3090", "5007", "2309", "866", "5319", "5432", "1653", "5430", "6389", "1015", "1469", "4031", "3579", "380", "1439", "1345", "842", "2178", "1713", "3578", "2771", "2622", "2382", "1782", "5586", "1390", "4453", "3350", "3200", "3610", "2557", "2447", "2766", "2004", "5852", "358", "5582", "5736", "1182", "580", "1583", "259", "1416", "6044", "36", "5025", "4052", "1120", "3206", "3518", "4637", "1692", "1458", "4263", "6495", "1914", "3976", "5732", "5470", "5263", "6046", "721", "6131", "3463", "792", "5040", "4963", "4904", "3318", "526", "2552", "5154", "1409", "6626", "3786", "3037", "1695", "194", "6413", "511", "3911", "4958", "778", "2853", "4191", "3179", "3329", "5510", "5419", "6093", "5385", "4782", "2030", "1704", "1463", "3402", "4340", "732", "5999", "4296", "1610", "1739", "5375", "4482", "4936", "5505", "3622", "4552", "2694", "4455", "2799", "6299", "1418", "6121", "4436", "806", "4929", "1746", "6418", "3735", "1395", "2268", "2938", "3753", "411", "4861", "6546", "1370", "3314", "3744", "5427", "6599", "3782", "1379", "1234", "4006", "5718", "3058", "776", "5656", "2594", "788", "4985", "2116", "4917", "3252", "3762", "555", "5931", "4978", "5832", "5364", "6343", "1966", "6653", "4658", "161", "3909", "438", "6488", "2684", "2207", "1630", "5526", "6574", "4838", "783", "5218", "5287", "281", "951", "5324", "3845", "1882", "1930", "686", "258", "6320", "3921", "2063", "3981", "6465", "3130", "4392", "2015", "977", "6160", "3659", "4190", "1623", "5965", "4590", "1612", "1444", "3984", "1569", "1943", "5986", "759", "2389", "1145", "2143", "1481", "2179", "2361", "53", "4835", "3963", "2872", "2467", "5155", "305", "2402", "2901", "2868", "5758", "6476", "325", "586", "6518", "5867", "2836", "3363", "1121", "2231", "3009", "6284", "276", "3283", "4254", "1786", "3890", "1743", "3113", "943", "5927", "3201", "6406", "3014", "5351", "1171", "6326", "6120", "32", "4378", "2832", "3526", "505", "434", "3063", "2831", "3317", "5864", "5030", "1631", "1874", "2596", "4390", "4540", "3655", "480", "5063", "1030", "5417", "774", "4280", "1954", "2672", "4619", "939", "4797", "6245", "5335", "5242", "403", "2615", "1488", "4711", "1460", "5770", "340", "4227", "3718", "6085", "534", "2677", "3772", "870", "4460", "5865", "5260", "3552", "64", "6162", "1466", "3022", "1155", "5691", "4099", "1240", "5347", "928", "5457", "3224", "5991", "649", "2506", "1244", "5540", "2093", "2259", "3649", "3952", "1490", "4347", "3656", "3807", "5545", "4434", "2671", "2395", "498", "1634", "5877", "3207", "5284", "5550", "1231", "1941", "1241", "1783", "860", "2110", "3161", "1210", "3277", "1009", "2532", "4757", "2889", "3040", "3162", "1312", "2121", "5792", "6472", "2177", "2734", "4428", "4734", "375", "5636", "4379", "5475", "997", "726", "4500", "6264", "2288", "5630", "43", "5368", "3858", "1430", "727", "5314", "6355", "3791", "5936", "5666", "1412", "1899", "5301", "4529", "5836", "3646", "4246", "2974", "4783", "3385", "4090", "2398", "2505", "4770", "6400", "2062", "4255", "11", "5407", "3432", "6119", "2676", "3769", "2349", "1910", "3021", "4094", "5320", "3585", "107", "2989", "3118", "2601", "6086", "155", "6542", "1124", "5240", "5519", "26", "5608", "3226", "1868", "1917", "5122", "2423", "4507", "4490", "6602", "3312", "976", "1356", "16", "2106", "4769", "3313", "2920", "6565", "5686", "301", "5592", "5118", "6060", "472", "3396", "171", "6512", "5303", "1034", "6404", "1265", "4209", "6612", "2675", "2556", "1615", "1386", "542", "635", "5840", "2365", "2664", "2898", "2618", "1701", "3205", "5220", "266", "1194", "3944", "934", "4131", "5013", "1541", "3896", "3733", "767", "3448", "808", "2842", "4042", "593", "4372", "1123", "2673", "2351", "3724", "2987", "1581", "3918", "2281", "6622", "1310", "1923", "5667", "2541", "5558", "6132", "4059", "4193", "2046", "4194", "5344", "2459", "204", "2629", "4050", "1086", "5254", "6314", "3789", "2001", "4643", "2466", "225", "3924", "1698", "285", "3694", "781", "6608", "2605", "6000", "51", "4855", "3746", "5197", "2818", "6112", "2563", "3887", "3720", "622", "5600", "2911", "5078", "4008", "3849", "1813", "6464", "569", "5047", "407", "3912", "4566", "4889", "5361", "3897", "3690", "2729", "570", "5231", "5027", "2986", "839", "88", "3522", "5451", "3738", "3519", "5973", "6013", "310", "1383", "3599", "6650", "924", "3607", "6236", "22", "5402", "2904", "6419", "5010", "3681", "5541", "3788", "5398", "6429", "5689", "4639", "2353", "157", "2455", "4973", "1948", "1607", "5853", "5938", "5191", "1722", "3302", "588", "1227", "3430", "3687", "4322", "42", "2440", "4273", "5255", "1997", "1509", "3323", "4425", "1062", "5295", "4647", "3472", "5243", "443", "1005", "4938", "5148", "5902", "91", "2370", "1135", "4793", "4279", "2098", "6322", "5298", "1727", "4685", "3193", "1902", "4239", "4326", "6008", "1788", "1506", "5658", "5743", "3450", "1968", "369", "1192", "2314", "3908", "6500", "387", "6237", "2316", "4351", "4532", "4067", "757", "3446", "2327", "3269", "1565", "3671", "5818", "3120", "4113", "175", "663", "4977", "604", "21", "4257", "4839", "4404", "4813", "5275", "2129", "3032", "4715", "1975", "6640", "2366", "6283", "4801", "5334", "103", "4885", "1901", "6117", "6439", "1908", "279", "2558", "351", "6325", "1388", "6180", "5409", "2464", "2486", "2758", "4585", "1817", "5411", "2379", "2344", "562", "3005", "5659", "1964", "475", "3818", "5579", "537", "1489", "5822", "4144", "882", "3537", "3336", "3653", "6195", "4431", "6059", "1017", "445", "6302", "3089", "1151", "4181", "929", "3810", "6002", "1126", "1626", "544", "4695", "226", "3682", "1932", "4584", "6654", "5083", "2222", "3920", "4720", "4995", "4226", "1056", "6463", "2772", "4411", "1807", "3499", "1830", "2554", "2567", "2302", "4606", "4942", "1665", "2746", "1003", "476", "6614", "4661", "6124", "941", "205", "5714", "5549", "4564", "2273", "3601", "4447", "524", "3839", "2176", "4116", "2383", "4899", "293", "2017", "633", "4323", "5485", "1907", "5121", "4994", "2251", "965", "937", "608", "4803", "2187", "745", "2137", "460", "5340", "4495", "2358", "3424", "5370", "3701", "3003", "6099", "572", "5825", "2119", "333", "1251", "3024", "4825", "2973", "3156", "3078", "849", "4810", "3102", "6373", "2542", "879", "423", "2249", "3580", "6367", "4339", "4331", "3170", "1720", "701", "5363", "1911", "2113", "2691", "5556", "2994", "4205", "1682", "5193", "1759", "2341", "3790", "660", "5597", "6369", "2029", "3854", "1255", "1413", "2685", "5627", "6089", "3641", "560", "1178", "4449", "179", "4150", "1590", "5527", "1369", "1906", "4694", "4117", "5651", "2784", "3712", "3531", "1949", "2587", "3197", "5581", "321", "5381", "596", "4604", "5416", "4189", "4472", "2566", "1951", "4759", "1006", "3953", "3219", "4512", "6012", "6005", "1351", "3803", "1767", "356", "2075", "2263", "5465", "4303", "5962", "583", "230", "6052", "5804", "1260", "2301", "948", "6567", "2804", "1449", "3266", "2821", "6201", "6443", "5929", "5376", "2992", "5602", "1553", "1564", "349", "3468", "5548", "4168", "1304", "2478", "4215", "378", "6226", "4983", "5021", "2894", "1237", "2614", "4645", "2472", "5157", "3958", "1350", "6126", "5459", "1429", "5502", "1518", "6205", "5859", "1160", "4475", "617", "3449", "2144", "1960", "6609", "5640", "4847", "200", "673", "4674", "3722", "4186", "4015", "3237", "2120", "1750", "3732", "6383", "5786", "5512", "5844", "100", "2125", "4771", "1747", "5563", "3101", "4503", "4136", "1546", "2756", "752", "124", "5001", "4328", "3077", "4778", "3979", "3609", "3731", "639", "5681", "5237", "309", "5662", "2422", "6288", "2419", "4617", "274", "5236", "3243", "3639", "4578", "390", "5081", "6442", "3321", "4416", "3511", "2182", "4184", "4547", "2640", "3119", "4554", "5265", "2334", "6598", "4408", "5837", "4109", "3435", "6538", "611", "3900", "896", "4517", "2749", "2481", "6462", "5870", "6356", "5624", "1820", "2884", "2915", "4808", "5162", "487", "700", "1538", "2965", "4824", "6658", "6604", "994", "1330", "5803", "3245", "4799", "1601", "5207", "6505", "6141", "4156", "2286", "2967", "4321", "4831", "1536", "3439", "5450", "6150", "5639", "4966", "473", "3334", "4212", "3645", "2825", "219", "4034", "2660", "5244", "3083", "1181", "3787", "3709", "602", "4760", "116", "1707", "3181", "6184", "2092", "2418", "5337", "6064", "4101", "5439", "1921", "966", "5200", "6239", "6079", "4336", "2360", "5134", "1591", "4712", "105", "4817", "1474", "3816", "6553", "3564", "5469", "1148", "289", "1440", "3817", "2272", "2323", "1990", "507", "329", "3155", "5633", "4987", "5423", "3844", "246", "5202", "3628", "1831", "5585", "867", "5146", "653", "5160", "4954", "6415", "257", "558", "577", "3391", "2203", "4842", "52", "4754", "3011", "3525", "4992", "5391", "2082", "311", "1372", "5235", "4979", "3070", "5915", "6143", "1841", "6643", "135", "5135", "2460", "1840", "5262", "3016", "4925", "1644", "420", "3062", "404", "5919", "1833", "441", "5062", "4040", "964", "5881", "6458", "4654", "5328", "2742", "963", "6460", "4011", "3250", "4354", "3972", "3512", "2428", "3835", "1242", "5644", "722", "6315", "2705", "2604", "6479", "5687", "4362", "3917", "1858", "6125", "5143", "6286", "5492", "4004", "5841", "6390", "1547", "6468", "1212", "446", "2572", "2928", "37", "972", "2416", "1061", "1642", "2636", "2988", "1971", "2152", "2232", "3670", "5127", "5428", "5050", "5103", "2551", "1399", "386", "2319", "1198", "656", "4085", "6144", "5390", "877", "4376", "6014", "1122", "5589", "3056", "1468", "5429", "5542", "970", "2307", "4785", "2047", "6164", "1609", "1158", "6584", "3474", "2698", "1311", "4709", "5546", "2996", "6561", "1886", "4123", "5611", "6362", "2060", "2958", "112", "6256", "439", "1035", "73", "1903", "277", "1119", "1859", "6021", "3521", "822", "3220", "3644", "1864", "6379", "2820", "324", "1994", "5685", "1988", "1142", "1573", "4439", "1805", "1051", "4077", "3618", "3994", "5802", "3293", "4002", "995", "4295", "5779", "1483", "1741", "6424", "5302", "323", "6430", "2181", "60", "4768", "1998", "681", "58", "758", "1498", "1635", "2141", "4586", "5934", "5856", "6082", "1687", "2165", "2056", "4523", "1274", "5625", "6007", "1671", "5968", "6304", "450", "4960", "447", "4114", "916", "3775", "512", "5431", "461", "4125", "6133", "4915", "2710", "2839", "5484", "1384", "1070", "1487", "4679", "3369", "1355", "2616", "6440", "5879", "6378", "3433", "5959", "6616", "3190", "4494", "4706", "4158", "501", "6333", "3395", "5729", "863", "3456", "2083", "5216", "2054", "927", "6039", "4258", "4788", "6004", "4651", "3214", "4506", "2271", "6360", "6204", "1809", "4821", "2789", "3105", "4632", "3483", "2294", "6664", "4921", "1309", "3546", "3826", "3581", "5042", "377", "6334", "489", "3763", "3566", "5913", "2623", "4913", "3590", "4655", "1421", "6647", "4395", "5574", "5311", "4780", "1222", "533", "5307", "3752", "3376", "2208", "6203", "678", "4563", "1563", "4202", "4217", "550", "6329", "2167", "1196", "4199", "3375", "2483", "118", "4648", "3723", "6096", "6652", "1649", "242", "2910", "917", "3939", "962", "5082", "797", "5990", "2183", "2377", "6393", "3851", "5707", "4313", "2369", "4748", "3216", "5975", "3163", "4089", "2525", "6257", "2313", "149", "3658", "3630", "2527", "5046", "5487", "2630", "4763", "6545", "3629", "979", "1437", "4974", "971", "2564", "125", "6074", "2390", "1676", "520", "1195", "1375", "6321", "6155", "39", "1136", "24", "4310", "5642", "2795", "5576", "5318", "3134", "4611", "601", "6181", "4231", "3287", "3591", "3380", "3398", "4521", "3750", "1614", "5765", "3882", "4240", "110", "5824", "1984", "900", "6182", "2724", "2151", "3902", "5838", "1074", "6011", "1931", "5744", "1404", "1705", "816", "2223", "3017", "2589", "1602", "6361", "2317", "243", "5723", "2886", "2548", "912", "2641", "6523", "1503", "1718", "388", "255", "5452", "2897", "1184", "4009", "2864", "6073", "5000", "2331", "77", "3431", "3139", "1373", "2701", "1894", "3708", "4438", "4777", "857", "245", "4133", "4138", "5883", "3481", "865", "31", "4025", "1093", "1947", "203", "6170", "3192", "5935", "3172", "0", "1053", "1916", "3447", "261", "2916", "5468", "2950", "1897", "3848", "5267", "6492", "3760", "1674", "4946", "4859", "967", "2849", "1450", "2949", "5551", "2969", "2510", "5018", "1738", "5478", "304", "856", "5629", "5012", "794", "1850", "4926", "4314", "513", "3136", "891", "6477", "13", "2147", "6047", "3482", "4626", "621", "2224", "1751", "872", "4836", "4130", "1329", "5166", "4092", "1327", "3796", "3510", "195", "807", "6187", "1915", "1933", "6425", "5557", "432", "4993", "4229", "1749", "3998", "2265", "3801", "4397", "6473", "1316", "5308", "3947", "5816", "4477", "3823", "4053", "2053", "790", "6103", "3515", "4819", "5290", "5504", "6018", "5703", "1884", "832", "1026", "3589", "5699", "5464", "2867", "5697", "5626", "1118", "6455", "4132", "4509", "104", "5928", "5981", "2256", "1787", "1092", "3927", "3910", "1073", "5573", "5710", "851", "6420", "4789", "4692", "2201", "2038", "1087", "899", "6412", "3045", "4261", "1273", "1360", "4965", "3346", "2237", "6444", "72", "5387", "6385", "19", "316", "2026", "3514", "496", "392", "2823", "5089", "3812", "1792", "1023", "5074", "303", "6581", "2173", "3504", "6094", "4581", "173", "1648", "3996", "75", "3426", "2628", "6457", "1270", "3343", "836", "5185", "4330", "5855", "4621", "2826", "1853", "6588", "6259", "2735", "4952", "1405", "483", "3140", "5238", "4159", "1683", "4493", "1319", "332", "993", "248", "1248", "6470", "4271", "5119", "1130", "3872", "3484", "1226", "4394", "1576", "3020", "5388", "4830", "918", "603", "4317", "2852", "1972", "4023", "2067", "3026", "3588", "3320", "3465", "4391", "2238", "30", "5269", "1519", "3221", "3444", "223", "6016", "1091", "1872", "2755", "495", "3211", "371", "6072", "4223", "4690", "4930", "1021", "6183", "1768", "5833", "5061", "4162", "3028", "4689", "2420", "1725", "3315", "662", "5704", "2896", "5857", "3946", "2645", "6475", "5090", "6003", "3970", "789", "2321", "6381", "33", "1011", "1239", "3353", "2895", "4047", "5967", "4890", "4360", "6208", "574", "6154", "2877", "4324", "6268", "5049", "4413", "4401", "2959", "5056", "6509", "5253", "427", "1977", "3559", "210", "1505", "4218", "5884", "4288", "6037", "521", "3044", "3495", "89", "141", "2138", "1993", "4299", "3714", "3168", "345", "6638", "191", "5174", "5455", "3928", "306", "18", "3409", "4809", "2753", "5145", "5109", "4901", "919", "478", "2593", "5722", "2059", "6656", "3051", "685", "1078", "2032", "5944", "5467", "1641", "3667", "2944", "6417", "3487", "3856", "4318", "575", "3384", "6303", "6287", "1109", "647", "5734", "4435", "4956", "318", "1785", "3704", "322", "1772", "3689", "2751", "559", "106", "1521", "212", "1530", "6055", "4853", "1795", "6615", "5543", "6197", "4691", "2997", "684", "3171", "421", "1476", "1071", "904", "5448", "936", "3091", "3084", "5842", "4541", "874", "1353", "5141", "3379", "2780", "4370", "4152", "3169", "3234", "2128", "894", "3112", "267", "897", "753", "5547", "4599", "2524", "2930", "539", "3904", "2438", "409", "3148", "142", "2261", "2480", "2145", "2666", "284", "2051", "1592", "4487", "6293", "986", "1526", "5655", "2117", "2376", "449", "2346", "2571", "6502", "2652", "5591", "1887", "1014", "5412", "5807", "2935", "6156", "3149", "2637", "6549", "6363", "327", "4111", "424", "357", "6105", "5823", "6530", "2131", "6646", "607", "3425", "2725", "4668", "3265", "2984", "1462", "2491", "140", "3133", "286", "4267", "4539", "4022", "3663", "2109", "1114", "2124", "4786", "1929", "2375", "3955", "4026", "4459", "1952", "6603", "840", "5019", "5462", "2731", "4024", "1103", "2107", "907", "1835", "3648", "3473", "2906", "4080", "1655", "691", "6015", "5752", "2170", "1485", "4335", "1364", "4592", "4305", "4677", "889", "4342", "791", "1152", "2727", "2171", "3598", "1451", "2000", "2461", "626", "5442", "5875", "4545", "518", "5520", "6010", "3819", "1937", "5024", "2266", "69", "299", "5142", "4074", "3804", "5698", "3419", "4722", "3975", "746", "2407", "5246", "6292", "4948", "5170", "1407", "4873", "3347", "1839", "6573", "4260", "2819", "4762", "2391", "2993", "2908", "152", "4900", "1079", "668", "768", "6587", "1283", "1391", "5250", "4511", "2227", "5531", "5637", "3811", "82", "3420", "2633", "4371", "4818", "3883", "4221", "4912", "1721", "1924", "1378", "2384", "4484", "5650", "5227", "5966", "4247", "2796", "280", "2239", "4192", "654", "4420", "2325", "5426", "3408", "456", "6663", "2626", "1380", "1955", "469", "5186", "5815", "59", "5932", "6459", "6569", "4344", "3734", "3486", "4660", "1271", "2570", "3485", "3319", "5108", "2458", "2172", "3678", "3378", "4143", "2250", "5138", "4791", "796", "3903", "1789", "2132", "4451", "4682", "3311", "6556", "3703", "3544", "3241", "582", "5529", "5819", "1925", "2537", "3159", "5866", "1048", "1001", "3373", "4488", "1710", "2080", "1347", "5144", "1691", "453", "1043", "2581", "2764", "1900", "2748", "4659", "2304", "2546", "2442", "1574", "2336", "3255", "3475", "4049", "744", "4237", "2210", "6359", "6382", "5436", "1410", "3926", "4107", "5198", "3866", "160", "4061", "2900", "3218", "5288", "3661", "5201", "5620", "6140", "213", "6478", "1689", "5506", "2388", "3239", "821", "3657", "6618", "4355", "3405", "5998", "1726", "3188", "1580", "1740", "1082", "3551", "772", "4515", "4852", "4531", "4406", "3033", "1494", "5362", "1111", "3852", "3244", "6529", "4259", "4943", "2057", "4276", "2392", "6122", "4419", "2540", "3977", "4633", "565", "5461", "2105", "5677", "4798", "650", "6266", "4588", "2962", "760", "6336", "5528", "3106", "5392", "1826", "4804", "2625", "983", "3942", "5562", "1535", "3164", "2202", "1523", "1796", "1677", "5518", "3410", "5892", "2785", "4456", "6431", "2043", "5325", "320", "2049", "6621", "6138", "838", "4333", "4693", "3971", "3060", "2159", "240", "3434", "1888", "6540", "3478", "5020", "6063", "2058", "6175", "5137", "1638", "5443", "3088", "5900", "3931", "6416", "5778", "330", "6649", "2740", "2851", "1228", "3535", "1522", "1599", "909", "835", "566", "3698", "6327", "4802", "4826", "739", "547", "4350", "878", "2522", "2186", "852", "2513", "5359"]], "valid": ["int", ["2761", "3988", "4157", "1477", "6501", "2846", "2971", "6068", "3561", "4670", "1139", "4312", "6607", "3516", "6025", "4327", "2123", "1187", "594", "1464", "3529", "3634", "1303", "1202", "3524", "5513", "5670", "354", "2621", "4021", "3496", "6294", "6535", "1040", "96", "1340", "4735", "2225", "2767", "5672", "5005", "1183", "292", "5424", "3160", "1987", "1039", "5041", "5149", "3674", "777", "4662", "5071", "740", "5315", "5064", "3842", "6438", "1365", "4811", "6589", "3572", "3438", "6244", "2292", "4587", "1471", "1556", "4805", "17", "1863", "5175", "5437", "66", "4243", "20", "4704", "2371", "144", "3351", "1207", "587", "961", "2424", "4516", "1660", "1115", "3573", "5730", "6305", "3868", "398", "2759", "4991", "5129", "6146", "708", "5596", "901", "6178", "6527", "6338", "5817", "1985", "38", "1694", "1699", "2800", "5839", "6555", "6531", "1020", "4230", "2838", "3754", "4699", "1188", "3523", "3567", "384", "3352", "1291", "4850", "6270", "1829", "65", "4862", "4730", "5215", "2479", "2722", "361", "1280", "130", "1646", "3805", "6563", "4422", "6398", "1161", "5009", "5184", "4567", "2096", "6026", "6606", "4462", "3270", "5898", "3208", "497", "2523", "3143", "3513", "3983", "4471", "6375", "5232", "3297", "6423", "3366", "844", "2600", "5700", "4463", "4820", "2977", "720", "5516", "3991", "4605", "5623", "5400", "3428", "1889", "4879", "4583", "4480", "5233", "1790", "1232", "366", "5224", "3271", "4537", "4911", "3240", "3081", "1652", "6104", "6077", "661", "4068", "3743", "1801", "6258", "709", "4598", "6516", "1153", "2887", "3124", "1719", "1678", "1582", "2150", "5017", "1703", "5869", "2189", "4304", "1507", "4361", "3356", "6075", "4726", "3129", "2624", "1811", "1895", "4964", "914", "6352", "1257", "391", "2448", "5350", "2276", "1658", "535", "3857", "302", "859", "3199", "264", "5794", "372", "4003", "2979", "4293", "6629", "5688", "4062", "4454", "9", "4278", "1616", "4491", "2649", "5406", "6335", "1204", "3452", "440", "216", "5196", "2359", "3406", "3962", "6402", "2598", "5051", "2174", "3574", "3348", "233", "5266", "3251", "5552", "1033", "2157", "1414", "1081", "5093", "3550", "4877", "2526", "211", "6242", "3443", "6295", "2421", "3461", "2023", "2757", "3285", "2330", "239", "2794", "4519", "5632", "4103", "4377", "290", "3086", "6272", "464", "5956", "6254", "980", "6297", "6076", "3292", "1755", "410", "1672", "1776", "682", "3278", "3695", "2576", "5854", "4717", "5945", "6221", "2320", "5769", "3691", "2899", "4738", "4249", "4522", "406", "3774", "3636", "76", "300", "5923", "5616", "5767", "2754", "5719", "6087", "3254", "2931", "2913", "6310", "5926", "4723", "761", "1335", "1154", "6351", "4206", "1879", "2312", "1566", "4947", "762", "63", "5177", "2657", "3098", "887", "2779", "5016", "492", "1597", "5705", "2863", "6578", "4036", "6053", "6137", "5495", "670", "4294", "6593", "4306", "1808", "6525", "5922", "1475", "3455", "5356", "3919", "5065", "1537", "6229", "4248", "6048", "2947", "360", "136", "723", "3103", "1199", "2462", "47", "6628", "5575", "3686", "911", "6118", "5332", "610", "1387", "2995", "4238", "4446", "3986", "3421", "3095", "3507", "735", "1769", "6421", "3833", "373", "4388", "139", "1336", "2322", "4744", "2040", "5297", "5835", "5721", "1441", "5580", "6220", "5777", "3853", "6550", "3123", "3846", "2005", "3974", "6347", "3203", "87", "5105", "799", "6467", "4083", "5848", "5831", "4029", "6506", "4551", "337", "595", "275", "6142", "3906", "1619", "4275", "3684", "6623", "331", "3204", "83", "4623", "2403", "959", "1299", "527", "5987", "592", "2876", "5310", "2003", "4845", "6395", "564", "2373", "5389", "3047", "1515", "14", "3423", "1532", "5886", "4151", "4283", "177", "5708", "328", "4745", "3480", "3491", "2084", "5257", "2078", "250", "841", "3274", "1203", "6083", "2827", "6324", "5982", "1465", "5895", "2354", "5279", "6200", "5039", "3235", "481", "4127", "989", "1013", "3142", "3128", "3937", "6575", "6627", "4172", "296", "2270", "4544", "6199", "1963", "3471", "1400", "563", "3399", "5329", "3498", "5957", "3184", "1268", "3992", "4232", "3666", "1735", "5323", "5241", "5489", "4102", "6485", "430", "6630", "646", "1482", "98", "2801", "2909", "2387", "5910", "973", "4485", "2127", "5401", "3638", "484", "5486", "6645", "6496", "885", "3500", "3990", "3956", "4170", "4909", "4137", "335", "2166", "5404", "5590", "1965", "6234", "4937", "1128", "3135", "341", "4538", "1854", "1137", "2457", "5276", "3737", "1647", "6031", "6227", "898", "1217", "2869", "4286", "5583", "5746", "3097", "2591", "3697", "5114", "49", "5760", "4822", "313", "4269", "115", "992", "5566", "1491", "2115", "1138", "1981", "1473", "2242", "317", "824", "431", "4962", "6547", "1389", "5712", "1668", "3560", "3257", "1185", "1845", "1918", "1402", "2501", "1419", "5272", "1613", "5456", "1499", "2190", "4414", "6078", "4010", "2102", "3534", "224", "4556", "1756", "2295", "1643", "6017", "1571", "4501", "736", "1496", "5048", "6128", "5533", "5209", "40", "3100", "4196", "3654", "2496", "5115", "3441", "4019", "4282", "1348", "648", "545", "1814", "2562", "5539", "6306", "6562", "2332", "4060", "782", "6449", "6", "1554", "4204", "4676", "743", "6298", "643", "5603", "5903", "1125", "5958", "4486", "5950", "4951", "1028", "6033", "6198", "5525", "4270", "3555", "5003", "5259", "5496", "1502", "3569", "1117", "557", "5116", "4713", "3041", "923", "2723", "2275", "1775", "4128", "2299", "454", "4988", "2810", "3413", "3023", "6447", "1577", "3993", "1857", "5858", "956", "1666", "166", "1224", "5414", "6583", "3337", "2101", "4898", "5282", "5199", "672", "2942", "2114", "4142", "150", "4081", "1798", "1089", "228", "2363", "5535", "2180", "3665", "5331", "6631", "3126", "482", "3517", "688", "1325", "4595", "1256", "4950", "2033", "6350", "6243", "831", "2752", "3802", "5983", "4872", "1628", "2298", "2404", "3371", "4533", "28", "6041", "3527", "2193", "2837", "6040", "2308", "1002", "4203", "5044", "4045", "5537", "6661", "869", "2396", "2946", "1298", "3829", "488", "5055", "2498", "6157", "3025", "4126", "619", "3127", "2362", "6644", "3943", "2333", "5173", "1066", "3360", "1366", "4562", "3414", "1525", "3707", "5445", "2620", "6632", "1834", "4774", "2482", "766", "5692", "548", "1149", "2400", "5029", "5277", "2854", "6410", "5206", "3836", "2471", "6071", "4174", "3489", "2347", "2531", "2791", "5808", "4496", "6579", "4667", "2998", "950", "5213", "2834", "1822", "3326", "3342", "3841", "3247", "1531", "4386", "5397", "6307", "2474", "737", "5357", "350", "5247", "4561", "5132", "6062", "6605", "719", "6665", "4173", "2399", "4905", "6508", "4665", "3176", "1501", "4369", "1332", "5251", "1134", "1992", "3174", "6216", "5453", "5588", "3664", "4756", "5172", "5925", "2580", "5595", "4088", "2777", "486", "4268", "5212", "2763", "2955", "3238", "4016", "5405", "6036", "3422", "2728", "5156", "764", "2858", "820", "3711", "389", "467", "5268", "4348", "2163", "3381", "4440", "2079", "3082", "6503", "2278", "2592", "1970", "260", "2122", "2394", "1696", "4752", "4135", "6101", "2502", "795", "2881", "1448", "4207", "1973", "6597", "5638", "3929", "6312", "5139", "6186", "1852", "2606", "4980", "5342", "625", "168", "5503", "5377", "6319", "6532", "2213", "3593", "6231", "338", "1427", "2405", "3950", "2452", "4982", "2585", "4418", "86", "6212", "6586", "95", "5565", "4367", "5845", "3699", "6481", "1055", "6211", "5035", "5664", "3675", "4091", "952", "5418", "4272", "1891", "847", "811", "185", "2880", "206", "581", "1909", "3114", "2061", "3248", "1065", "4129", "4146", "5810", "2586", "5511", "3693", "642", "4784", "4989", "3894", "1555", "2303", "174", "5151", "2744", "599", "6498", "892", "2036", "6148", "6377", "3945", "3066", "414", "3562", "3627", "3256", "80", "2515", "694", "6526", "3260", "4502", "3505", "938", "5813", "3186", "3213", "674", "5171", "3493", "5299", "3074", "5554", "2234", "451", "1146", "6108", "5479", "5584", "4678", "1019", "3824", "154", "1824", "5230", "4841", "6613", "1252", "1235", "3715", "4285", "837", "4134", "6185", "4753", "4616", "3855", "2662", "1288", "4740", "1175", "623", "3683", "3605", "363", "2097", "3279", "4319", "773", "2068", "5570", "6136", "344", "231", "1452", "3467", "1560", "6277", "10", "6028", "5482", "5043", "3368", "2658", "5508", "3300", "4907", "1220", "5748", "5104", "553", "81", "5948", "4166", "1967", "5158", "2726", "3464", "955", "5037", "6577", "108", "945", "45", "1225", "4437", "4185", "5396", "6097", "381", "3166", "2284", "4033", "2536", "412", "5820", "5978", "5961", "2191", "5890", "1757", "192", "4", "3650", "207", "1760", "2052", "6610", "3884", "4235", "5782", "1261", "3571", "884", "6129", "1936", "5993", "931", "4858", "6639", "2865", "984", "5771", "5793", "2720", "5872", "4443", "2010", "5994", "262", "5294", "493", "2866", "3416", "5646", "4005", "1572", "3964", "2730", "465", "5996", "3310", "2968", "2786", "6189", "68", "1935", "1396", "5727", "4577", "636", "861", "5181", "5985", "94", "3951", "3272", "4627", "2560", "606", "3340", "2549", "5701", "750", "1669", "1667", "6341", "5458", "4407", "2012", "1177", "5725", "1305", "5002", "405", "4596", "4683", "1794", "4536", "1247", "613", "2951", "2492", "5386", "6483", "6497", "854", "1711", "2497", "4827", "998", "4630", "2683", "800", "5140", "3387", "522", "3365", "90", "2441", "5694", "2775", "4570", "4387", "3565", "5946", "4589", "4986", "6022", "1165", "3284", "2164", "244", "6519", "41", "5214", "657", "2318", "5194", "5933", "6235", "1823", "1604", "1559", "3280", "3982", "4208", "5605", "519", "4076", "999", "1962", "576", "1219", "3059", "4277", "2300", "1593", "2168", "2583", "4051", "3895", "3542", "1296", "2632", "6380", "202", "1622", "5330", "3647", "3335", "2963", "5776", "2200", "1455", "830", "4399", "3860", "4078", "6368", "5631", "3158", "5270", "1716", "3985", "3995", "3859", "4955", "3065", "2269", "2619", "3705", "3930", "1107", "1116", "4886", "1341", "1904", "543", "2924", "4473", "5634", "1173", "6636", "62", "2717", "4896", "5026", "536", "5481", "6090", "2610", "218", "5273", "55", "3506", "3327", "4999", "716", "1693", "1497", "4875", "6374", "2372", "4183", "5695", "199", "2013", "3966", "4098", "5728", "5601", "6051", "1640", "5435", "5084", "6107", "5352", "1934", "637", "5904", "397", "5413", "2199", "590", "703", "1529", "1276", "2305", "3436", "2489", "6123", "2064", "4332", "5607", "6524", "4356", "2260", "2934", "193", "5380", "1424", "5028", "4664", "6489", "6571", "2130", "6641", "5072", "382", "4576", "3575", "78", "2807", "4724", "271", "5969", "2230", "1913", "4409", "197", "122", "6346", "3768", "817", "5787", "2185", "4188", "1661", "1838", "2449", "4967", "890", "3888", "459", "5790", "2927", "4075", "5555", "283", "4163", "1804", "6165", "2737", "509", "2830", "4458", "2682", "664", "1511", "3602", "3372", "1045", "3362", "4953", "4844", "6517", "6240", "3776", "6279", "2328", "5281", "5684", "2699", "1744", "6035", "1961", "6029", "3236", "5911", "1357", "2450", "5604", "3806", "2790", "2921", "5471", "5514", "6514", "3626", "48", "5190", "462", "4919", "3151", "3961", "2975", "4396", "3759", "4747", "4874", "1939", "127", "222", "1527", "4736", "312", "396", "1029", "624", "4733", "5106", "5372", "2077", "2578", "6376", "5789", "4214", "3043", "4057", "4104", "362", "5834", "5382", "1435", "4884", "2569", "733", "5761", "1542", "3568", "5970", "2879", "3915", "1050", "3289", "4728", "1791", "5317", "5150", "3923", "5572", "379", "1411", "5764", "571", "1275", "6088", "4828", "6487", "5229", "1300", "5809", "738", "5814", "4364", "6042", "2007", "23", "1200", "176", "2638", "5187", "2530", "3583", "3677", "1765", "2048", "4505", "3122", "3454", "2519", "3797", "883", "5751", "3935", "6111", "3668", "6466", "1063", "4860", "4972", "2142", "5920", "1976", "2529", "528", "2070", "2050", "6169", "3676", "6548", "4634", "629", "2002", "4412", "5466", "2885", "5977", "6316", "4476", "1098", "6436", "2445", "3822", "2733", "3809", "5713", "4213", "503", "6173", "308", "3010", "925", "942", "2412", "8", "627", "5008", "4175", "4976", "2409", "1290", "294", "1262", "3530", "1922", "5441", "5649", "3898", "2291", "5092", "802", "1847", "1077", "2721", "1174", "525", "2960", "1208", "3189", "1621", "6163", "5", "3196", "5850", "4087", "3547", "6262", "4981", "2335", "2732", "4289", "2229", "4072", "1728", "5754", "3178", "2954", "3013", "1543", "178", "123", "2809", "2612", "1285", "227", "982", "4649", "5067", "2156", "3758", "4035", "1272", "5939", "2348", "6151", "1540", "3249", "2154", "3766", "1982", "3210", "1862", "4225", "3273", "2940", "376", "2258", "3121", "4393", "1843", "4701", "2835", "2855", "4198", "2814", "3460", "5346", "1215", "2874", "2066", "5955", "2374", "1259", "3370", "810", "4902", "56", "2484", "5749", "3079", "3034", "1688", "2235", "2248", "2678", "1049", "1163", "452", "3780", "422", "4939", "6570", "6552", "3050", "5614", "1097", "1338", "1528", "1588", "6642", "5781", "5567", "4382", "1827", "1094", "5963", "2453", "3390", "2602", "2535", "3110", "5497", "1422", "1627", "4672", "2656", "1956", "5491", "573", "2882", "237", "1596", "3282", "3109", "3457", "958", "4641", "6331", "2507", "552", "6241", "3232", "3878", "1875", "198", "5682", "3069", "5167", "652", "2212", "6147", "3258", "2937", "5759", "6515", "3532", "5690", "3965", "1754", "1883", "2859", "3299", "4714", "5383", "6176", "5211", "1534", "5168", "775", "2665", "1969", "2293", "2782", "2607", "4432", "1403", "4082", "1245", "4154", "6409", "3417", "4417", "34", "3941", "6034", "2188", "4607", "5449", "2192", "3625", "4148", "5099", "5271", "1733", "2736", "6624", "578", "4017", "2952", "3494", "5733", "458", "5847", "2822", "2843", "1216", "2500", "968", "3451", "6544", "3869", "35", "5117", "5086", "3798", "4876", "217", "4115", "3147", "3673", "2099", "631", "4358", "4365", "1844", "1246", "508", "1815", "2674", "3111", "4636", "3019", "2277", "2028", "1106", "2888", "4816", "3899", "2508", "119", "4176", "6510", "5726", "2433", "5128", "2413", "2661", "6659", "846", "1060", "4684", "6066", "3276", "5766", "5293", "2431", "2287", "1172", "6450", "2815", "5693", "3545", "1358", "3577", "748", "429", "5096", "5560", "172", "3404", "2634", "3006", "1752", "5742", "2069", "1057", "3407", "4478", "920", "6134", "4971", "5930", "2706", "3157", "3553", "903", "1595", "1010", "1842", "3107", "3793", "3967", "3889", "126", "6513", "2355", "2350", "4897", "554", "6408", "1263", "2841", "4256", "1731", "5683", "5918", "4334", "1295", "996", "2923", "2465", "251", "5544", "278", "2148", "2209", "981", "3048", "3643", "1293", "506", "4832", "2711", "2762", "6344", "1267", "5524", "2718", "2436", "6271", "3053", "3660", "3031", "1629", "1090", "1292", "5336", "1729", "873", "5828", "2401", "845", "1816", "2945", "4698", "5292", "5483", "665", "5073", "4934", "2650", "1806", "5339", "4568", "5183", "4337", "2976", "1486", "5374", "3333", "5643", "3185", "4341", "3751", "6261", "4640", "5901", "5473", "5147", "4742", "2695", "4932", "4514", "5415", "148", "5621", "1608", "4071", "5606", "5500", "5569", "1169", "2220", "4866", "4719", "6411", "5680", "754", "1363", "1800", "5434", "2668", "6461", "4096", "3767", "1880", "6345", "4766", "1600", "4571", "4557", "680", "1218", "3747", "385", "5204", "4579", "913", "2112", "5571", "4201", "1052", "988", "3027", "3470", "4161", "5054", "1709", "2936", "3202", "1771", "2135", "5006", "2081", "2042", "1598", "6427", "5882", "1008", "2707", "2264", "6069", "1393", "1585", "2902", "2470", "3815", "1322", "3261", "4574", "4167", "4467", "6520", "3296", "3015", "4325", "6192", "4725", "1558", "3383", "718", "1905", "5321", "4236", "3594", "6167", "1673", "2252", "1083", "3477", "3748", "3582", "3132", "232", "3339", "5980", "600", "6210", "326", "3386", "5783", "910", "6238", "3429", "3831", "3233", "4840", "4969", "5394", "6486", "4300", "6339", "5821", "3442", "4054", "6633", "2828", "6171", "1104", "1442", "1301", "4466", "1147", "6590", "209", "2829", "6252", "747", "3490", "6009", "2553", "3847", "1492", "6127", "2703", "3865", "598", "1044", "5532", "6230", "2041", "2870", "2982", "2961", "2469", "6207", "4843", "131", "5711", "3558", "1031", "2939", "3061", "5561", "2941", "5014", "5219", "4241", "2415", "1837", "5296", "3619", "3400", "2241", "2555", "6453", "5798", "1436", "2712", "249", "687", "1870", "499", "3771", "5702", "5460", "3191", "4656", "666", "5280", "4613", "1072", "704", "2217", "3543", "1946", "3341", "1168", "1284", "5333", "287", "1127", "5873", "3936", "1110", "6267", "2781", "1166", "479", "1799", "6371", "805", "5153", "2343", "4739", "2194", "5964", "1926", "4069", "1438", "2538", "2883", "515", "468", "2108", "3007", "4642", "1983", "1112", "6135", "5225", "5663", "2774", "5152", "298", "1957", "5100", "4030", "4302", "2367", "3459", "5300", "444", "1075", "5628", "1802", "1764", "418", "4746", "4329", "4961", "4628", "3364", "769", "235", "2", "4461", "4265", "6065", "3115", "1099", "4141", "4700", "2893", "428", "4548", "1950", "6619", "2435", "1974", "5217", "2490", "437", "675", "926", "5305", "5203", "4707", "6386", "3934", "128", "3344", "4732", "1849", "1685", "669", "336", "3783", "4881", "6392", "3354", "756", "706", "164", "1774", "2517", "2750", "1562", "113", "5354", "4618", "4718", "5924", "5988", "4316", "3875", "3377", "1209", "1603", "4891", "5669", "3093", "5899", "5517", "1670", "5679", "5327", "3324", "4888", "4758", "1206", "5889", "597", "3596", "4346", "4058", "2565", "676", "291", "3756", "1645", "638", "6166", "2932", "3886", "2681", "1999", "6113", "4908", "4308", "2919", "485", "2966", "1663", "1742", "3613", "895", "2512", "2282", "3064", "531", "4857", "829", "2094", "4427", "6582", "1810", "695", "3717", "5755", "2280", "2021", "3301", "1570", "6536", "1361", "5070", "1179", "2770", "1038", "2247", "5676", "6494", "3730", "3332", "801", "4675", "6637", "2985", "2907", "5897", "2833", "6091", "1454", "1828", "6591", "3850", "1860", "4653", "6342", "1443", "2045", "6491", "5887", "1406", "1708", "2890", "1724", "4800", "4120", "6289", "1314", "702", "1426", "6061", "6275", "307", "5949", "3359", "1323", "6620", "4266", "930", "1618", "6534", "612", "5953", "4097", "1", "905", "201", "5326", "5210", "3394", "6159", "6217", "3669", "4014", "4615", "5355", "4741", "6043", "3071", "2311", "2356", "4924", "4856", "3549", "6432", "6391", "3230", "2073", "2134", "4559", "4764", "4374", "3862", "5097", "2860", "2233", "2760", "2427", "4686", "4441", "2219", "3635", "1942", "466", "585", "2970", "3728", "4234", "2039", "6441", "4421", "4448", "2118", "1995", "4721", "5091", "2111", "4492", "3940", "2680", "2494", "252", "2577", "4385", "2716", "987", "3765", "1144", "5463", "6585", "1058", "734", "3049", "54", "2840", "4582", "79", "2595", "510", "4697", "165", "5312", "4121", "1617", "5261", "3000", "679", "3502", "214", "5615", "3367", "3183", "6246", "932", "5077", "6153", "4483", "3777", "2599", "823", "5851", "4020", "1324", "2651", "6384", "3773", "3871", "3223", "4244", "4043", "1024", "4871", "5480", "3874", "2169", "4140", "690", "813", "3729", "5747", "3749", "2037", "4997", "1085", "4398", "4949", "1989", "2687", "779", "3584", "1867", "2009", "2425", "1945", "1690", "4846", "6370", "1397", "868", "3867", "1504", "658", "4410", "3167", "6580", "3075", "4274", "1253", "1445", "5345", "3345", "6566", "1512", "5753", "3563", "568", "5536", "2216", "3029", "4550", "1520", "1131", "3716", "1317", "855", "4612", "5408", "4573", "3642", "67", "4095", "589", "1953", "765", "516", "4601", "6102", "1249", "1762", "4608", "4497", "6543", "2575", "5706", "5107", "4894", "5741", "1781", "3597", "2504", "809", "435", "4457", "1278", "1848", "3721", "1297", "1686", "5599", "5075", "6276", "5995", "634", "6596", "288", "4933", "715", "6095", "3615", "1238", "2873", "6179", "1514", "4262", "6019", "413", "5161", "714", "2146", "5034", "940", "1697", "1381", "902", "455", "3187", "1980", "4165", "2011", "6250", "3212", "6648", "4940", "1376", "1025", "5812", "4878", "3612", "4622", "628", "5534", "6139", "84", "2518", "2957", "2071", "1748", "4389", "2692", "615", "1100", "6188", "3745", "6445", "2847", "3096", "3864", "2386", "6265", "2670", "3770", "1229", "2417", "4591", "693", "3116", "1878", "5057", "4093", "5553", "5367", "229", "2055", "1958", "5472", "1067", "6399", "3054", "2020", "1944", "5674", "6353", "1561", "6330", "4530", "6278", "1129", "5976", "6020", "5101", "1281", "2905", "6228", "1339", "6223", "784", "3072", "6511", "6174", "238", "2485", "3466", "4669", "2008", "5366", "2095", "3825", "5205", "523", "3146", "1431", "4124", "44", "6248", "1258", "1080", "314", "6435", "5843", "2747", "5278", "3755", "2539", "4153", "4705", "5947", "1927", "2031", "6206", "2437", "2006", "2246", "6045", "4534", "5756", "1832", "2925", "3137", "347", "5757", "5130", "1277", "3891", "4468", "3246", "4381", "3195", "1877", "4895", "2160", "6145", "2648", "3740", "1313", "3784", "4380", "1321", "1732", "6233", "5917", "1589", "1354", "2875", "5716", "2253", "97", "3624", "1042", "1978", "3960", "4776", "1236", "4426", "491", "6482", "2738", "683", "1352", "780", "5004", "3987", "3291", "3520", "2473", "1650", "3938", "1885", "6098", "6273", "5827", "3080", "137", "3652", "2655", "2653", "5715", "3832", "1723", "4012", "1102", "5285", "2798", "5258", "3800", "5587", "3", "5896", "471", "975", "1912", "4792", "4155", "1548", "5648", "3165", "2929", "2715", "101", "4957", "6348", "5011", "3830", "2196", "2808", "4245", "3374", "6172", "4442", "5538", "1606", "3600", "2644", "858", "3412", "4975", "6557", "3528", "3445", "538", "4880", "4048", "1851", "5522", "2745", "4527", "6365", "5829", "985", "6260", "6213", "1856", "2509", "3706", "364", "5871", "425", "5223", "1919", "4849", "3038", "208", "3242", "4549", "2434", "6539", "3725", "5826", "6194", "4703", "6149", "133", "2022", "3294", "2215", "6522", "5618", "4178", "5905", "4000", "256", "2074", "3268", "3152", "4224", "4593", "4970", "5274", "915", "6247", "957", "5673", "2690", "5880", "1012", "3606", "2016", "4657", "1778", "1328", "348", "6428", "4775", "651", "3595", "697", "5940", "3905", "3175", "3361", "5795", "632", "343", "5069", "3827", "818", "5801", "6451", "5830", "3820", "2783", "6050", "4968", "4118", "1493", "2076", "346", "5661", "751", "2550", "4452", "2149", "474", "1770", "4220", "730", "1876", "6158", "906", "3087", "1306", "2778", "1552", "4553", "5738", "419", "3231", "871", "6219", "4558", "5490", "6660", "2568", "4424", "546", "4644", "4343", "5165", "4345", "2044", "6484", "4761", "1479", "834", "4629", "4610", "1657", "1516", "888", "5111", "401", "5523", "5060", "4603", "4405", "2364", "2559", "5399", "6657", "5076", "5341", "4149", "3479", "1855", "2090", "400", "2743", "5984", "1191", "4366", "5874", "1046", "4526", "399", "3125", "1264", "1344", "99", "698", "2065", "6225", "4055", "4147", "3330", "2290", "6568", "4575", "2345", "270", "1243", "5053", "5568", "5951", "5031", "6560", "5494", "5309", "4572", "5264", "6595", "2793", "504", "5785", "3892", "2236", "5578", "1812", "6572", "151", "1286", "5446", "3150", "2528", "4524", "3306", "1544", "5059", "2878", "3497", "3881", "7", "812", "5593", "5476", "6317", "189", "3541", "2803", "4403", "2158", "3742", "1326", "4063", "46", "4110", "724", "770", "5868", "3180", "5641", "5788", "234", "6600", "4848", "6405", "5338", "2817", "490", "6081", "3828", "2240", "221", "862", "1150", "875", "1991", "1423", "3999", "4812", "457", "2978", "1211", "2659", "5937", "6193", "5226", "3914", "2811", "1170", "4195", "864", "1938", "6507", "1893", "3228", "6541", "1736", "4108", "220", "6537", "2315", "755", "4465", "3557", "3308", "2714", "1069", "415", "2338", "3068", "696", "436", "1432", "3973", "1717", "1675", "893", "2871", "2914", "3893", "6634", "2861", "2140", "5622", "2713", "3458", "2411", "5954", "3002", "1007", "4222", "1047", "2289", "4200", "5774", "3403", "5425", "1684", "4513", "93", "6263", "247", "5862", "2161", "4755", "1105", "1157", "2072", "6635", "4056", "2451", "5038", "6454", "5085", "1064", "5997", "74", "2892", "6080", "4415", "2534", "4829", "1027", "2590", "2283", "3536", "803", "187", "3415", "1333", "2903", "2613", "5921", "5775", "826", "29", "1088", "6114", "614", "2088", "4179", "5668", "1866", "1714", "5488", "5120", "3209", "3104", "3397", "1737", "3316", "2221", "5348", "2126", "6084", "1584", "4444", "5033", "2342", "6394", "12", "1059", "4464", "2279", "3229", "1076", "5313", "1113", "2816", "2953", "4944", "4073", "6407", "4931", "3145", "1679", "1539", "4543", "5763", "2034", "5515", "442", "3453", "1928", "843", "3727", "5805", "3286", "5800", "4216", "5696", "3141", "2310", "4479", "374", "1766", "156", "5972", "1524", "4620", "4210", "2943", "6023", "3035", "6215", "6109", "935", "4652", "804", "3055", "4781", "1664", "180", "6032", "4600", "111", "4252", "763", "3267", "1568", "3085", "707", "4182", "3540", "4814", "2211", "4923", "1681", "3969", "3933", "4320", "1605", "4914", "514", "3503", "3785", "1470", "3357", "2267", "1773", "5509", "3603", "5675", "4027", "2972", "2514", "659", "6437", "5113", "655", "6130", "1620", "3295", "408", "5249", "2352", "3997", "2014", "4863", "4160", "5609", "4287", "1401", "4555", "27", "57", "4594", "2631", "2027", "4749", "1517", "5283", "5087", "946", "4007", "6349", "1472", "4597", "1205", "6448", "3042", "188", "4887", "1394", "5796", "2244", "1156", "2999", "551", "2588", "4169", "196", "5195", "4663", "3713", "3305", "4635", "4602", "265", "4298", "4429", "825", "3873", "6422", "3925", "5189", "1819", "1349", "4854", "1140", "5306", "5660", "1480", "3621", "4807", "1784", "5395", "5080", "1550", "5740", "339", "3623", "5860", "1797", "2521", "393", "3794", "143", "4984", "4039", "1362", "3298", "3307", "584", "1132", "4164", "3108", "2964", "2329", "4727", "2704", "4264", "3834", "5942", "6309", "2696", "5916", "1821", "2397", "5239", "4990", "828", "254", "4542", "5095", "3198", "5907", "6058", "4528", "2255", "6426", "253", "5635", "3980", "3418", "3076", "6340", "1624", "71", "771", "2406"]]} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft-test-split.txt b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft-test-split.txt new file mode 100644 index 0000000..d2d9a0e --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft-test-split.txt @@ -0,0 +1 @@ +{"xvalid": ["int", ["187", "699", "1190", "586", "1102", "1457", "2151", "1560", "1326", "1204", "1493", "2146", "436", "2196", "2314", "492", "765", "2743", "3254", "2563", "789", "1855", "2831", "911", "2350", "198", "1020", "1931", "1018", "257", "3176", "1164", "139", "3142", "1433", "236", "380", "441", "1441", "1054", "1274", "1928", "152", "2703", "294", "2300", "1565", "350", "3277", "2077", "742", "2933", "1026", "1275", "1133", "2765", "1926", "170", "1938", "1352", "2592", "914", "2287", "385", "2494", "2202", "1902", "2371", "202", "444", "1953", "1620", "1650", "2587", "3242", "3132", "2470", "1836", "2571", "1272", "1717", "2032", "968", "1675", "2716", "2481", "1778", "1236", "2505", "1588", "1544", "2557", "870", "3172", "1619", "2473", "2058", "610", "926", "830", "1414", "1227", "754", "1944", "1205", "493", "647", "2278", "1764", "1247", "2135", "1061", "593", "2810", "2123", "828", "727", "356", "842", "1862", "2789", "2600", "1219", "2472", "958", "1676", "2246", "976", "3215", "723", "461", "807", "1662", "578", "360", "3259", "401", "1613", "1438", "330", "2119", "2628", "2569", "1770", "2461", "1253", "1107", "2219", "30", "1538", "1952", "2764", "2280", "623", "1791", "3191", "668", "1858", "1327", "1660", "2180", "2936", "140", "2468", "1372", "1682", "1989", "1271", "979", "1044", "1502", "485", "642", "2378", "363", "1988", "2550", "2871", "2034", "3038", "3291", "59", "1835", "690", "515", "881", "2804", "2931", "1244", "701", "1376", "1890", "1324", "357", "2904", "427", "305", "740", "487", "2006", "871", "573", "3219", "703", "1104", "1088", "2160", "1285", "954", "1130", "197", "3028", "1394", "562", "3253", "2850", "3207", "878", "2293", "1689", "2945", "1170", "2026", "3153", "2103", "199", "661", "358", "2545", "1805", "584", "1771", "2247", "2236", "1740", "1622", "862", "1543", "1004", "3318", "2869", "1332", "377", "1569", "1762", "653", "3025", "3124", "3167", "349", "758", "669", "636", "505", "1156", "1062", "2564", "1254", "2948", "1353", "471", "2780", "2649", "1319", "1413", "1445", "1723", "117", "973", "3310", "2093", "33", "372", "2647", "2319", "2190", "3326", "2374", "1111", "295", "599", "322", "3092", "2747", "3007", "1708", "2626", "2445", "2060", "2705", "2140", "1651", "2239", "3112", "2195", "480", "3123", "2594", "2412", "861", "1551", "3329", "1473", "2456", "821", "2892", "115", "1339", "2339", "1663", "2", "2718", "868", "683", "1895", "3101", "2847", "1399", "1305", "2506", "1630", "804", "2538", "2420", "1564", "1516", "1496", "863", "997", "1366", "119", "1687", "1962", "1943", "183", "1742", "707", "3224", "3054", "1669", "2048", "2234", "2429", "1615", "424", "241", "1914", "1179", "213", "3286", "706", "84", "1267", "2197", "889", "3000", "2851", "2916", "2108", "2157", "3208", "341", "2259", "2465", "2453", "3057", "260", "292", "875", "825", "1861", "169", "2029", "229", "2706", "912", "28", "1000", "490", "2223", "2828", "694", "1908", "752", "3022", "3311", "3147", "621", "2109", "510", "1900", "144", "3013", "2059", "961", "101", "35", "3218", "1896", "43", "2188", "3190", "672", "1474", "370", "2393", "1539", "277", "1760", "652", "270", "3236", "2664", "622", "1064", "408", "909", "1035", "1220", "3117", "748", "3056", "3233", "1442", "1873", "1734", "802", "1617", "3111", "2785", "231", "1747", "2713", "1155", "2690", "1163", "2290", "1405", "1718", "1961", "1513", "3141", "3179", "2312", "1387", "2279", "2459", "3135", "560", "2091", "406", "342", "1290", "2409", "2968", "1409", "402", "1213", "1114", "1719", "1498", "2349", "978", "2240", "1800", "143", "1359", "691", "66", "2821", "2591", "2992", "1273", "888", "250", "2774", "3029", "3185", "452", "297", "2845", "1909", "1983", "38", "39", "2844", "908", "932", "1116", "1098", "2369", "1056", "1079", "79", "3325", "2766", "1768", "1099", "2714", "1580", "2508", "2072", "684", "1724", "670", "715", "2642", "625", "299", "1587", "2346", "1610", "720", "244", "572", "2761", "1958", "3041", "2053", "2977", "398", "2853", "1579", "1567", "1057", "2763", "1154", "1840", "125", "1798", "2900", "1024", "1106", "17", "2276", "656", "1011", "3174", "1769", "2340", "1093", "3069", "756", "2709", "3275", "1069", "2903", "1269", "2167", "847", "3047", "2079", "2269", "1489", "2243", "287", "1804", "2673", "1257", "2942", "221", "3290", "1846", "1162", "1025", "2734", "15", "1110", "2375", "1999", "200", "3300", "133", "1256", "335", "1978", "134", "1527", "1970", "156", "1383", "2524", "829", "1594", "1221", "2520", "692", "1834", "456", "2281", "2999", "2746", "2533", "3004", "3059", "648", "1832", "1497", "1032", "2469", "3014", "1808", "3284", "2395", "3145", "2163", "619", "3182", "1385", "2779", "2615", "2542", "2352", "849", "2262", "3020", "1794", "394", "2687", "1535", "0", "1081", "2680", "44", "3071", "1866", "2516", "331", "3211", "1086", "2593", "2439", "848", "2116", "1187", "759", "2570", "2446", "174", "3053", "1492", "167", "271", "2786", "784", "72", "1436", "1100", "2848", "2324", "2723", "1224", "2898", "1599", "1710", "274", "2362", "3040", "2370", "646", "1223", "1796", "162", "1477", "1055", "1686", "2913", "1019", "1439", "785", "2065", "882", "3046", "3180", "2726", "942", "1506", "491", "595", "1670", "443", "3079", "592", "662", "2584", "1434", "814", "1974", "2133", "5", "2401", "1310", "2490", "1829", "901", "1553", "1134", "3100", "1888", "1029", "2113", "2771", "2444", "1673", "1139", "2296", "3252", "2450", "657", "1680", "1681", "2479", "1298", "3272", "2422", "2567", "1874", "1514", "2137", "1002", "2301", "2532", "3301", "982", "466", "216", "2341", "2924", "1259", "113", "1519", "3067", "1317", "769", "2106", "688", "1609", "2322", "289", "381", "1452", "1775", "1706", "453", "2517", "2448", "193", "1685", "728", "282", "1279", "733", "3109", "413", "142", "92", "1297", "527", "2298", "2039", "1049", "1869", "2221", "2178", "70", "2040", "577", "2294", "518", "1820", "2895", "3321", "3314", "164", "1967", "2304", "1657", "1341", "1469", "1076", "2523", "3195", "1128", "2998", "904", "3133", "1407", "189", "2738", "2497", "1554", "1666", "2783", "2896", "2795", "2604", "2226", "3026", "1484", "1766", "1491", "1972", "3076", "1432", "2403", "990", "2722", "509", "1030", "2819", "2363", "1264", "2250", "1147", "1654", "1051", "514", "2306", "3006", "1919", "102", "1591", "3156", "160", "630", "1758", "2249", "1214", "2654", "1965", "1316", "910", "2672", "3245", "3154", "2865", "1384", "2585", "2432", "3240", "178", "291", "1042", "2820", "2961", "3015", "1982", "1508", "185", "995", "365", "3289", "2695", "2022", "386", "867", "2054", "608", "2891", "1801", "2066", "1979", "2754", "1980", "2073", "2554", "1417", "311", "426", "3187", "2912", "317", "31", "463", "1841", "2084", "687", "126", "1023", "1080", "1447", "583", "1582", "2177", "1600", "1825", "1918", "1113", "2671", "2775", "524", "3089", "242", "1863", "3328", "2937", "3166", "2019", "2396", "3220", "2792", "3169", "2777", "1103", "1893", "362", "1280", "2983", "2366", "1488", "2978", "986", "569", "2463", "2547", "1174", "2201", "1328", "227", "3216", "1849", "1229", "2455", "2704", "2071", "1509", "2495", "2224", "312", "531", "2956", "1228", "948", "1215", "834", "883", "1541", "196", "345", "2946", "1022", "1137", "2796", "2577", "2273", "2885", "1186", "2960", "927", "2030", "535", "591", "3150", "1759", "2104", "3082", "977", "1821", "2809", "2144", "2883", "962", "2272", "2864", "3110", "1052", "2868", "2629", "223", "2458", "2794", "1903", "1653", "836", "13", "2229", "2372", "590", "1779", "2007", "521", "1124", "2595", "2938", "238", "1635", "1021", "268", "2773", "1292", "1209", "2974", "281", "1006", "1589", "697", "654", "2361", "2271", "1361", "974", "664", "966", "3231", "460", "201", "1501", "3239", "1714", "1391", "839", "1410", "2118", "1185", "1132", "766", "1001", "3261", "3250", "1031", "708", "2087", "379", "718", "303", "1360", "1123", "2358", "1059", "940", "3323", "1074", "190", "2323", "1652", "2637", "1471", "1012", "3149", "2142", "348", "1082", "624", "2607", "2172", "1249", "2408", "613", "2252", "512", "97", "2311", "1306", "1520", "2781", "1540", "1322", "2359", "1641", "1633", "151", "127", "3159", "2684", "1455", "2222", "2745", "1621", "957", "118", "1606", "2964", "2136", "3051", "1857", "822", "1806", "20", "1592", "32", "1146", "1007", "617", "994", "99", "793", "732", "3021", "2437", "234", "2299", "175", "2337", "1997", "2089", "529", "3234", "1728", "2833", "2527", "2799", "3165", "628", "337", "790", "2559", "3267", "1827", "2131", "3203", "2330", "2047", "3094", "495", "3012", "1843", "3330", "2907", "517", "1364", "606", "220", "243", "2728", "1345", "389", "2558", "60", "913", "1287", "165", "1235", "1138", "2758", "3074", "1072", "1819", "171", "2070", "2860", "2662", "1136", "315", "96", "2165", "1730", "934", "405", "400", "1788", "2121", "2105", "2263", "2206", "1369", "2181", "2086", "2541", "2191", "2405", "2110", "1278", "2255", "2753", "719", "1168", "1429", "714", "451", "1149", "2677", "685", "2193", "3087", "2996", "633", "533", "923", "3146", "2043", "1440", "86", "479", "53", "1172", "2926", "2011", "93", "3027", "1222", "3122", "601", "640", "2669", "388", "508", "1741", "3288", "1143", "2854", "2724", "3188", "3036", "1161", "2521", "447", "3221", "884", "1304", "548", "2935", "2611", "2750", "1756", "2824", "1898", "3322", "247", "2164", "3175", "393", "1424", "638", "2153", "872", "1067", "458", "1954", "1810", "2291", "252", "667", "1165", "319", "2742", "87", "1584", "1977", "3152", "2575", "1738", "1337", "561", "2166", "81", "2989", "1612", "1388", "3324", "2417", "3052", "919", "1581", "853", "520", "2608", "588", "2343", "2134", "306", "761", "2957", "522", "2025", "1395", "2530", "2757", "320", "513", "895", "19", "1268", "1921", "1321", "1709", "3035", "1203", "2320", "799", "1623", "2225", "264", "991", "3113", "2620", "1356", "3260", "1015", "154", "1443", "3032", "2120", "1188", "1812", "1150", "1255", "2451", "173", "1141", "2127", "1604", "2347", "2016", "1294", "2509", "2292", "2426", "2149", "2100", "2128", "1823", "3128", "1960", "2812", "1182", "58", "1334", "2175", "2872", "2389", "525", "1705", "2855", "1802", "2125", "2215", "1495", "2815", "2995", "2549", "3151", "1811", "2870", "1435", "2749", "2565", "681", "239", "496", "2331", "929", "2367", "1749", "222", "2217", "2474", "286", "1425", "361", "1344", "261", "16", "359", "1562", "462", "1777", "2431", "2625", "2971", "739", "2702", "1301", "580", "2858", "2811", "418", "1879", "450", "254", "2627", "2947", "3157", "108", "516", "3313", "810", "835", "2973", "1524", "3183", "1178", "975", "2840", "26", "1336", "1906", "182", "2327", "1125", "921", "1158", "336", "2965", "421", "217", "2009", "2184", "1891", "2099", "2665", "1643", "1700", "1934", "1428", "2076", "9", "1830", "1555", "953", "433", "2471", "430", "382", "2825", "2930", "435", "671", "2051", "568", "832", "3001", "1971", "1852", "3199", "893", "55", "440", "972", "1289", "1423", "1202", "1071", "1463", "332", "650", "3280", "431", "805", "2275", "2260", "837", "1389", "920", "1404", "110", "1191", "3181", "2797", "1586", "1153", "1118", "1722", "45", "2744", "2659", "2394", "2908", "576", "3164", "73", "3192", "1750", "1075", "2391", "1552", "3064", "787", "2631", "2364", "992", "2102", "90", "3170", "3279", "547", "3086", "1248", "383", "3327", "3223", "1693", "615", "111", "741", "2496", "886", "1743", "581", "941", "985", "2576", "1323", "1945", "2257", "798", "2038", "1212", "2407", "773", "1948", "1715", "67", "698", "204", "11", "1787", "1872", "2055", "1815", "1973", "301", "423", "2634", "1237", "1266", "1561", "860", "1206", "858", "1744", "507", "1363", "1939", "1731", "1036", "3120", "746", "246", "2816", "172", "2126", "2037", "2598", "2834", "933", "147", "2132", "3088", "1860", "1453", "563", "184", "2814", "2013", "2150", "3050", "543", "1871", "2802", "155", "2650", "753", "1822", "788", "1546", "1464", "1583", "677", "2653", "2485", "1963", "1780", "150", "1536", "3158", "474", "1547", "1720", "711", "1419", "2663", "3121", "846", "645", "2877", "1251", "62", "146", "967", "1904", "2923", "3114", "1331", "494", "3210", "2207", "141", "2183", "2791", "2148", "3140", "1503", "1183", "3269", "750", "1640", "2082", "3193", "1260", "2635", "304", "943", "717", "232", "2238", "2633", "1745", "1528", "1782", "2313", "812", "2701", "3134", "907", "2187", "1034", "420", "1446", "290", "2580", "1097", "104", "475", "2214", "1702", "1039", "604", "65", "3213", "2512", "2360", "3037", "2668", "399", "1699", "1790", "2837", "253", "611", "1231", "1665", "559", "233", "3039", "1729", "2617", "2932", "2943", "1602", "2866", "2543", "416", "48", "792", "2838", "2969", "1883", "641", "285", "2784", "1101", "136", "1627", "857", "368", "609"]], "xtest": ["int", ["2068", "2697", "3258", "544", "3294", "651", "637", "1751", "2826", "1929", "1634", "1704", "2101", "3144", "2953", "2717", "255", "3249", "2425", "2539", "40", "94", "2605", "826", "2739", "1240", "3130", "122", "550", "1483", "2601", "1017", "757", "1847", "309", "2067", "1129", "1374", "2274", "1016", "755", "23", "866", "457", "3033", "818", "1993", "123", "1342", "2000", "2919", "469", "1713", "1180", "225", "22", "2876", "1505", "2619", "2988", "1171", "1595", "3248", "696", "874", "1784", "2546", "478", "1772", "3197", "1115", "915", "411", "2158", "2027", "2526", "946", "851", "2171", "2681", "69", "2406", "2098", "2398", "869", "675", "890", "2760", "1996", "1120", "2582", "2035", "532", "432", "1448", "459", "1739", "2303", "3005", "2152", "2556", "2478", "2210", "1195", "1347", "34", "2688", "1964", "2124", "2111", "1854", "1711", "1207", "2428", "2581", "629", "776", "1028", "391", "3189", "343", "2036", "439", "3102", "2380", "2602", "2020", "951", "2921", "1598", "1043", "3062", "930", "1907", "1703", "384", "3273", "2315", "1692", "24", "1574", "896", "2489", "107", "1089", "2980", "1173", "1732", "1412", "2230", "632", "3", "387", "1941", "71", "1813", "2731", "1095", "2265", "1951", "1696", "649", "3043", "2130", "1577", "983", "1916", "1456", "2477", "2529", "283", "2237", "551", "1949", "464", "3107", "3209", "1803", "2675", "2955", "1522", "14", "2666", "2379", "655", "2502", "3034", "2282", "738", "1605", "2316", "2397", "3091", "88", "2046", "3295", "1335", "2641", "2519", "3178", "582", "1870", "422", "1401", "780", "56", "2920", "2768", "1349", "2325", "3103", "3296", "1911", "2590", "844", "700", "2354", "1302", "224", "2863", "3309", "415", "211", "85", "2922", "3104", "2856", "2176", "511", "902", "1245", "749", "1677", "1402", "1281", "2873", "325", "205", "2694", "2525", "3292", "1511", "827", "1923", "1258", "1917", "2997", "366", "2616", "2057", "964", "1218", "959", "2348", "1485", "2440", "2390", "3160", "987", "1842", "3097", "1628", "1053", "1614", "2511", "3081", "100", "1066", "1340", "1818", "502", "2987", "2679", "1526", "772", "1754", "98", "1765", "903", "1649", "355", "1008", "1763", "1073", "2457", "984", "2852", "2231", "536", "725", "2884", "248", "1601", "1166", "1239", "2934", "1085", "840", "795", "2823", "3320", "2962", "2958", "1672", "2984", "503", "1853", "541", "2534", "2902", "192", "2415", "2651", "791", "855", "2667", "2024", "1109", "676", "2867", "2772", "2886", "1572", "1578", "2447", "1045", "499", "2241", "542", "2033", "2056", "1955", "2373", "2063", "2888", "1091", "2596", "504", "54", "302", "1548", "2522", "2638", "2555", "1459", "3139", "340", "3241", "1912", "2357", "2487", "618", "2491", "2335", "1683", "8", "2381", "713", "1193", "3278", "454", "3303", "27", "2710", "82", "486", "1885", "42", "3048", "1288", "2685", "2941", "2890", "2967", "2092", "763", "988", "3030", "2205", "3018", "1530", "1105", "259", "3085", "2887", "1225", "3243", "600", "1200", "3235", "3229", "428", "1311", "2245", "1558", "876", "2698", "859", "2881", "63", "2897", "1817", "334", "538", "2139", "2507", "1894", "2233", "392", "1038", "103", "730", "1241", "3200", "2652", "2069", "2769", "2334", "1727", "2535", "1998", "2411", "534", "2400", "1753", "3072", "918", "64", "1985", "2915", "1159", "2344", "659", "6", "1421", "482", "1295", "1875", "2251", "2332", "1479", "293", "1350", "3009", "2384", "781", "1664", "455", "284", "3201", "501", "130", "1406", "1785", "1243", "163", "2493", "2503", "1504", "1690", "1532", "37", "1721", "352", "3205", "2661", "567", "1773", "2985", "1355", "191", "704", "3095", "1976", "2645", "3078", "2261", "468", "2484", "1468", "1381", "965", "1618", "378", "2803", "3268", "922", "3126", "49", "2235", "2700", "2383", "1833", "993", "2213", "1986", "489", "2021", "2751", "1648", "1181", "180", "1905", "1809", "1478", "483", "1068", "1636", "230", "307", "3265", "157", "308", "2017", "2208", "824", "326", "7", "179", "1678", "1216", "1152", "2646", "660", "2696", "944", "3217", "263", "3264", "1348", "1573", "1625", "2720", "2686", "1379", "2767", "3198", "1490", "1816", "1211", "77", "159", "3137", "1933", "2078", "2940", "1995", "952", "296", "2711", "1362", "564", "2875", "2894", "2644", "897", "906", "1679", "3214", "2266", "570", "445", "612", "1246", "3316", "21", "673", "1737", "747", "166", "813", "1824", "996", "2993", "885", "523", "1882", "1987", "2168", "2699", "1371", "1482", "2418", "310", "2159", "2441", "3255", "1925", "2442", "137", "354", "2001", "373", "3011", "2917", "1735", "1060", "1126", "721", "407", "1994", "477", "2950", "1647", "129", "2005", "2606", "2589", "3105", "2404", "3010", "2842", "1593", "2424", "1403", "2500", "936", "208", "266", "797", "2790", "3061", "556", "1411", "1194", "2566", "1033", "768", "2182", "2267", "1774", "1487", "1119", "472", "3266", "1671", "168", "1430", "2518", "2540", "1500", "10", "2353", "219", "2356", "2643", "658", "1427", "2014", "678", "2966", "2880", "528", "616", "771", "1397", "89", "2064", "2003", "1127", "686", "135", "3080", "51", "1358", "955", "2096", "775", "1346", "1697", "3317", "1307", "2818", "1848", "2618", "1947", "865", "1545", "800", "2218", "777", "2878", "371", "1420", "553", "3060", "275", "938", "838", "425", "2421", "644", "2216", "2658", "2042", "497", "1047", "2462", "3093", "3161", "1521", "442", "3304", "1990", "2510", "3148", "3202", "2297", "2285", "1382", "1624", "864", "2793", "2835", "1992", "1177", "1396", "1486", "2859", "579", "2562", "3262", "1659", "1157", "2586", "1169", "207", "3016", "2012", "2660", "1418", "327", "2338", "2480", "300", "2994", "1748", "682", "786", "1668", "2499", "2514", "1367", "438", "2173", "314", "2544", "596", "2736", "2074", "770", "2402", "2083", "803", "1176", "1005", "2382", "2925", "214", "2154", "329", "1542", "1494", "364", "1930", "745", "333", "565", "1608", "1507", "2385", "1046", "689", "3083", "2318", "2454", "2735", "1856", "1370", "2599", "1009", "1886", "3017", "3171", "806", "1144", "2578", "2808", "3285", "3058", "3075", "212", "1308", "1283", "57", "778", "414", "1531", "1167", "1426", "2438", "3119", "2258", "1529", "2807", "1795", "2552", "587", "1646", "2944", "3237", "3131", "1867", "2579", "1184", "1449", "1390", "3084", "1230", "2268", "1701", "2336", "2254", "626", "1112", "900", "3049", "1897", "1959", "1924", "2928", "3031", "1291", "635", "1991", "3263", "210", "2986", "925", "91", "3230", "2198", "1462", "729", "1932", "256", "1922", "1887", "3283", "249", "1776", "2377", "2486", "2122", "2410", "429", "2573", "620", "2692", "203", "2002", "3125", "1232", "819", "1481", "3106", "1571", "1050", "2081", "186", "1537", "3096", "3238", "1616", "1070", "313", "2049", "245", "121", "905", "195", "1901", "369", "679", "403", "2248", "1881", "3136", "3186", "95", "2676", "1950", "1284", "1559", "2289", "1575", "3108", "1320", "1299", "555", "2433", "1415", "1626", "2211", "1631", "1845", "1499", "1969", "783", "1799", "2423", "734", "404", "1844", "1083", "3307", "1712", "1523", "2186", "2929", "1691", "251", "540", "2610", "2199", "1199", "2693", "894", "2062", "3196", "947", "2018", "639", "2624", "2488", "2949", "2712", "1585", "3003", "161", "852", "2879", "3116", "634", "396", "557", "917", "3168", "981", "75", "2901", "2939", "3063", "2253", "2179", "2830", "78", "2914", "434", "558", "397", "318", "3118", "3270", "2832", "2430", "181", "2874", "2918", "500", "1330", "1458", "1937", "2800", "194", "3066", "188", "1655", "1040", "1160", "1864", "722", "2145", "1252", "823", "2553", "1234", "1315", "1767", "709", "726", "25", "666", "2008", "258", "2639", "209", "1392", "705", "928", "1077", "1716", "1300", "2528", "603", "1142", "145", "2114", "1140", "237", "963", "112", "3282", "2991", "2212", "1915", "899", "1090", "3212", "956", "2326", "2307", "2621", "1807", "1357", "1467", "2138", "960", "1695", "710", "2862", "3008", "124", "2857", "1450", "449", "2551", "605", "2052", "1108", "3287", "2333", "3305", "2861", "12", "2376", "1838", "3163", "375", "1688", "2954", "1868", "1131", "602", "3099", "1726", "3225", "1078", "1380", "1645", "549", "526", "83", "215", "998", "1314", "2387", "2416", "2910", "2277", "736", "2483", "153", "2756", "2959", "1603", "1400", "47", "1639", "2689", "898", "395", "2302", "3019", "3162", "873", "74", "1510", "298", "2097", "316", "2475", "1196", "1789", "2640", "2467", "1638", "1309", "2648", "2015", "481", "2155", "2435", "2839", "1884", "3315", "1755", "1262", "2741", "737", "665", "1910", "2317", "2905", "267", "999", "2004", "537", "412", "1707", "1010", "1596", "2392", "3065", "674", "2951", "2443", "3204", "3068", "2970", "1461", "3055", "1889", "1913", "854", "1557", "1831", "1368", "4", "2143", "353", "206", "1797", "114", "2107", "1752", "3184", "2622", "2295", "762", "3023", "2156", "2574", "3299", "1837", "1333", "1037", "1533", "2778", "2141", "506", "2044", "18", "1590", "2090", "1570", "3155", "1733", "2612", "1629", "1014", "2345", "2737", "2708", "971", "2656", "2031", "2927", "106", "589", "1981", "2911", "887", "3077", "158", "3319", "566", "1597", "3298", "2762", "1151", "50", "2788", "2162", "46", "643", "1013", "631", "109", "1058", "1899", "2050", "2719", "3251", "1968", "347", "465", "2727", "2161", "1048", "1238", "2204", "3226", "1084", "321", "2309", "2492", "36", "351", "545", "949", "2623", "1422", "2630", "2715", "724", "2597", "1027", "751", "552", "879", "1122", "1839", "1644", "1525", "2452", "131", "1226", "1480", "2730", "2963", "735", "1210", "1343", "1935", "2981", "2531", "939", "1148", "2537", "2094", "1534", "530", "1444", "176", "2365", "1975", "1563", "2755", "29", "2822", "1792", "2427", "3173", "3244", "2683", "2244", "1217", "2682", "273", "1465", "2270", "2657", "575", "2805", "1263", "1828", "892", "2770", "1277", "2504", "2232", "2286", "2801", "877", "1197", "731", "2414", "2023", "1956", "1793", "1092", "2843", "2283", "1957", "937", "344", "2889", "809", "1121", "1927", "744", "1393", "1684", "2733", "240", "2583", "2080", "607", "269", "850", "3138", "116", "2075", "2561", "663", "1878", "374", "1377", "437", "278", "148", "924", "76", "2174", "2482", "3256", "3312", "2386", "41", "760", "409", "3276", "1145", "1265", "2476", "1198", "969", "2560", "2636", "2829", "3129", "1063", "1698", "2434", "2169", "2691", "1667", "767", "120", "3042", "1296", "970", "2841", "279", "1814", "1276", "1451", "2906", "1378", "177", "1312", "1786", "3247", "2827", "2284", "1242", "2674", "702", "3227", "815", "2836", "695", "2464", "574", "1517", "782", "743", "2220", "2782", "1386", "1338", "61", "950", "2721", "1313", "1041", "2112", "1865", "2501", "1375", "3308", "3302", "272", "149", "2194", "2613", "2449", "779", "856", "2732", "2614", "2045", "2972", "417", "488", "2203", "794", "2979", "3271", "2436", "2147", "2952", "3044", "138", "3331", "1270", "614", "2678", "546", "2899", "1783", "1851", "80", "2351", "367", "235", "390", "3332", "2095", "1192", "1607", "280", "1642", "843", "2707", "831", "2603", "2975", "1282", "3206", "2355", "2670", "716", "2256", "2846", "2588", "3115", "2909", "1329", "680", "1549", "1656", "1877", "2759", "2189", "276", "1781", "2466", "1096", "1550", "3232", "265", "2227", "811", "2813", "228", "2498", "498", "1351", "346", "1408", "1876", "1725", "1880", "105", "817", "2632", "2882", "1065", "338", "1515", "473", "1850", "1746", "2655", "3194", "410", "3098", "980", "808", "3090", "1984", "2413", "989", "52", "1189", "1512", "712", "1576", "2748", "627", "571", "1293", "2787", "3228", "1094", "1942", "2185", "2729", "891", "2228", "2568", "2990", "3257", "597", "3293", "3070", "1946", "328", "1566", "833", "1460", "2085", "796", "2028", "2115", "2041", "1568", "519", "3274", "1318", "2893", "935", "2982", "2242", "2310", "1087", "1936", "3306", "1556", "585", "1476", "2288", "845", "484", "1761", "2460", "2419", "2849", "2209", "1303", "3073", "2117", "470", "2010", "2976", "3297", "2798", "1470", "2328", "1920", "1661", "68", "539", "2776", "764", "2170", "323", "2368", "2388", "2806", "1658", "376", "2740", "2536", "945", "1611", "1325", "1757", "1003", "1175", "1250", "1201", "1365", "3246", "2399", "226", "3222", "774", "1466", "1261", "467", "693", "1416", "594", "2515", "288", "3127", "419", "1694", "2548", "1674", "1940", "1475", "132", "1632", "1518", "2329", "1454", "339", "1117", "3024", "554", "1431", "2609", "816", "2321", "446", "2308", "820", "1398", "3002", "916", "3143", "1", "1966", "2752", "2088", "3281", "1826", "476", "880", "448", "2513", "2192", "3045", "2200", "1208", "1892", "1637", "598", "2264", "1437", "3177", "1373", "1233", "931", "1354", "801", "2129", "1286", "2572", "1135", "841", "128", "1736", "2305", "2817", "218", "2725", "1472", "2342", "1859", "262", "2061", "324"]]} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft.config b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft.config new file mode 100644 index 0000000..b681784 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/aircraft.config @@ -0,0 +1,13 @@ +{ + "scheduler": ["str", "cos"], + "eta_min" : ["float", "0.0"], + "epochs" : ["int", "200"], + "warmup" : ["int", "0"], + "optim" : ["str", "SGD"], + "LR" : ["float", "0.1"], + "decay" : ["float", "0.0005"], + "momentum" : ["float", "0.9"], + "nesterov" : ["bool", "1"], + "criterion": ["str", "Softmax"], + "batch_size": ["int", "256"] +} diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist-split.txt b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist-split.txt new file mode 100644 index 0000000..e14d7fb --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist-split.txt @@ -0,0 +1 @@ +{"train": ["int", ["20720", "26055", "29223", "48857", "37477", "34672", "54127", "2593", "27737", "46788", "42365", "13355", "48024", "9155", "49041", "42818", "27198", "18826", "36083", "50082", "11602", "43786", "19218", "37843", "22180", "8338", "46325", "10100", "14453", "24589", "31499", "27887", "15496", "51194", "35832", "10101", "34203", "5547", "43446", "41844", "6908", "52372", "12756", "19", "46737", "14940", "43644", "13331", "51910", "577", "41828", "48207", "14434", "11359", "24400", "5667", "8638", "8409", "36632", "52386", "16105", "26276", "8482", "52674", "17990", "34303", "32787", "38220", "41504", "20286", "35468", "44178", "24570", "4025", "55210", "22957", "10120", "54222", "3049", "23142", "47585", "48012", "30999", "33277", "7669", "53416", "24168", "59780", "16350", "53190", "46841", "41579", "28853", "42482", "4667", "5133", "28140", "6955", "40149", "43882", "32105", "20797", "10130", "25981", "36222", "49390", "55991", "46123", "14091", "4619", "20805", "6057", "40552", "16003", "28716", "59825", "23947", "249", "19039", "3635", "43942", "27106", "49612", "54294", "28366", "32832", "49044", "41875", "25118", "24114", "34931", "28912", "44868", "4873", "1556", "18541", "48184", "49061", "46251", "36215", "45371", "13893", "38877", "50769", "42617", "44345", "14449", "25452", "23023", "9855", "19052", "57468", "824", "40794", "36816", "30515", "19232", "58761", "31993", "48554", "41717", "28557", "34776", "1385", "54955", "46414", "23821", "44230", "19298", "169", "20787", "34282", "45183", "36907", "30068", "45770", "15956", "992", "9742", "19211", "20155", "5074", "12370", "37818", "34344", "5615", "44577", "36600", "57033", "54460", "285", "41749", "21427", "32225", "2419", "46293", "54010", "21674", "50481", "33469", "55822", "37926", "11229", "16181", "44736", "8217", "4993", "41733", "8549", "22723", "55732", "8825", "44875", "10141", "18416", "38350", "40826", "54108", "50020", "17278", "36402", "58941", "11410", "33723", "52104", "53807", "15604", "53773", "28376", "49374", "44777", "8629", "3153", "20014", "12982", "56615", "48648", "31502", "45378", "378", "54748", "47436", "2486", "49339", "52119", "44732", "31270", "31268", "4893", "37781", "10754", "59499", "6079", "19279", "13875", "19193", "55854", "49939", "10173", "9511", "14299", "59952", "29510", "52698", "32274", "5099", "44021", "58291", "2387", "51700", "3758", "5425", "45445", "21272", "51871", "38481", "43412", "14959", "51969", "1042", "14013", "21460", "22494", "36786", "6564", "11732", "43458", "53153", "50890", "44858", "57339", "40215", "56726", "4172", "35520", "53648", "50788", "34745", "11571", "30477", "59380", "8624", "59165", "28672", "59925", "1529", "50698", "40671", "6635", "59564", "44804", "26683", "11523", "35447", "2661", "5476", "17814", "9199", "38425", "32306", "39290", "33627", "24234", "28878", "1013", "27412", "49741", "44744", "53137", "26570", "22352", "9930", "44910", "50135", "48032", "55866", "28187", "25291", "31151", "13677", "28402", "14963", "55177", "45523", "11879", "7687", "43219", "17978", "34480", "35221", "39970", "25663", "46989", "10361", "33148", "38543", "53120", "1230", "22328", "47355", "40272", "52440", "54122", "40204", "10324", "1241", "55681", "55502", "3552", "53480", "36852", "27567", "34393", "29204", "50771", "4476", "6715", "24618", "47939", "31742", "36716", "31064", "7606", "6068", "39086", "9535", "7806", "36486", "41883", "42666", "38100", "39431", "37899", "20833", "30287", "5828", "28567", "2956", "1118", "48599", "37521", "37039", "42930", "21056", "51696", "9203", "49359", "21211", "12626", "16366", "3178", "21508", "32429", "52012", "28232", "49645", "14394", "26558", "29920", "39052", "32095", "25655", "52158", "47793", "28907", "59333", "3122", "17057", "31136", "3534", "6182", "52568", "17536", "59953", "6505", "9313", "47047", "1169", "48706", "21428", "25438", "598", "11738", "15418", "12078", "6628", "36463", "29111", "55078", "30563", "2378", "28027", "3056", "32658", "21454", "41084", "39257", "7572", "3191", "37233", "13579", "46891", "22543", "5887", "49767", "54702", "41334", "50841", "55745", "28998", "18063", "30714", "9400", "54301", "39922", "1834", "31080", "22444", "23312", "26544", "44682", "59966", "56757", "13069", "9926", "49919", "42593", "34484", "54623", "49947", "57441", "293", "6695", "7497", "33047", "30823", "53430", "34852", "3400", "47804", "3929", "22361", "41497", "12608", "38953", "55670", "7718", "58335", "24360", "24933", "57696", "11680", "11178", "34068", "34160", "38332", "5847", "9978", "28561", "44645", "59963", "44905", "46094", "30000", "19213", "51804", "15866", "40454", "42396", "40962", "56555", "11426", "9062", "51262", "11319", "18706", "26356", "17661", "43717", "13630", "43750", "47810", "33650", "56790", "35079", "55526", "8766", "13223", "56539", "41403", "14243", "30126", "1804", "9791", "31779", "42382", "30589", "16027", "29164", "39477", "27776", "36144", "20301", "48503", "55631", "18709", "35050", "51419", "1702", "40479", "9918", "8730", "18962", "5970", "43663", "13917", "9267", "18436", "51190", "15433", "17722", "5895", "55099", "7312", "20879", "23577", "685", "23184", "7087", "48654", "31894", "27692", "32544", "34761", "28092", "42672", "1797", "36893", "27763", "39739", "32010", "17137", "42677", "19578", "26574", "57260", "6074", "15238", "26305", "42692", "24026", "28207", "2831", "59109", "44464", "35528", "15541", "27294", "35686", "30765", "11245", "41047", "36964", "7102", "13725", "54276", "17810", "58173", "33405", "990", "27242", "31969", "13601", "50257", "52126", "11982", "59488", "54340", "11107", "45762", "6006", "53363", "30712", "47423", "48089", "25548", "28772", "45033", "32998", "14972", "21716", "44580", "28015", "10406", "43186", "227", "16317", "42182", "55936", "42779", "33079", "42568", "2934", "10018", "16109", "23380", "32062", "27820", "53429", "55927", "8961", "30405", "3130", "1740", "26204", "56712", "616", "18450", "53123", "20680", "50169", "46342", "39046", "8823", "35288", "25628", "213", "45729", "172", "23084", "39201", "25356", "8748", "36004", "4967", "12547", "8360", "52501", "27676", "12367", "4964", "48703", "21594", "38880", "27751", "17981", "47534", "12950", "2492", "9393", "10966", "11230", "58861", "27054", "7897", "58101", "48324", "20894", "19862", "47390", "55935", "35090", "52379", "26822", "52303", "55581", "21918", "21742", "16121", "53245", "39798", "25376", "56758", "2698", "42739", "12681", "38894", "22961", "19513", "23209", "54149", "28066", "26751", "4387", "47097", "39554", "25671", "46959", "59468", "35423", "44037", "33827", "46309", "24227", "17265", "17961", "17282", "17122", "41004", "45743", "53259", "14407", "21565", "35035", "45317", "38835", "25905", "15818", "19397", "58153", "1878", "13487", "20363", "58739", "33380", "35232", "43654", "29275", "58288", "26481", "24268", "29723", "19257", "55780", "20541", "8778", "12430", "19522", "18664", "10537", "24997", "30971", "26884", "49959", "29021", "18158", "20962", "38454", "3175", "10969", "2004", "18333", "30201", "39709", "23333", "9476", "17199", "36590", "15076", "21088", "38270", "5818", "23102", "6470", "58576", "3342", "12786", "25629", "19509", "23226", "59028", "51170", "15705", "34690", "34552", "19423", "44212", "41102", "36724", "34036", "16205", "28569", "21766", "30743", "31982", "59467", "41745", "20133", "18360", "1251", "47966", "29991", "54225", "50214", "55827", "46557", "10596", "10802", "50836", "51586", "27423", "42801", "52001", "29503", "58395", "3847", "42336", "32031", "22243", "19234", "42776", "7258", "54491", "47472", "37909", "57043", "11726", "48595", "36797", "42167", "53740", "23434", "54129", "35012", "53665", "20484", "46772", "37948", "31850", "22946", "58332", "6536", "14716", "48422", "18568", "58236", "26571", "40214", "44833", "34187", "794", "22763", "38909", "45022", "59006", "8018", "52844", "4804", "53774", "36672", "10443", "34919", "12423", "24895", "10235", "7154", "13556", "26614", "21084", "32496", "16946", "28625", "42001", "42085", "33955", "11412", "38108", "57057", "50982", "46894", "18510", "12952", "37758", "42686", "49507", "18182", "55704", "10816", "59389", "11266", "11322", "55727", "26468", "39424", "58423", "17614", "29183", "3908", "13580", "41352", "10169", "16701", "40951", "16305", "20076", "14207", "23233", "55947", "41536", "37460", "57180", "294", "20493", "45711", "27032", "54030", "21274", "8697", "15331", "20207", "36390", "9615", "36003", "56314", "23815", "26732", "35566", "11064", "40426", "36995", "11641", "24118", "2188", "2195", "26795", "34421", "51653", "14594", "6578", "31676", "24389", "43303", "34628", "54652", "38255", "35396", "48813", "30093", "706", "56954", "29154", "6845", "46971", "26885", "20185", "49885", "42005", "59999", "50992", "12621", "11355", "27154", "49827", "5604", "21570", "33324", "40843", "40798", "35580", "22816", "46651", "2452", "32757", "33211", "22075", "2147", "55167", "42125", "2226", "57427", "26826", "4098", "9670", "3652", "34367", "53916", "32926", "24665", "18993", "39697", "29025", "23405", "37525", "25667", "40443", "37727", "1733", "8076", "1433", "37265", "12692", "14838", "48442", "12745", "30693", "34094", "44728", "28018", "40709", "35411", "28489", "34089", "16647", "2234", "7269", "30274", "28118", "30276", "9875", "10690", "20762", "45740", "50792", "33860", "20162", "18965", "49487", "44572", "22396", "48045", "25341", "20834", "21162", "39804", "678", "46969", "41180", "7167", "52328", "27372", "31446", "42081", "9073", "43120", "28148", "44999", "3850", "1402", "9501", "59071", "4128", "11763", "41342", "26805", "45482", "34948", "10777", "56875", "57181", "31498", "2912", "32741", "28917", "20577", "15600", "56724", "55543", "54367", "8687", "47141", "43773", "6328", "49371", "21306", "38268", "18232", "45035", "25681", "50270", "38896", "47357", "45805", "25448", "1391", "24710", "59367", "44348", "40489", "40508", "7078", "17668", "23724", "35722", "5209", "41346", "42333", "52763", "482", "24202", "1448", "17414", "13160", "27286", "11104", "6352", "56286", "58110", "6530", "41784", "35728", "58639", "34904", "8387", "24391", "56366", "5410", "20288", "9291", "41037", "51775", "19387", "38068", "52370", "14696", "57903", "21293", "52883", "10199", "13517", "20995", "56500", "29345", "19034", "47719", "37205", "43025", "57844", "20489", "18803", "50007", "23361", "37059", "45576", "11626", "53683", "35587", "4500", "34432", "4164", "29989", "17620", "21740", "39989", "53920", "33367", "25152", "4663", "19669", "33171", "24371", "58995", "54854", "43734", "42176", "44915", "37415", "49584", "46750", "31083", "33253", "17390", "33244", "35417", "55144", "11933", "58092", "55463", "49331", "4989", "24425", "45038", "7553", "21818", "12596", "18947", "22339", "36070", "5616", "44550", "37151", "15978", "38827", "25080", "57557", "3763", "17190", "38696", "29798", "4699", "29568", "14681", "1722", "508", "23079", "3475", "24353", "27639", "43286", "39449", "41775", "7361", "49897", "20648", "58751", "35379", "8440", "30910", "24999", "38902", "23694", "57089", "44379", "21582", "3666", "50849", "36636", "59528", "45077", "46514", "38417", "30690", "20256", "47136", "9263", "54223", "20048", "41204", "29180", "57647", "5831", "52902", "5325", "53812", "14115", "22713", "53962", "33746", "53167", "42949", "56928", "49178", "12387", "31214", "1043", "26566", "7514", "8659", "2049", "45437", "41938", "24352", "37747", "9069", "5054", "41129", "37391", "57428", "23159", "14860", "51796", "9604", "51717", "22732", "37071", "9076", "30523", "35917", "33585", "51405", "59661", "32789", "59648", "26686", "2704", "48254", "50", "50178", "17304", "46570", "8235", "42231", "42285", "2094", "6171", "2075", "49301", "43578", "6084", "48437", "54256", "55654", "5437", "55367", "58732", "2703", "6386", "56093", "45436", "27655", "39284", "13329", "44578", "40291", "12226", "11222", "22621", "17430", "17453", "10309", "54666", "55300", "58247", "39792", "14011", "12971", "16049", "8353", "8555", "8709", "39699", "24379", "27826", "18686", "51402", "39019", "55298", "13040", "40765", "24546", "57617", "23771", "11562", "50935", "16167", "45574", "11679", "49735", "36686", "12021", "58036", "37376", "8701", "5064", "54318", "29646", "26740", "37912", "8867", "46016", "48493", "19712", "56051", "29978", "33185", "6515", "17362", "55547", "35513", "9905", "26185", "40652", "33857", "6998", "29586", "29309", "11016", "52232", "29770", "9678", "19440", "48729", "29419", "44546", "23247", "14468", "24158", "26519", "32226", "45946", "9640", "14049", "31207", "22671", "53506", "51782", "20549", "17894", "45238", "23105", "24795", "47582", "56637", "8911", "59413", "56280", "16401", "4151", "9323", "50149", "561", "9697", "16526", "27747", "35029", "52414", "17326", "19344", "8368", "8015", "3894", "23786", "44841", "8055", "30445", "43466", "52090", "14697", "2450", "52601", "30471", "58447", "526", "16607", "20195", "42850", "9280", "18934", "38536", "39409", "53704", "26764", "47948", "39304", "18500", "47859", "4402", "34470", "32853", "977", "284", "29495", "3926", "21966", "41972", "4033", "19436", "8816", "55910", "28915", "13768", "52376", "40159", "52483", "36619", "38198", "11791", "30921", "28748", "15499", "626", "51128", "28256", "9493", "57176", "21731", "18779", "36674", "3008", "14551", "26272", "6275", "30822", "36108", "59020", "42297", "20352", "1014", "18835", "22596", "24707", "22570", "1408", "42217", "34839", "49504", "35602", "19478", "58477", "19848", "56596", "34833", "19472", "15916", "52505", "1026", "14648", "8599", "3255", "25326", "8407", "32119", "51432", "15317", "13528", "2491", "33463", "14397", "12669", "45207", "58122", "46276", "5121", "3573", "34583", "40805", "44741", "22836", "27983", "58991", "26487", "40140", "24608", "37735", "30849", "17317", "29340", "16652", "30614", "34200", "55432", "10586", "44355", "9376", "13882", "27238", "10927", "58760", "20967", "24240", "14490", "33974", "50630", "32207", "27405", "16246", "10806", "44303", "3710", "53298", "15059", "50255", "5051", "402", "23720", "12103", "35752", "27477", "3944", "53859", "21639", "12645", "18202", "54094", "10613", "39190", "51157", "37089", "59215", "43294", "22794", "54969", "31280", "13728", "14682", "48165", "24509", "26395", "41721", "26022", "59342", "4044", "25533", "8857", "29955", "27290", "34182", "57405", "56952", "39026", "35761", "26074", "32289", "55206", "43307", "37754", "39688", "45361", "11681", "42901", "53246", "15485", "21101", "49388", "39255", "2385", "28529", "45567", "57017", "27587", "16179", "43008", "41960", "13300", "25771", "13145", "46108", "15477", "43664", "29960", "26030", "47006", "16527", "1020", "28295", "39379", "35989", "51214", "14841", "47232", "17862", "25787", "15556", "15137", "55628", "17096", "47631", "18792", "51085", "10401", "12205", "52848", "48647", "46009", "50828", "58716", "51765", "3137", "50292", "54097", "4952", "6760", "21233", "19209", "31191", "32179", "59176", "31528", "28448", "39874", "2834", "18707", "28927", "38736", "8411", "44961", "39867", "52510", "7049", "4182", "11624", "39783", "37536", "30703", "51891", "960", "39590", "39685", "21870", "1090", "28319", "50103", "31463", "3505", "57475", "14563", "40757", "53001", "14076", "52467", "31670", "33456", "37183", "21155", "39632", "31353", "44799", "24337", "3341", "15630", "27430", "55624", "29406", "29889", "4522", "21414", "39545", "1688", "29961", "50884", "8435", "38830", "58209", "2862", "10433", "50921", "48237", "33145", "28039", "45906", "45293", "12017", "59964", "41057", "2310", "11420", "5786", "19058", "4276", "36291", "16449", "20302", "15598", "18995", "3138", "1531", "41965", "13947", "47963", "37568", "29791", "6992", "48117", "15301", "1585", "19514", "47608", "12873", "20898", "39596", "51010", "14310", "33920", "3239", "416", "49609", "32755", "33196", "30779", "18193", "583", "13101", "19598", "5845", "53717", "37958", "42580", "18290", "7325", "28382", "20523", "32122", "51931", "11842", "50542", "48781", "1267", "4877", "24566", "26179", "16704", "48205", "33425", "43626", "53458", "11382", "42639", "29232", "25284", "28012", "1728", "20563", "45875", "20775", "50188", "48868", "32564", "43722", "35161", "6535", "24780", "38710", "6375", "32703", "14892", "3528", "821", "58115", "7421", "45827", "34096", "45514", "34568", "59665", "55775", "26477", "22546", "25695", "48635", "58878", "24473", "36982", "7041", "23392", "57252", "49296", "40302", "43643", "24358", "26342", "3429", "607", "34180", "44065", "34837", "20405", "59074", "43641", "52632", "8955", "51340", "28833", "59340", "27556", "1605", "3123", "49378", "117", "28896", "24335", "54019", "4669", "54758", "15682", "55992", "57895", "16345", "35821", "36603", "27912", "51614", "4959", "14667", "50987", "40091", "36542", "50764", "12953", "9895", "39271", "11854", "14685", "18839", "33232", "47212", "43474", "38343", "23810", "44857", "18452", "40266", "33229", "49084", "55731", "47971", "34136", "46306", "54881", "14471", "59693", "42283", "23959", "59344", "37866", "35163", "38651", "4267", "58929", "53623", "17499", "14849", "459", "3522", "44316", "38618", "35224", "38577", "45970", "52047", "50983", "21791", "5890", "2447", "30979", "34633", "50291", "27008", "6934", "8325", "21889", "44319", "1771", "19097", "56914", "31762", "4717", "16655", "21762", "58799", "25991", "5737", "7686", "34395", "43521", "16782", "24116", "13477", "44808", "49367", "2015", "50124", "6495", "43624", "16127", "15756", "5998", "59328", "1144", "47997", "36998", "10743", "23276", "42806", "47026", "13162", "36317", "40436", "5870", "53138", "15035", "42318", "14186", "27493", "57621", "49706", "57285", "21531", "7962", "31404", "30984", "6582", "15103", "42947", "30263", "29731", "2523", "4244", "16481", "36821", "48338", "53609", "34249", "6252", "30791", "44441", "59659", "40558", "41734", "6364", "3389", "47639", "8813", "22830", "19308", "54298", "29438", "46152", "1283", "42960", "34240", "39294", "9674", "33565", "22341", "54707", "27760", "35563", "29743", "18864", "42703", "4379", "325", "52", "37861", "27082", "7830", "55781", "12473", "16319", "49689", "46463", "36105", "50623", "10711", "50484", "48578", "46138", "40428", "6949", "26296", "23657", "35342", "45419", "52251", "50536", "3751", "10181", "30771", "41996", "11973", "25227", "31579", "46709", "49431", "3197", "43189", "23098", "40321", "672", "2532", "48356", "59244", "37446", "22849", "19781", "6449", "4442", "44398", "53547", "22468", "51009", "9084", "54837", "54781", "20096", "18005", "50416", "41471", "14584", "25428", "34194", "42357", "12560", "46798", "17401", "6011", "52543", "44847", "17853", "41158", "36420", "25490", "40726", "25147", "16892", "28520", "10340", "38587", "29797", "38237", "54015", "33750", "47258", "55067", "45468", "18951", "44980", "45255", "56488", "53263", "56364", "13467", "51117", "23022", "2466", "51865", "20773", "20135", "1520", "10493", "53296", "15880", "56654", "4788", "58933", "8674", "25601", "47268", "34580", "36451", "51544", "8355", "43810", "11162", "32080", "31934", "19950", "17478", "12019", "48017", "27577", "46520", "19054", "37221", "52233", "3728", "49646", "6960", "5863", "25007", "26223", "1115", "35667", "12353", "31717", "35242", "23330", "20020", "9594", "43122", "30957", "40923", "52910", "52064", "35228", "31666", "10413", "42046", "15274", "31456", "8938", "54339", "30148", "47922", "25946", "27830", "389", "42549", "20672", "43913", "32536", "31114", "26643", "22260", "5707", "36913", "8975", "21346", "2822", "5455", "58775", "31027", "23921", "37956", "14925", "15987", "27012", "29258", "47500", "53730", "30117", "174", "5065", "19603", "45144", "6811", "1780", "20922", "53569", "52744", "28131", "19114", "43295", "24972", "37121", "31937", "3866", "33894", "45886", "8494", "22093", "19294", "25931", "55163", "32868", "53785", "15551", "3785", "30088", "54146", "5946", "45498", "13836", "54209", "37792", "4514", "34615", "534", "15735", "49942", "32420", "57488", "48086", "33628", "18151", "49466", "31533", "30478", "699", "12043", "57005", "15632", "56711", "30273", "6851", "32667", "31777", "20657", "39779", "47959", "51359", "14969", "44978", "33298", "33525", "52122", "53845", "20368", "38023", "13906", "11000", "42830", "11566", "40988", "1534", "36379", "59134", "18689", "18632", "44136", "52429", "18199", "20066", "34506", "7451", "36261", "41214", "14641", "16253", "33695", "2240", "21774", "5222", "55377", "55908", "58428", "18282", "50094", "4822", "23336", "48211", "6228", "4744", "46536", "46789", "42787", "58828", "15060", "48460", "26352", "51021", "8553", "43590", "53842", "1769", "16557", "57551", "34377", "22070", "45818", "29069", "36588", "21562", "37745", "6953", "59537", "4721", "42214", "36642", "28710", "13807", "6098", "51281", "54730", "31554", "54820", "12584", "23804", "47184", "18503", "28543", "28848", "936", "20362", "44492", "38414", "44336", "22776", "7220", "31837", "54667", "5984", "48758", "49478", "51643", "2382", "11437", "51711", "4597", "32879", "24515", "56048", "51933", "28534", "20872", "17755", "23508", "25495", "4776", "7113", "37269", "4444", "33176", "22871", "46081", "35169", "4790", "13870", "59914", "27897", "17898", "18773", "57071", "5872", "59010", "25869", "24531", "9125", "6911", "37196", "40352", "41838", "16171", "27989", "4400", "15627", "48632", "22063", "4225", "10172", "58779", "52525", "32313", "10222", "12688", "48167", "44221", "1507", "16387", "15837", "7367", "57362", "24848", "41837", "27295", "3980", "43589", "42165", "59681", "58856", "49089", "32284", "24094", "33636", "1835", "16881", "36751", "36788", "37186", "11338", "27016", "34424", "46386", "19864", "2656", "41280", "9072", "8710", "38084", "59486", "48380", "35977", "1986", "26710", "18733", "50389", "25776", "50972", "41153", "16740", "31331", "6991", "1185", "48660", "57747", "43974", "19464", "6721", "29760", "26165", "32843", "33685", "11648", "35093", "9047", "1500", "32264", "17993", "25584", "55877", "20832", "11297", "7517", "55472", "25027", "26865", "17775", "43693", "32534", "45737", "52795", "59706", "26665", "7381", "57821", "25223", "20421", "1290", "15455", "55211", "23008", "181", "27217", "31053", "46627", "5431", "54890", "32163", "55458", "56595", "45941", "47083", "22115", "42800", "35866", "29208", "48791", "11886", "40568", "5139", "11015", "46879", "31071", "43871", "20348", "37526", "59742", "37681", "25265", "56838", "41130", "44005", "48797", "11184", "51041", "49964", "35698", "21850", "54784", "45180", "10090", "41402", "40702", "40334", "9897", "57585", "58790", "34650", "43728", "57830", "26164", "37969", "26105", "25912", "26852", "38193", "41558", "32935", "30034", "23048", "6326", "44917", "59872", "39383", "24285", "31996", "48423", "15864", "37663", "12677", "32449", "8362", "39514", "41581", "50086", "43701", "25487", "28857", "20246", "21585", "18012", "21117", "19577", "52998", "52686", "24231", "18758", "4229", "17951", "19296", "26205", "54138", "51728", "14560", "31907", "13423", "56035", "10543", "33194", "29867", "31541", "5354", "22588", "53063", "38524", "40707", "59336", "35795", "9732", "31156", "53515", "52511", "25827", "34683", "9089", "37057", "31697", "21603", "34417", "35462", "24926", "17750", "39535", "21896", "18946", "48821", "42195", "20025", "21139", "8546", "20674", "3067", "38194", "3784", "12153", "51035", "8491", "32763", "43373", "22846", "9701", "43783", "46584", "27841", "56763", "27182", "10186", "38151", "24601", "39404", "9117", "39910", "3370", "39584", "55598", "4510", "12176", "21079", "56730", "45853", "11561", "36823", "19525", "12658", "14092", "8577", "54701", "51096", "28769", "44006", "12442", "41930", "54472", "4181", "27671", "22886", "51549", "33230", "11035", "7898", "48709", "53284", "21138", "56728", "9213", "58675", "21240", "9636", "17434", "33884", "43099", "25564", "14431", "50742", "26201", "37207", "32813", "50576", "55477", "27733", "13025", "3169", "57991", "54913", "18646", "54972", "33590", "24757", "19473", "43262", "16103", "25051", "28491", "57175", "20733", "38629", "2010", "35045", "5015", "51146", "56682", "35077", "25437", "33046", "51715", "5326", "5205", "23372", "19938", "6040", "39738", "3810", "26575", "16136", "41934", "2583", "47203", "50078", "54591", "29820", "46226", "16865", "16443", "20739", "37254", "35251", "26959", "385", "38833", "11691", "24679", "20283", "46447", "25134", "20434", "22582", "28708", "32353", "40915", "21174", "33060", "46907", "21200", "20825", "34292", "45198", "43986", "12675", "25095", "52867", "29660", "6425", "5079", "16891", "3786", "57877", "19063", "39822", "20953", "8008", "58116", "33634", "26484", "31847", "42337", "28178", "59330", "24657", "44958", "7396", "4773", "48517", "34452", "45905", "22464", "52077", "31279", "30607", "18781", "4871", "53018", "27871", "55703", "16471", "41981", "500", "36969", "10217", "38811", "903", "14801", "2578", "15845", "43402", "54610", "28077", "54382", "27843", "58664", "2596", "48560", "55951", "25534", "17228", "28151", "23761", "47111", "54348", "18207", "55730", "17719", "7670", "42350", "56476", "29825", "54783", "15370", "16447", "49642", "11715", "5533", "39753", "58486", "59962", "55566", "37307", "5200", "3005", "27387", "22037", "2241", "20850", "59715", "39237", "13002", "13599", "18370", "42768", "54039", "16862", "47301", "13241", "11903", "11354", "10369", "1173", "11418", "27959", "9346", "51246", "47707", "42423", "39226", "23493", "2538", "44052", "27785", "6884", "43109", "34117", "19852", "28766", "46923", "37513", "12028", "29845", "59816", "17791", "47323", "32267", "51062", "43800", "40556", "46518", "6488", "36990", "21641", "9940", "32881", "46595", "30390", "43802", "52141", "23566", "29915", "35705", "58683", "24299", "9643", "56931", "39674", "47876", "30374", "59097", "50770", "11236", "55922", "27686", "25403", "47329", "46274", "49969", "23115", "29320", "51763", "40186", "16547", "5542", "22806", "41474", "56736", "13098", "22039", "73", "35638", "23863", "28502", "32818", "54578", "2533", "16959", "13863", "1141", "47982", "10674", "126", "28311", "58942", "42664", "16247", "711", "59764", "59439", "11353", "2", "23202", "59062", "44310", "59469", "8427", "33553", "55092", "5802", "57590", "36942", "5983", "15535", "20345", "32402", "37585", "58782", "15058", "11569", "3421", "24647", "8426", "14387", "7743", "51120", "3427", "36351", "9341", "8138", "39055", "33393", "57211", "44039", "40761", "30301", "55773", "19891", "6388", "31683", "14949", "22013", "40080", "12871", "52636", "7914", "14879", "10315", "57812", "22144", "17021", "29103", "31638", "27937", "7186", "35290", "15364", "54471", "26239", "11026", "58692", "25245", "28392", "43679", "43542", "59036", "54197", "28206", "25978", "14451", "54620", "19942", "30802", "7554", "32432", "29290", "45608", "20009", "16996", "41914", "27537", "12057", "7847", "54213", "21553", "28203", "13319", "55870", "38241", "20406", "4461", "19476", "44451", "9907", "50081", "41542", "9752", "58863", "11475", "14534", "42024", "11999", "48382", "45480", "27063", "47568", "23584", "14814", "6672", "30657", "49051", "48104", "5645", "43248", "53843", "34791", "52654", "21566", "21129", "59268", "3595", "16751", "32033", "18348", "4786", "50064", "1455", "24514", "40825", "14215", "22758", "35863", "51055", "10885", "46938", "16712", "3253", "37804", "39238", "11588", "44135", "34410", "34649", "58297", "47486", "53714", "47955", "11240", "47577", "20336", "34050", "40483", "10839", "56191", "56439", "16771", "3098", "28134", "29361", "15228", "16364", "53527", "2337", "28314", "32302", "35488", "19991", "17220", "1537", "45407", "41444", "44935", "47648", "32203", "5493", "24099", "14636", "1620", "53635", "44299", "3995", "25976", "29543", "514", "15736", "49025", "23462", "11873", "25605", "45899", "32003", "43398", "38702", "1387", "44012", "31439", "16467", "25945", "32742", "16383", "13447", "16150", "22505", "3608", "12761", "10851", "8087", "47817", "38319", "23213", "33358", "24199", "40090", "49663", "22607", "50817", "49557", "30067", "20704", "30170", "16007", "40780", "35011", "44493", "34882", "52087", "47992", "33056", "49282", "51642", "28302", "41885", "33069", "11349", "1915", "3449", "13285", "14173", "14151", "54583", "46072", "24905", "37740", "37973", "6220", "33068", "50060", "57642", "2289", "19145", "35421", "48269", "27818", "21411", "36834", "52872", "46925", "13257", "38703", "48402", "20482", "36887", "22695", "43047", "8626", "37135", "39771", "48498", "49302", "183", "15763", "14029", "50058", "484", "28130", "14044", "18622", "56746", "7194", "59029", "29079", "25560", "37252", "10004", "288", "55523", "58840", "41240", "44771", "1671", "2281", "35346", "43980", "55440", "5324", "50552", "47371", "13250", "21001", "3022", "31829", "23126", "10108", "44727", "54068", "37219", "16035", "38784", "43431", "50343", "40242", "27925", "24196", "12574", "16441", "45434", "41028", "8516", "209", "16114", "34634", "27075", "39733", "58380", "17905", "14048", "14501", "7852", "50835", "50818", "27201", "14622", "46820", "5271", "42431", "3997", "47256", "19882", "5640", "58120", "49893", "28615", "41699", "6492", "51148", "1163", "7801", "49127", "55492", "20309", "50099", "26690", "26699", "23342", "41604", "5624", "55322", "45962", "6346", "46846", "44436", "28687", "30893", "53203", "45823", "2889", "33133", "12557", "5777", "2009", "45636", "662", "48467", "53793", "7175", "38975", "34708", "47644", "52507", "4368", "3089", "30318", "34978", "39476", "50216", "19877", "24211", "35309", "42200", "42075", "27001", "26704", "48429", "27033", "13810", "19137", "43506", "36477", "32646", "22572", "33778", "41625", "23619", "52384", "51114", "17425", "6809", "39233", "8988", "27148", "31659", "35295", "44045", "31987", "55019", "8848", "51044", "10007", "58009", "19199", "24504", "35959", "22288", "47662", "14251", "35769", "19696", "54828", "49101", "48225", "32193", "38942", "6661", "20393", "16713", "24996", "34834", "4519", "57490", "35793", "27884", "16961", "35071", "34582", "30078", "12413", "51553", "15613", "26618", "29640", "26092", "41087", "38612", "59487", "52365", "32541", "14848", "18045", "12933", "23944", "49491", "18678", "37195", "933", "49135", "41693", "11023", "19331", "24673", "28265", "49679", "17842", "4951", "8611", "9081", "28852", "13748", "13006", "45738", "37914", "19493", "50847", "15954", "16307", "18330", "35848", "3343", "42965", "57118", "42555", "32088", "18260", "36369", "35806", "56337", "23069", "11849", "40148", "40781", "58573", "20365", "10965", "8309", "25425", "19821", "18425", "40883", "1094", "14210", "6165", "58207", "45280", "20594", "39324", "36320", "14418", "25565", "46574", "11321", "57641", "2472", "49712", "45477", "4535", "2501", "51894", "46516", "56270", "54769", "1758", "4111", "3551", "58590", "41674", "56427", "26769", "9349", "18975", "14723", "28042", "14666", "17848", "14231", "34911", "57730", "26447", "22317", "872", "31260", "47424", "52972", "29353", "39486", "57085", "17859", "5508", "27869", "30838", "37065", "30378", "47088", "46454", "44102", "23850", "6817", "777", "17557", "54343", "25287", "46312", "18421", "41839", "50729", "27614", "30121", "10956", "2187", "6942", "365", "24343", "7809", "6865", "31478", "21503", "11979", "30241", "50379", "46573", "4192", "7166", "16585", "38067", "56030", "21618", "39509", "23714", "16524", "8096", "34063", "32844", "43239", "45703", "15211", "40896", "56479", "12018", "55911", "37509", "887", "7952", "10534", "23285", "41114", "7570", "19998", "2182", "30891", "11698", "51205", "38437", "31448", "54566", "6408", "18984", "20622", "20198", "59899", "2266", "25716", "54393", "40547", "9536", "53697", "57501", "29217", "44272", "25261", "15825", "44872", "44794", "34681", "17651", "34355", "39444", "53318", "32476", "38389", "1117", "14402", "40028", "16580", "40921", "38069", "13296", "10811", "49907", "1754", "33023", "22647", "41271", "3128", "37951", "43289", "42613", "20370", "15710", "59270", "42823", "13551", "57404", "55387", "11139", "54526", "45541", "20171", "55459", "48349", "27927", "54196", "27062", "53519", "52481", "53833", "47482", "20371", "24102", "36288", "31644", "20606", "30479", "7083", "323", "7050", "30286", "43170", "51279", "16716", "22533", "57772", "46503", "11721", "727", "38748", "36813", "59160", "52790", "29086", "55384", "50577", "19558", "57143", "4299", "2626", "2629", "45125", "1343", "40997", "24551", "2326", "46255", "7356", "15924", "25867", "46200", "37497", "57883", "19906", "6771", "23293", "47017", "34137", "27643", "32373", "15268", "35258", "46269", "14200", "24089", "13274", "44880", "48645", "59691", "15868", "2216", "29026", "51404", "5442", "1911", "43833", "4659", "4808", "27229", "29102", "50608", "26317", "2540", "49892", "18486", "15830", "19939", "41920", "4540", "22428", "39829", "56525", "12809", "6480", "53565", "9302", "55673", "12258", "5596", "17849", "38768", "1051", "38773", "43678", "10601", "26421", "28829", "4935", "54171", "9886", "49708", "56699", "54195", "57094", "32574", "32025", "1977", "9821", "59300", "46218", "8292", "59512", "53984", "37679", "15953", "48606", "52825", "20153", "32904", "14858", "48741", "20702", "4610", "34407", "37543", "44705", "37870", "28580", "42111", "42705", "8340", "8764", "55659", "55929", "21901", "59283", "34441", "40375", "13546", "14420", "34884", "49286", "33611", "23461", "18050", "42688", "14620", "16074", "32578", "49904", "18419", "9975", "8071", "9524", "42162", "59606", "30575", "45982", "27162", "47640", "56520", "47730", "33488", "11260", "50795", "14064", "6445", "3508", "20961", "17880", "26525", "2589", "36855", "49923", "26141", "50656", "35657", "58030", "33946", "36536", "46922", "23624", "3888", "47820", "23780", "54918", "8791", "21994", "43525", "44146", "52498", "30460", "20891", "44022", "27593", "8128", "24913", "11723", "36962", "5764", "32531", "55844", "21151", "51048", "11820", "35762", "35015", "36872", "58475", "20638", "17440", "38765", "59070", "58710", "48283", "28335", "54706", "40010", "20580", "54928", "35558", "57462", "19178", "21989", "1304", "6482", "39491", "41635", "34381", "6947", "59718", "14405", "5993", "37213", "51701", "29939", "48967", "2400", "37890", "37272", "19191", "32394", "52023", "25206", "25094", "49999", "42689", "23414", "47556", "8951", "45865", "6653", "28735", "49073", "3366", "50696", "49102", "21954", "30339", "53495", "13432", "48820", "19046", "59813", "14938", "29031", "20542", "50532", "18488", "13177", "29233", "59219", "1904", "12147", "49579", "10351", "41215", "10859", "51145", "30764", "9648", "9665", "19221", "35673", "16491", "32985", "17119", "56777", "9992", "51732", "31406", "13193", "30446", "2106", "37837", "45626", "11060", "43766", "35109", "8852", "15713", "35708", "3420", "23670", "18528", "50186", "3286", "5812", "36482", "51322", "53014", "8294", "50525", "5876", "9340", "3492", "31529", "32859", "12803", "51987", "55861", "16843", "32462", "17240", "9494", "5329", "23764", "17866", "10570", "24169", "26742", "9672", "6291", "55287", "24441", "1359", "28393", "7676", "16627", "33424", "46837", "58179", "43330", "21108", "26956", "23550", "2170", "46663", "20439", "25167", "33472", "47404", "4795", "6214", "52168", "31108", "23110", "47510", "11605", "4892", "19460", "17500", "33610", "9431", "44474", "20758", "37734", "38043", "13479", "53988", "59235", "8630", "24765", "29238", "26780", "943", "25920", "22157", "38346", "16993", "26854", "22460", "28711", "52669", "42581", "49800", "37649", "15896", "13699", "19672", "59951", "51802", "2428", "37798", "24427", "6350", "15490", "40281", "12331", "418", "13632", "58307", "6592", "38018", "865", "36863", "11340", "29105", "20084", "19589", "2138", "39844", "52022", "33100", "37129", "48477", "3252", "32743", "35830", "31491", "57251", "22766", "5942", "5525", "52537", "10330", "59169", "49420", "17261", "53756", "30758", "39039", "22899", "13682", "10150", "49205", "55591", "1330", "7057", "48544", "17973", "19804", "23318", "3582", "6090", "48141", "58824", "26913", "536", "45511", "53690", "40178", "4531", "3544", "44927", "27464", "24461", "48881", "24935", "2262", "2366", "6354", "21609", "53494", "21098", "58872", "19985", "50157", "16815", "35168", "24960", "702", "49443", "26490", "823", "15288", "55639", "48814", "5898", "56796", "28678", "9411", "49023", "5693", "3476", "19823", "59604", "36184", "49640", "44126", "15709", "21822", "17595", "16571", "17578", "45965", "51159", "7564", "34616", "45625", "6770", "57795", "47706", "15010", "56037", "14417", "33375", "56953", "18627", "38904", "50862", "2695", "30808", "47616", "45499", "1317", "57688", "41167", "45967", "47035", "45951", "29389", "27987", "55280", "19910", "51739", "50489", "32573", "7386", "7403", "8729", "32057", "39287", "3445", "52192", "2438", "23543", "37433", "51888", "59724", "45616", "3299", "42265", "15862", "24592", "51389", "52927", "25381", "14698", "26080", "13010", "11144", "6117", "23835", "45851", "37429", "38546", "14904", "31025", "27855", "54084", "52202", "17311", "19290", "4725", "6134", "44284", "23169", "56810", "45921", "18721", "33454", "40008", "38168", "57009", "57755", "7376", "24329", "1339", "28477", "39266", "20480", "51025", "42598", "35337", "58218", "46242", "9225", "27885", "36328", "8800", "31175", "23569", "44694", "54425", "41063", "39932", "50797", "15236", "19391", "3984", "38763", "8655", "28490", "45607", "5568", "40051", "52281", "20734", "49247", "27485", "30769", "51178", "56057", "28359", "53126", "10529", "28008", "2960", "43257", "20170", "23141", "30777", "49204", "44409", "38473", "13656", "21587", "41337", "39672", "56381", "50584", "684", "42896", "29625", "11801", "20851", "31912", "10545", "3563", "45650", "59213", "25017", "20703", "24600", "3882", "3952", "51095", "6855", "12088", "42534", "59024", "14882", "24298", "33543", "18092", "22842", "39960", "25209", "8428", "5012", "24000", "19201", "5908", "36461", "9357", "45248", "15848", "46613", "53831", "41349", "35387", "7392", "46566", "26587", "50423", "57357", "45845", "6334", "39299", "105", "25940", "29335", "24375", "46799", "692", "9782", "41519", "51940", "55175", "57377", "33701", "14659", "27007", "24072", "34827", "42049", "56406", "20180", "14499", "46064", "29819", "35648", "4104", "31274", "31822", "12425", "4226", "5867", "49702", "41803", "53612", "55268", "33124", "17492", "16578", "7368", "2829", "6522", "54371", "1838", "17003", "33759", "47865", "33713", "22399", "46604", "50896", "36442", "41162", "11623", "52161", "30578", "39269", "44943", "33654", "55849", "46124", "32277", "22567", "25870", "58023", "49960", "7027", "23516", "50498", "20102", "33793", "2384", "18314", "40548", "13706", "51916", "7035", "43816", "31923", "1210", "11871", "16650", "1471", "29348", "21805", "3827", "36133", "2449", "43900", "32210", "14039", "11092", "48974", "56186", "1701", "49405", "48359", "17050", "47800", "45599", "17684", "27255", "22293", "3940", "44996", "22980", "47351", "38596", "18302", "44076", "27449", "27697", "57930", "16077", "581", "30868", "57311", "16408", "44625", "51748", "36223", "35839", "31769", "14533", "7425", "38960", "37633", "19816", "10854", "36726", "54609", "12346", "43526", "17078", "55339", "16140", "10684", "56782", "17777", "28468", "5902", "17352", "44093", "51431", "33936", "39821", "40789", "35819", "21571", "11361", "51539", "6570", "19312", "32650", "3541", "22084", "46137", "37666", "34909", "23658", "59301", "44643", "17613", "6295", "55715", "58835", "59839", "6148", "24615", "50201", "56803", "2214", "8920", "50397", "56895", "21067", "1506", "30586", "9939", "2543", "19749", "18999", "21532", "10638", "14205", "30864", "50362", "38279", "21866", "13924", "1766", "59806", "6163", "41915", "11565", "25189", "35930", "7882", "17743", "1502", "20631", "55444", "48190", "54798", "56584", "41198", "57777", "42467", "36345", "55607", "35971", "52426", "12598", "6838", "26872", "9419", "44693", "51702", "13765", "9498", "20575", "33944", "33510", "42405", "31416", "45925", "5100", "58063", "33985", "5228", "37726", "4976", "16242", "28839", "19600", "25829", "58439", "20741", "29945", "40180", "15202", "40818", "53895", "28438", "53212", "57653", "17160", "18161", "42908", "10722", "29263", "55818", "10166", "34163", "41958", "9660", "55764", "37197", "48370", "45074", "40135", "20963", "55307", "59031", "28850", "24890", "31956", "50146", "45154", "38226", "38993", "9456", "16002", "26085", "21647", "3464", "2778", "48916", "41850", "39806", "47110", "6687", "47731", "11415", "15760", "10569", "8610", "8027", "43553", "20870", "58341", "502", "15420", "40697", "20324", "44402", "52712", "15342", "13457", "45696", "13185", "16659", "42147", "32745", "7056", "11814", "43044", "25024", "56946", "26509", "27660", "43481", "40523", "28046", "55575", "55595", "59436", "43994", "59917", "57666", "21598", "17705", "24664", "19860", "32457", "1750", "59658", "49021", "52273", "22641", "44609", "49636", "32652", "43557", "8779", "25270", "55283", "33104", "11568", "25508", "13163", "24687", "36487", "1195", "41082", "12986", "56744", "47294", "4554", "2397", "5397", "31640", "39009", "28355", "42978", "30303", "393", "43704", "23075", "46582", "29822", "9696", "32431", "20094", "19404", "49934", "33479", "30647", "46119", "9585", "37156", "39223", "17310", "32369", "16151", "14775", "6135", "20558", "17135", "47552", "10374", "1083", "54050", "8861", "55671", "16061", "13980", "33496", "115", "54265", "40404", "52179", "925", "17387", "25726", "16868", "14750", "7370", "59172", "42123", "31809", "13418", "36130", "25470", "55664", "5791", "15820", "2560", "50066", "30569", "9471", "17891", "53904", "5067", "16595", "18687", "45351", "47185", "15232", "49226", "4614", "2404", "55610", "19445", "42540", "1575", "3383", "3601", "27108", "58325", "26763", "26695", "21081", "38639", "27204", "30031", "56115", "19870", "21682", "59173", "30763", "4520", "35649", "13670", "13685", "21777", "40493", "1082", "56344", "2985", "40309", "58284", "28681", "46238", "50543", "15747", "7419", "56029", "16632", "45671", "37722", "45715", "29812", "48639", "25230", "24692", "27341", "57751", "19358", "51635", "25786", "28415", "49294", "55549", "14981", "17056", "33703", "46715", "23163", "3788", "28603", "20977", "47050", "52360", "28333", "11327", "46455", "9372", "39862", "35626", "47068", "27507", "24107", "54884", "25738", "25801", "37583", "59352", "38026", "9562", "43947", "8651", "6219", "32123", "23668", "40534", "57274", "43929", "26671", "30828", "18429", "34670", "22717", "24542", "43183", "45680", "13271", "4543", "15121", "33073", "43194", "12746", "9421", "59518", "29092", "55370", "3183", "41956", "45108", "23002", "41318", "58556", "13390", "22327", "17952", "3450", "21075", "11766", "36039", "33395", "57635", "35049", "15054", "20114", "17954", "33344", "59602", "46143", "14276", "31540", "53729", "39001", "56196", "21358", "46635", "33961", "34888", "52007", "30336", "40122", "36922", "13308", "26358", "51343", "2616", "16939", "3126", "53829", "24022", "51704", "51692", "805", "11072", "54695", "8547", "11313", "29677", "25520", "31059", "48345", "3294", "15113", "30208", "20810", "48305", "47575", "3936", "10021", "30044", "10935", "20516", "9779", "54935", "10337", "54669", "52888", "55115", "37030", "24214", "21282", "54038", "32886", "2164", "17553", "20039", "29963", "14922", "2966", "466", "41339", "48341", "5592", "32356", "23288", "18593", "47598", "1860", "7055", "55380", "6893", "24818", "10713", "30316", "7262", "25998", "40539", "17673", "31077", "59736", "15300", "47322", "2749", "4258", "52417", "3566", "44882", "6974", "32698", "1329", "48361", "3006", "10388", "29857", "296", "25102", "29262", "23430", "52496", "4116", "53651", "5244", "41036", "33326", "3934", "33999", "23688", "45357", "31036", "38979", "11851", "56519", "25311", "19043", "48566", "40764", "55517", "4228", "14415", "34041", "16280", "11303", "11878", "45402", "36659", "24428", "30695", "14811", "26634", "55603", "44965", "13799", "42433", "41632", "14400", "4357", "19345", "23826", "32680", "27350", "56628", "15990", "20658", "35353", "55306", "7986", "11014", "7463", "11859", "56949", "16279", "39386", "17266", "35136", "3234", "43363", "56776", "56955", "27723", "11785", "19023", "18846", "17972", "1407", "51988", "2478", "13766", "51707", "4201", "7777", "25886", "8317", "26115", "16852", "48061", "30831", "53004", "46413", "11301", "19660", "45825", "40460", "17405", "49682", "51671", "7034", "33926", "43577", "54247", "18998", "46399", "19304", "9055", "25040", "4796", "45150", "21580", "3254", "47234", "44800", "27112", "6736", "52723", "50277", "23541", "37888", "23537", "54204", "2633", "49399", "17020", "19719", "59053", "8541", "44854", "54478", "55552", "25980", "57981", "36459", "25686", "33306", "37286", "11976", "37471", "30936", "10987", "14519", "32676", "56101", "23418", "30191", "45196", "4153", "39187", "46723", "46860", "25535", "41137", "50244", "26976", "42038", "4688", "49032", "31645", "57537", "39355", "14015", "30974", "59206", "39893", "55803", "21624", "57502", "6455", "19237", "41374", "21388", "15610", "257", "50663", "2962", "54453", "58705", "21104", "7827", "10356", "31451", "599", "1233", "25057", "6677", "37180", "55032", "36741", "4574", "37118", "1676", "54774", "15608", "48279", "37465", "51721", "33340", "29134", "46880", "19306", "15334", "36413", "32911", "2519", "56036", "20243", "19515", "1323", "23359", "5720", "59545", "53608", "17756", "49280", "17971", "48932", "28021", "13110", "56981", "27688", "16099", "43141", "4035", "37169", "20490", "2046", "53035", "33016", "35164", "24293", "54497", "39577", "26281", "54940", "9105", "35941", "2474", "53806", "55619", "22981", "22360", "32407", "34549", "8752", "1690", "25207", "43936", "16197", "28279", "5558", "28224", "51860", "33179", "9966", "17919", "6812", "29519", "48068", "32975", "43349", "35811", "31360", "31118", "51115", "28665", "30016", "2026", "29494", "9809", "39808", "35845", "18222", "50344", "18142", "31876", "29846", "20860", "51366", "56182", "19011", "27450", "16", "41516", "32217", "42209", "19775", "27321", "12514", "38445", "57760", "57810", "35438", "15588", "35719", "34958", "32191", "19969", "13942", "33059", "44753", "32329", "28849", "31181", "6764", "21182", "56145", "53810", "47569", "23858", "55584", "609", "54711", "50219", "46482", "20465", "18104", "6204", "1715", "13507", "21465", "26124", "35697", "18553", "40940", "14785", "36569", "57566", "55592", "12089", "44313", "19662", "8286", "56665", "13057", "50183", "34527", "13638", "1219", "22052", "4169", "45754", "31458", "58230", "37058", "11504", "59678", "9845", "46031", "27214", "55503", "55226", "7114", "13950", "36871", "26819", "3745", "53691", "53927", "4227", "57631", "55052", "47591", "421", "10301", "58567", "24", "58762", "12460", "13292", "45002", "20926", "38421", "28119", "41805", "36452", "9033", "3818", "39392", "55236", "15841", "1060", "11554", "55505", "45406", "22863", "9480", "45635", "19353", "20047", "1489", "43277", "43449", "30234", "12759", "56513", "36429", "58812", "40233", "20086", "55263", "54530", "41150", "49563", "57145", "42928", "17224", "16531", "5460", "14496", "54558", "53178", "34886", "12186", "597", "27764", "59326", "35419", "30045", "7170", "19283", "194", "5379", "24775", "37632", "13755", "1256", "17691", "23409", "49561", "3035", "29704", "56462", "21006", "42077", "51185", "13642", "50021", "36460", "28307", "54847", "40747", "34500", "45190", "41712", "32800", "37045", "24904", "14277", "20497", "39722", "27816", "49372", "41950", "12391", "8497", "33455", "54630", "40387", "14061", "45942", "27035", "15500", "26706", "23262", "750", "40578", "36174", "6318", "34127", "37504", "2201", "20232", "58231", "38022", "43144", "20499", "6678", "12354", "9553", "13835", "58784", "24273", "48156", "42412", "5975", "33503", "437", "1492", "26079", "36580", "26336", "58532", "51973", "35501", "49464", "5545", "826", "878", "39920", "30915", "42188", "17188", "14212", "26937", "23522", "17156", "28124", "32733", "57008", "15050", "19140", "27314", "2675", "44375", "41467", "42569", "12993", "34188", "3989", "28213", "13552", "54085", "40777", "47968", "24951", "49886", "29977", "7492", "50385", "37751", "38840", "25033", "42150", "12179", "20325", "19426", "59471", "31105", "24294", "34166", "59551", "2679", "46757", "17596", "34482", "36701", "43483", "42945", "36348", "55359", "37345", "45397", "19750", "19330", "51143", "44792", "7654", "53364", "4189", "30421", "37695", "7780", "41788", "38039", "41744", "25404", "884", "6374", "52095", "19447", "42586", "50574", "5490", "24074", "41175", "45542", "38336", "38796", "11644", "6879", "32137", "23723", "15902", "2717", "49629", "29653", "20173", "16473", "57775", "29446", "45026", "52901", "14748", "43201", "40850", "30206", "23384", "2343", "25133", "58653", "58920", "42102", "8566", "20005", "6772", "36224", "46432", "51489", "45292", "10099", "41487", "48935", "51774", "10035", "38966", "39002", "51999", "22206", "42026", "37603", "3848", "29661", "23707", "49849", "39882", "50495", "24313", "6684", "12558", "49233", "43290", "49836", "35374", "25339", "32426", "37675", "41774", "32885", "47205", "51335", "27457", "57885", "41186", "11165", "34933", "51380", "48585", "27345", "33139", "204", "52176", "57811", "15937", "52290", "33497", "44245", "22452", "51847", "30733", "57938", "31227", "43341", "8683", "25578", "12162", "25400", "15133", "48344", "40143", "40101", "41018", "46599", "39744", "58746", "19873", "40717", "44250", "18985", "32235", "13978", "2296", "54258", "27474", "47824", "35500", "54427", "32305", "22829", "7031", "17471", "8079", "55872", "44642", "10094", "40708", "43022", "26924", "56103", "26267", "72", "51941", "31613", "900", "12544", "25967", "3677", "33786", "10465", "39241", "29407", "31338", "58829", "37070", "22905", "26933", "42531", "48869", "25410", "35046", "42983", "37707", "9310", "29336", "1961", "4872", "40688", "17547", "17881", "54796", "6842", "35456", "42614", "18067", "44302", "30870", "58151", "12533", "49317", "28637", "17984", "7674", "7285", "48556", "29150", "230", "57115", "52410", "40837", "3028", "50585", "7074", "40228", "28033", "45928", "24180", "17737", "9651", "11864", "36496", "54170", "22055", "11729", "33907", "56257", "7008", "42488", "27360", "12905", "56072", "30701", "29286", "38249", "45524", "7808", "5654", "30236", "20957", "34351", "7520", "27509", "18467", "38922", "44074", "48251", "48941", "38049", "22416", "39995", "30906", "12058", "43873", "11815", "24776", "11130", "18039", "50462", "54410", "14398", "38789", "56025", "40027", "45582", "29131", "57905", "39876", "38167", "42401", "37947", "22356", "54715", "55353", "40072", "27645", "39130", "57474", "32089", "58523", "46166", "13151", "15564", "24321", "59178", "27306", "24001", "50085", "49170", "24105", "5479", "10370", "36550", "35718", "27612", "47045", "27441", "45525", "26399", "3380", "32040", "32006", "13351", "30853", "8507", "282", "28176", "2433", "52605", "23879", "2848", "48582", "45708", "58647", "42223", "20579", "19791", "51699", "17745", "38847", "12399", "13970", "29022", "55401", "22147", "7414", "11041", "48284", "15877", "22541", "14854", "52368", "20560", "35444", "26771", "58506", "56148", "20936", "29316", "11996", "13399", "845", "50454", "27664", "36523", "12713", "22073", "42133", "15580", "47754", "58959", "34015", "49929", "16195", "32097", "22777", "10451", "40398", "20529", "24563", "21237", "56034", "15006", "34949", "19332", "7145", "4087", "38072", "28777", "47075", "58206", "49622", "45001", "24811", "28867", "21894", "10737", "2544", "20532", "58513", "52691", "28768", "33607", "6372", "6912", "32069", "20190", "9582", "41276", "17031", "34825", "23789", "43091", "57552", "10043", "21807", "5951", "26869", "48494", "54137", "54226", "3185", "44626", "41588", "147", "16663", "6755", "17184", "56784", "28094", "42543", "53874", "49878", "15622", "57129", "45670", "56817", "13092", "44028", "41238", "58382", "47198", "38303", "54076", "33502", "17482", "9945", "26456", "31709", "35747", "48719", "13119", "39764", "50808", "25489", "56680", "39884", "1036", "21852", "4061", "45369", "42893", "42464", "51837", "29250", "47867", "47324", "17438", "41888", "45409", "12010", "28065", "48944", "35965", "33101", "23997", "7538", "50800", "44691", "35976", "59755", "42975", "13348", "14440", "46354", "1726", "42064", "38899", "16857", "20699", "36798", "48893", "34525", "30289", "24777", "54336", "18434", "6430", "11282", "5439", "8117", "27127", "43382", "259", "15090", "50455", "50726", "38722", "10292", "40048", "29787", "8735", "37992", "52705", "40885", "19328", "21313", "34853", "29314", "11696", "46070", "37190", "43281", "55921", "22449", "57447", "29850", "51519", "24988", "15378", "19867", "56804", "38002", "40174", "26398", "10118", "21100", "27996", "53681", "35898", "5404", "10568", "22431", "29252", "24426", "43523", "35100", "58159", "14857", "58056", "8471", "5128", "51769", "21942", "57914", "23769", "2336", "57579", "30226", "13711", "35207", "16638", "9719", "43134", "41552", "38774", "15655", "33783", "18021", "46176", "32519", "28363", "12597", "3403", "17383", "17212", "36218", "53199", "7668", "59343", "7886", "35586", "28957", "2646", "10884", "27102", "27856", "5652", "39103", "12271", "52069", "1053", "24668", "54157", "7844", "26779", "472", "47089", "3669", "45470", "25042", "22866", "4975", "17701", "16460", "56459", "30556", "31556", "42738", "55162", "10970", "40495", "38838", "20169", "29917", "25358", "54735", "21841", "24038", "25280", "41272", "13562", "31534", "15567", "37082", "38205", "16084", "43279", "15814", "23938", "42244", "6054", "20461", "50093", "19162", "12620", "4917", "6301", "2709", "30597", "52300", "11510", "34", "39963", "48815", "45377", "46191", "42578", "18923", "28455", "37113", "57956", "57105", "21107", "59942", "18662", "54823", "8195", "43214", "9726", "10106", "8438", "40117", "25557", "42251", "30731", "38571", "13676", "23422", "51839", "59400", "3696", "661", "50718", "49719", "56089", "43225", "33438", "33957", "31397", "6271", "27116", "3738", "12097", "39347", "38808", "4438", "23616", "29634", "58161", "41441", "22790", "49267", "41069", "33052", "25901", "14403", "17763", "37042", "45453", "37830", "16577", "24738", "11069", "53191", "56583", "50294", "35559", "2986", "7283", "48479", "14865", "10453", "31802", "43731", "12772", "16551", "55086", "26496", "55139", "827", "56964", "32200", "22466", "18449", "16107", "5393", "17258", "49116", "54286", "19504", "48622", "5245", "53711", "42270", "37284", "27024", "27508", "6836", "8194", "15774", "58432", "31425", "56396", "23355", "55414", "35302", "39541", "1189", "268", "21049", "44713", "24436", "20111", "25810", "42449", "3506", "14799", "1716", "50779", "48435", "16277", "34871", "29706", "16729", "28354", "39660", "14842", "40874", "46192", "51612", "32924", "39156", "32514", "8331", "12429", "12516", "37673", "35028", "2065", "57406", "30606", "38797", "58505", "51449", "17665", "28643", "41765", "36683", "33950", "9575", "7764", "37553", "17197", "13837", "48513", "42606", "44223", "36080", "45879", "38079", "39561", "54861", "22877", "31960", "9307", "26631", "18117", "28058", "13932", "36949", "49160", "4187", "12351", "16503", "22911", "29418", "11286", "4536", "48799", "1313", "29491", "46326", "56768", "37225", "35429", "43856", "23215", "32451", "13437", "15068", "28463", "49529", "20844", "12", "54802", "28478", "2277", "16424", "58469", "1178", "18797", "47243", "23147", "48514", "20881", "41842", "38489", "29076", "33146", "49714", "16275", "1194", "39200", "59004", "28932", "11228", "7160", "46725", "8383", "6600", "11522", "24134", "35041", "48081", "33381", "48362", "59133", "11436", "918", "2777", "29400", "21755", "16683", "28872", "7360", "10906", "8508", "3977", "43282", "24644", "22594", "46395", "59339", "50626", "47775", "10339", "17254", "34349", "31073", "21936", "8524", "15440", "39850", "34767", "49626", "44119", "34457", "9679", "29712", "24931", "52892", "47732", "56377", "22123", "4080", "13620", "11758", "35412", "11956", "19158", "30966", "19136", "46391", "21316", "43387", "5107", "56734", "20328", "12774", "12135", "71", "40844", "54463", "4823", "53554", "10579", "43612", "21372", "34463", "20356", "19108", "8763", "30132", "25650", "56228", "17395", "49662", "48291", "49092", "41395", "51184", "42929", "41348", "11171", "3051", "39695", "41000", "53535", "4055", "34032", "57194", "17270", "20321", "55733", "35410", "31738", "434", "30752", "11998", "36577", "39715", "52354", "29042", "21113", "6559", "39852", "20380", "17963", "47819", "50547", "12495", "14948", "7073", "54272", "33718", "7245", "6594", "6939", "51424", "28668", "34610", "32621", "11089", "25351", "41459", "38247", "43071", "29548", "47664", "57723", "58225", "45587", "54728", "2723", "19911", "55868", "1718", "35094", "40755", "26189", "55895", "49568", "36815", "51075", "31606", "364", "57113", "9463", "2461", "3262", "46819", "55028", "15949", "29910", "57327", "21491", "54693", "16787", "20258", "29858", "56759", "42756", "14525", "49669", "31091", "19557", "34981", "49361", "26364", "48317", "52678", "9233", "55544", "1183", "5456", "23524", "39904", "40806", "28072", "25527", "54093", "56846", "48694", "48853", "58478", "54344", "5552", "23464", "33181", "44598", "1573", "40823", "53537", "4789", "22998", "45454", "57148", "41241", "37359", "6840", "12562", "24794", "46383", "40150", "19336", "27211", "2480", "42097", "48364", "32059", "782", "35058", "40635", "19069", "15670", "30074", "46359", "38281", "27467", "16283", "53866", "13707", "25832", "59641", "17878", "11688", "2376", "6792", "41515", "16164", "202", "18623", "8186", "12649", "47397", "32348", "29513", "59992", "40207", "4235", "3418", "25688", "15522", "1129", "59535", "59154", "6091", "33200", "4053", "35363", "27778", "31532", "49400", "33965", "24190", "28546", "11200", "12617", "34305", "55737", "42933", "48710", "9760", "57352", "40522", "3084", "57808", "4913", "52597", "17145", "29456", "53207", "30827", "35531", "30173", "12795", "29740", "35354", "46411", "337", "40322", "47821", "31269", "836", "34967", "42725", "34906", "44966", "4324", "594", "53808", "13031", "40924", "12435", "30119", "46190", "163", "10047", "26419", "571", "32091", "57290", "32707", "43759", "21340", "16913", "18408", "30193", "13000", "30376", "47283", "43890", "26465", "44581", "35327", "709", "4148", "23690", "20235", "30494", "48150", "5481", "30188", "34173", "55679", "18097", "46987", "21273", "47101", "55488", "28700", "34678", "5202", "58390", "31955", "49506", "3070", "9145", "15208", "39060", "3268", "58848", "19935", "21390", "57336", "54616", "46403", "29821", "36007", "15683", "50510", "57993", "31862", "46652", "14729", "8899", "54", "13085", "544", "36168", "40545", "11790", "19612", "22212", "23849", "12589", "1731", "16682", "15645", "6828", "45432", "33787", "2577", "7101", "36457", "54175", "19964", "37944", "1853", "26091", "27497", "21693", "47792", "48078", "43353", "35860", "785", "41490", "11759", "15861", "4353", "58837", "18397", "17540", "18243", "53264", "2324", "17873", "15390", "38359", "14258", "57732", "27634", "26254", "43014", "45316", "30391", "47449", "2473", "34033", "40459", "54838", "24719", "13076", "33167", "44080", "43952", "3141", "37005", "19617", "47621", "20970", "41979", "46580", "5841", "18724", "10119", "12503", "30131", "6124", "34655", "46472", "58666", "14100", "22376", "26513", "27495", "58311", "16731", "158", "42994", "11806", "52619", "35669", "2253", "58496", "10267", "45451", "14271", "53115", "35889", "22992", "57154", "10182", "11798", "50831", "49925", "7475", "968", "49796", "55657", "2208", "59341", "28951", "109", "57240", "15946", "57554", "14136", "4491", "6880", "24620", "6620", "54486", "47181", "8663", "18379", "18569", "14366", "31363", "17465", "12879", "59177", "37310", "36120", "30619", "1833", "53304", "33448", "38793", "21150", "19805", "7461", "10319", "22404", "5485", "9371", "2327", "30362", "4805", "2500", "33696", "43027", "35689", "39229", "8793", "48055", "48333", "23316", "55208", "19550", "57366", "42748", "28405", "59748", "18582", "25562", "4932", "56911", "45368", "46260", "34037", "53699", "24963", "46702", "9567", "9158", "58483", "37523", "3579", "7949", "4624", "585", "29090", "51061", "17388", "18471", "51029", "2059", "21738", "28828", "57665", "56585", "54755", "23203", "55876", "13115", "54307", "57802", "55113", "4306", "33655", "24259", "48397", "59562", "34454", "36248", "32515", "47741", "28593", "55120", "37938", "38693", "1892", "40908", "35616", "32580", "41783", "5429", "17084", "21934", "24480", "53243", "14803", "3057", "9188", "25386", "32021", "58290", "22884", "17815", "17076", "20742", "33668", "40860", "24756", "24917", "35320", "31730", "44103", "30809", "987", "14836", "20591", "30624", "12563", "51069", "50834", "18910", "45182", "57047", "53689", "41732", "18828", "14738", "1080", "9427", "50447", "35717", "34191", "42324", "41643", "26380", "57568", "9013", "7774", "35853", "17015", "40576", "8645", "13664", "57288", "1375", "17346", "44890", "4175", "32811", "1907", "28313", "41778", "58409", "29544", "5721", "12246", "26062", "41505", "28523", "18040", "4401", "26603", "7943", "30152", "40992", "53882", "23095", "19379", "9227", "44608", "57426", "27058", "27232", "46340", "5403", "13189", "24698", "8344", "39490", "45945", "37808", "59967", "57049", "14765", "7870", "28101", "2861", "54452", "14522", "43819", "24130", "49543", "47740", "43518", "46082", "50342", "37794", "22224", "52761", "53107", "3179", "42762", "51484", "47303", "50655", "40412", "36567", "44470", "41281", "13206", "29246", "42161", "9342", "18903", "29784", "12817", "38204", "7484", "23746", "51520", "18856", "39279", "35128", "46459", "45467", "21135", "37147", "33335", "28914", "38568", "53394", "29464", "45766", "28136", "41587", "35386", "49469", "18713", "5395", "4569", "8757", "54563", "36162", "11024", "2851", "53180", "51253", "4638", "13134", "43894", "36648", "41688", "17171", "54408", "19434", "14885", "12855", "37479", "29535", "9000", "59284", "8361", "55668", "9834", "41228", "20058", "38780", "31944", "33458", "34403", "11211", "24536", "21398", "27514", "16068", "9683", "26410", "11206", "41052", "50028", "35865", "7746", "262", "35576", "21482", "45598", "50448", "8595", "26705", "20186", "57367", "43805", "51363", "53605", "34851", "41984", "52036", "11827", "56541", "33449", "45629", "6153", "12365", "430", "55924", "7856", "34448", "9613", "10719", "47245", "14291", "15612", "48447", "2077", "2980", "43362", "58502", "38634", "57107", "50247", "48310", "2590", "56827", "3778", "51353", "18507", "47799", "2531", "17865", "19730", "31898", "34153", "59656", "10659", "51326", "52151", "9434", "53233", "59194", "57060", "18107", "56168", "40527", "54853", "16541", "19706", "58579", "57355", "604", "5532", "33203", "49185", "18932", "11948", "12719", "26546", "8382", "34101", "3793", "44075", "14555", "10405", "57511", "42925", "2123", "43359", "34167", "59498", "15245", "41166", "50347", "46295", "1978", "38177", "17740", "27348", "39516", "2163", "11112", "6624", "50356", "25890", "11958", "27640", "37776", "3922", "4859", "58926", "57460", "820", "7135", "40040", "14773", "255", "42722", "28457", "37764", "449", "39907", "32409", "24464", "23398", "38875", "21946", "28926", "17932", "15840", "58931", "13547", "54398", "23948", "54932", "623", "52005", "54515", "55193", "7136", "51787", "53077", "59429", "31018", "6323", "4511", "58538", "7272", "27059", "42702", "1632", "31368", "45060", "41022", "28812", "41414", "8213", "630", "35239", "37385", "40906", "28727", "29897", "25573", "26889", "16869", "21861", "30072", "58652", "59307", "46629", "30669", "23854", "37323", "12541", "24479", "1394", "16567", "43674", "48124", "36653", "32764", "10271", "29710", "48504", "22639", "31015", "14526", "31104", "13788", "11294", "59970", "21516", "9270", "153", "41649", "36275", "5295", "2066", "7273", "2497", "28736", "2090", "29835", "39770", "22186", "13784", "50311", "6569", "27305", "32619", "39540", "41777", "59888", "50825", "45497", "14164", "18757", "38483", "49161", "28111", "15039", "21748", "30858", "37392", "15606", "54260", "5111", "58068", "31871", "22634", "21702", "9499", "36510", "26716", "55762", "9667", "2873", "634", "55305", "21210", "53775", "56107", "1662", "29638", "23496", "56883", "43695", "59731", "43414", "23465", "45819", "12957", "16714", "31765", "23565", "34369", "32898", "47100", "44432", "15998", "53230", "13349", "11677", "4837", "18829", "23966", "37379", "43435", "43197", "5237", "6126", "3749", "58869", "12008", "52900", "17537", "33356", "56177", "38804", "31830", "14014", "3845", "9221", "16476", "52382", "34779", "38337", "20587", "19886", "21279", "10010", "29007", "55319", "6391", "4302", "48436", "59929", "28821", "26321", "14984", "48357", "4542", "13986", "34507", "6961", "41684", "52074", "40927", "49611", "36240", "20314", "52676", "51587", "20030", "13967", "38832", "51261", "34021", "26068", "8258", "41939", "27784", "11902", "16278", "49222", "3058", "15085", "45644", "32258", "11232", "17496", "33295", "7024", "26542", "8923", "23996", "20033", "20653", "24261", "29055", "30850", "26607", "6683", "23832", "331", "27406", "38195", "58935", "3890", "35807", "9876", "41752", "47588", "31861", "58449", "25818", "22956", "15824", "46199", "17247", "56523", "15514", "15752", "6212", "45092", "58189", "14571", "57692", "24519", "5770", "45632", "2809", "34849", "57975", "50199", "28519", "20045", "563", "29682", "17479", "14649", "24770", "22828", "36845", "53183", "20382", "49355", "59332", "4006", "50651", "20595", "57678", "50535", "26896", "17058", "59846", "43445", "48769", "27889", "57813", "55485", "20730", "32051", "6655", "16349", "57968", "56552", "6867", "10905", "51314", "8388", "51989", "1652", "14230", "27971", "43864", "41260", "30386", "6002", "17839", "17581", "8472", "36521", "11031", "44586", "43787", "25071", "28132", "23186", "44547", "58358", "43041", "56858", "20217", "21420", "52108", "27849", "25159", "30178", "5987", "42280", "52713", "870", "45758", "19060", "18682", "33881", "40756", "46202", "32721", "380", "40050", "3139", "47753", "16599", "7955", "154", "51509", "59852", "39772", "5083", "18279", "7697", "41077", "35585", "54914", "48626", "8030", "58783", "3289", "21671", "18185", "49921", "14107", "16925", "59653", "3800", "15428", "36297", "23277", "3764", "5551", "51501", "56014", "4308", "30738", "20032", "38342", "46569", "29746", "47225", "29999", "18863", "58402", "56982", "28788", "40461", "1307", "18442", "11149", "6460", "39439", "42684", "2052", "49553", "13207", "55413", "28840", "22072", "55695", "59448", "11342", "46560", "33066", "846", "18451", "41943", "5292", "52424", "23436", "4206", "6853", "20501", "4880", "25964", "50479", "14715", "12419", "21574", "56716", "45298", "44194", "29913", "48302", "329", "30852", "55224", "35191", "47147", "13592", "50185", "45326", "47164", "22660", "2684", "42453", "33404", "53468", "24597", "5813", "58992", "34283", "49018", "6591", "5723", "42298", "19393", "44838", "10437", "46956", "39112", "45047", "45259", "18547", "28953", "10364", "49979", "39532", "906", "6496", "44601", "22928", "1151", "40305", "17825", "2716", "56674", "42604", "16905", "47505", "731", "17176", "4024", "18778", "47942", "45972", "19711", "53983", "38496", "59572", "48320", "5683", "32696", "17463", "7839", "50335", "12433", "17202", "53578", "42355", "38934", "22490", "59986", "18536", "44343", "54806", "47512", "50671", "48994", "44755", "15731", "25402", "20", "39385", "46706", "1480", "1463", "21712", "14764", "36200", "48575", "55620", "29230", "29572", "11907", "16692", "17790", "37922", "42277", "38592", "35206", "10933", "58258", "493", "32882", "52059", "51172", "44754", "2207", "52991", "31103", "21246", "54632", "12343", "8210", "47690", "16512", "1078", "40366", "59201", "59569", "7501", "14992", "15639", "43010", "25911", "56316", "5254", "4170", "51420", "28037", "26887", "39677", "53146", "10898", "38011", "1727", "29895", "29520", "4409", "58593", "33917", "36370", "1011", "13431", "21305", "31990", "48785", "44029", "9784", "23874", "55699", "18095", "53818", "47296", "41205", "45545", "40773", "47716", "2351", "50549", "8025", "26687", "43725", "12777", "51895", "41917", "7237", "44289", "43408", "44829", "53170", "59267", "53235", "17008", "32963", "21512", "56325", "11046", "46588", "7309", "53757", "10404", "58964", "35170", "47128", "22234", "152", "44044", "383", "46224", "15276", "16842", "22204", "5913", "35985", "9370", "40965", "41017", "46385", "20920", "9344", "16342", "29536", "58983", "44561", "14901", "52493", "6634", "45013", "56296", "48259", "51902", "11840", "53291", "18850", "41821", "7716", "25330", "3355", "38927", "58621", "4665", "51306", "12978", "21430", "12458", "25507", "48492", "23982", "15178", "8122", "51310", "2920", "4336", "15302", "26152", "9820", "465", "31193", "28845", "21284", "20485", "53276", "5457", "11936", "9698", "41332", "17682", "19217", "11145", "41358", "9378", "28474", "19111", "37580", "8877", "36950", "42713", "8161", "2612", "25196", "52266", "48539", "2445", "36747", "28459", "9731", "50254", "37189", "30147", "42362", "2365", "43680", "39259", "5613", "17580", "1019", "19354", "14146", "11639", "18495", "27264", "20038", "58383", "1552", "34731", "26307", "49319", "21906", "6959", "43975", "51676", "31393", "17052", "40115", "24559", "4734", "869", "24821", "25328", "33567", "31436", "8417", "10580", "4475", "29058", "3446", "1698", "54673", "37812", "54987", "42989", "23601", "49728", "59930", "50680", "23118", "25066", "19602", "48463", "41630", "29541", "57172", "16320", "18049", "43502", "29006", "45251", "26391", "16605", "50037", "31945", "29392", "33990", "27479", "5932", "5873", "47998", "37072", "16419", "12755", "30122", "52647", "4504", "20774", "38242", "27606", "1799", "23911", "21189", "41881", "9865", "34328", "13813", "21837", "58194", "29256", "52743", "16483", "15981", "13937", "18575", "37383", "35067", "12283", "23697", "34447", "46792", "29509", "56431", "56524", "58580", "28036", "2784", "928", "26536", "35617", "5231", "20748", "56047", "53398", "29451", "9409", "9050", "8876", "21072", "680", "33134", "27958", "43203", "47697", "59434", "51472", "19863", "23892", "45287", "37670", "26878", "46619", "32173", "56553", "55342", "54886", "456", "26155", "30618", "9928", "40087", "3872", "41123", "25834", "14889", "8404", "26177", "53820", "56598", "33735", "43028", "16302", "31411", "50143", "51286", "51346", "41853", "53033", "11990", "18267", "15079", "8706", "57090", "55865", "25272", "31719", "47647", "38836", "34443", "30372", "27847", "19526", "46935", "39289", "32471", "35557", "51778", "57761", "51773", "21222", "24543", "36742", "48653", "25856", "6558", "35736", "8276", "44516", "42633", "4256", "51709", "7408", "45673", "4370", "45161", "40636", "19739", "57304", "58278", "20813", "25644", "31562", "9446", "20306", "32863", "39705", "43397", "40378", "51106", "31816", "14108", "13849", "25706", "19368", "14314", "41120", "54119", "135", "35507", "38656", "29401", "22343", "15886", "37088", "9671", "36189", "47063", "14568", "2040", "49450", "48855", "49638", "5278", "18382", "25290", "52764", "45780", "37000", "12780", "33246", "38628", "39079", "29950", "23138", "33513", "23494", "56278", "33274", "47613", "10727", "22969", "59508", "58839", "8345", "55965", "19259", "29172", "49459", "2592", "54133", "15858", "50113", "14304", "3593", "57482", "51348", "730", "55771", "6023", "9402", "19958", "6274", "47326", "5852", "36365", "50559", "57815", "16051", "57582", "56740", "56549", "41898", "7983", "37077", "9171", "31101", "1991", "33785", "28315", "33007", "26248", "50031", "28844", "18926", "36195", "3830", "52974", "45510", "9986", "36229", "29381", "23514", "40123", "5198", "47913", "56239", "42936", "36302", "24797", "42919", "27824", "49974", "15753", "14165", "6708", "36374", "50371", "14489", "15690", "2516", "59750", "38306", "28062", "22436", "28814", "28240", "14661", "45010", "37859", "38626", "12796", "41964", "21143", "16021", "14964", "29040", "40630", "24705", "50687", "28447", "45334", "28720", "4555", "18060", "48969", "42575", "17773", "53590", "47923", "57061", "39014", "13899", "53982", "3513", "36156", "18710", "11591", "32624", "15882", "4633", "34310", "16889", "20543", "22964", "34496", "3999", "33932", "25046", "14329", "53615", "31831", "8953", "55999", "50934", "38843", "28248", "54579", "32507", "21300", "19540", "40904", "50693", "34365", "52390", "33899", "16718", "16548", "44324", "57280", "44450", "27812", "13357", "45195", "40905", "31922", "49145", "44712", "28919", "52693", "2610", "30461", "16957", "21973", "5900", "59573", "52101", "48472", "42631", "33995", "39826", "53954", "31267", "43738", "32128", "29463", "36076", "38013", "2179", "59054", "18468", "56972", "22130", "15755", "48921", "54031", "19115", "6283", "53415", "39492", "57003", "47830", "54214", "58240", "41235", "37540", "23713", "37444", "8112", "38937", "57988", "32322", "45725", "45612", "6670", "27710", "13436", "54759", "16001", "24669", "18150", "51751", "5762", "14970", "31515", "8694", "12664", "35957", "38850", "7257", "12577", "33992", "57880", "55636", "47403", "15186", "28223", "2007", "54160", "41980", "49214", "33260", "11692", "56613", "4396", "43944", "41995", "51289", "30747", "27136", "11594", "43907", "18160", "17475", "40942", "1368", "57870", "12797", "28993", "44762", "32268", "28423", "22209", "8461", "57847", "4797", "42838", "48050", "57158", "13157", "29060", "5519", "1358", "35676", "33308", "15263", "10804", "47367", "57402", "11880", "52040", "25843", "59027", "5697", "30925", "33688", "27212", "45618", "52038", "35204", "11531", "19798", "12467", "43308", "24365", "48732", "55583", "33551", "58321", "30058", "27759", "52305", "32517", "2799", "23599", "17627", "17226", "20621", "5483", "23298", "7155", "5146", "3676", "30424", "36895", "6223", "23127", "41935", "39321", "8408", "24562", "57224", "55126", "17002", "39314", "44823", "53769", "43939", "27857", "56697", "52215", "9805", "13808", "19517", "33977", "17065", "17165", "12728", "24356", "47803", "36492", "35159", "56440", "25603", "15022", "24633", "1560", "40760", "57797", "33473", "58361", "38488", "8958", "42976", "53787", "29532", "26630", "30279", "58461", "27095", "31431", "35201", "3184", "44147", "19637", "34688", "43063", "1988", "58178", "25414", "50587", "41647", "8741", "26253", "37516", "12115", "4718", "7560", "47949", "36378", "12358", "52267", "39659", "20641", "8668", "34543", "30620", "54506", "55155", "59182", "52730", "10185", "41615", "44758", "59387", "33092", "53294", "6595", "51878", "50979", "20671", "48555", "24408", "20686", "37097", "28451", "23872", "12929", "1426", "57636", "30930", "58126", "54331", "23612", "56306", "44276", "28144", "56700", "14547", "46451", "26430", "21264", "59331", "55534", "1940", "14608", "4414", "30648", "31718", "21541", "21344", "50190", "56117", "8350", "19943", "5287", "13272", "46284", "47617", "1406", "48990", "1828", "44940", "47698", "44188", "45387", "36844", "16973", "37782", "26529", "31729", "27315", "51543", "50283", "35522", "59976", "42340", "30360", "22844", "33221", "56297", "549", "25694", "50327", "26512", "11896", "29912", "53147", "1105", "5044", "43505", "28555", "27310", "17575", "54372", "37780", "35723", "23923", "2984", "52615", "24902", "1793", "45569", "9937", "35027", "41078", "24460", "10635", "20973", "32662", "8457", "25919", "46912", "7120", "21980", "11084", "22950", "58912", "40685", "5859", "1558", "22850", "11042", "33552", "39935", "38920", "55153", "30819", "37807", "20670", "12802", "11764", "50069", "30947", "52837", "40690", "31433", "40864", "59401", "49357", "3439", "8072", "41020", "5178", "7940", "53549", "36792", "56401", "43127", "7519", "5620", "27993", "15012", "19204", "36678", "41192", "57942", "55399", "42829", "19089", "55218", "22478", "38416", "18601", "9603", "15324", "51980", "17622", "40646", "55989", "6473", "21648", "39249", "32269", "38442", "44523", "20901", "34363", "45217", "18616", "39692", "25389", "8557", "58004", "33640", "40304", "28659", "1029", "8403", "9424", "21419", "32609", "59877", "11532", "29880", "45814", "6561", "22058", "40621", "36924", "38326", "35241", "16410", "15355", "57489", "59181", "11901", "45412", "25849", "17112", "20008", "39349", "2443", "50274", "44719", "23149", "27280", "40111", "46958", "32830", "50673", "37880", "21405", "7344", "53652", "21992", "41133", "27565", "37935", "52595", "54739", "30983", "41080", "8458", "45242", "3642", "50394", "27815", "58981", "17018", "33337", "46027", "39615", "56956", "2628", "45018", "52154", "18034", "13833", "44747", "8564", "55309", "48219", "32735", "21788", "1862", "59038", "55194", "38261", "50594", "56262", "55212", "49183", "12579", "49535", "47180", "16630", "8201", "20158", "36406", "50725", "32653", "57487", "18389", "52931", "3545", "53248", "25625", "2350", "36738", "1626", "23620", "57770", "18198", "30516", "51944", "35753", "3349", "24333", "22752", "28506", "9214", "7047", "25164", "32880", "55712", "52610", "59815", "45691", "13558", "15569", "12291", "55364", "26483", "46780", "55850", "558", "10467", "26895", "27758", "1429", "28407", "17288", "8558", "48306", "50712", "44695", "56032", "21875", "1349", "18073", "44739", "52323", "40569", "17766", "14813", "5926", "29169", "9912", "13564", "28905", "22832", "42939", "49954", "29862", "18938", "36417", "23051", "29493", "2719", "34754", "54916", "13824", "29549", "53835", "21351", "12071", "27642", "39630", "20353", "45490", "53953", "14252", "9264", "335", "15478", "21938", "30029", "56236", "42765", "16465", "3046", "50697", "55774", "15513", "18920", "45172", "14871", "20745", "19438", "58667", "42398", "50393", "9175", "23433", "51354", "42812", "11772", "21180", "50493", "45559", "35117", "57774", "43425", "8372", "25143", "58304", "32995", "24660", "8817", "40801", "39417", "24469", "52861", "23459", "49199", "57773", "28782", "43666", "29084", "44207", "40611", "34440", "40507", "44788", "48976", "14086", "35931", "59473", "11503", "58791", "45058", "4162", "57420", "46936", "12201", "3116", "46645", "45095", "4708", "58111", "20964", "16272", "27703", "41068", "24449", "56923", "17974", "8605", "54619", "46256", "52334", "59521", "50981", "56334", "40914", "42615", "56136", "524", "20127", "50609", "23681", "51429", "45664", "41051", "25969", "42799", "39305", "41636", "53872", "4593", "13023", "50053", "38518", "9835", "51729", "38557", "39507", "23484", "37696", "622", "23385", "7235", "57536", "36705", "46180", "38711", "48120", "38607", "13328", "39875", "409", "12765", "32777", "5778", "2255", "16676", "7936", "40752", "31755", "45871", "32208", "57784", "23596", "22388", "39069", "39413", "50158", "10127", "17762", "23684", "44911", "38515", "5505", "10881", "36598", "21824", "48895", "15636", "38925", "53475", "9257", "32577", "9771", "22968", "59651", "29624", "1943", "6398", "18296", "13323", "56071", "16779", "57522", "24908", "53530", "25952", "24399", "16784", "13692", "36592", "46055", "49747", "31032", "54077", "19408", "2921", "24532", "40395", "14147", "36754", "28312", "38448", "50476", "31947", "38767", "47172", "23020", "21476", "54234", "28743", "11094", "22426", "46950", "26627", "9384", "18286", "55891", "35124", "14436", "48531", "50977", "232", "19957", "47151", "33206", "22387", "15968", "52702", "37876", "57253", "26856", "14030", "13686", "2513", "59557", "48783", "43420", "32026", "14637", "44678", "53105", "22386", "16004", "4392", "31667", "1483", "42720", "23951", "22757", "16722", "56497", "11213", "59303", "14346", "19134", "27117", "14769", "22121", "25685", "48856", "20418", "40187", "6246", "14211", "46034", "16644", "55969", "32692", "2459", "5605", "21077", "59800", "30333", "11233", "20972", "14523", "45656", "26109", "58452", "36804", "28865", "8592", "55648", "10357", "31785", "15237", "34204", "10289", "38169", "47312", "20293", "30994", "46267", "6004", "56092", "34974", "22229", "42673", "58800", "22053", "32443", "58081", "12295", "7749", "39453", "36259", "59610", "56707", "9023", "59635", "48844", "43059", "23104", "25100", "42831", "59583", "49559", "50665", "54481", "31335", "26781", "27846", "4122", "34704", "24909", "54883", "8480", "51216", "17572", "27513", "49685", "10234", "6045", "52552", "27416", "38141", "29240", "10219", "53722", "54505", "22256", "14444", "8229", "53145", "34542", "53678", "40724", "59618", "53999", "27942", "47638", "32466", "24522", "33495", "21976", "59351", "49094", "21073", "50686", "59082", "50074", "29166", "22348", "24123", "58870", "15138", "53761", "33377", "315", "14356", "5540", "43133", "28293", "42010", "18415", "44337", "339", "33031", "19789", "6921", "54997", "41237", "15264", "46300", "6757", "47972", "31090", "12622", "33143", "28266", "5564", "35961", "47107", "57858", "8822", "2818", "33150", "51210", "56766", "1871", "18047", "40126", "25267", "26", "24501", "32929", "798", "6022", "16792", "21890", "38685", "29615", "30026", "20572", "46622", "586", "52185", "22975", "20212", "7011", "7946", "45872", "19933", "28442", "14377", "14380", "53071", "53669", "49713", "34446", "18100", "4641", "53625", "41142", "16885", "523", "23518", "37499", "56022", "35735", "17741", "52113", "33301", "59245", "18287", "43620", "38089", "51654", "49851", "16861", "13901", "50744", "9882", "31488", "24760", "8205", "41933", "58668", "33354", "24407", "17108", "41521", "29692", "37975", "26990", "45021", "26798", "50295", "42061", "37512", "24866", "3991", "40274", "38288", "52054", "31438", "35465", "2836", "29080", "59195", "9762", "55015", "26011", "39691", "17411", "11833", "54688", "44455", "14713", "47086", "18174", "14052", "14082", "19762", "20436", "8660", "37702", "59985", "53766", "44730", "11480", "811", "2073", "51685", "37979", "54879", "31323", "46382", "5763", "46053", "31000", "17976", "48453", "20390", "38125", "18996", "3709", "11858", "13647", "39990", "11811", "17938", "48295", "46980", "29971", "31367", "18070", "32678", "32731", "26520", "56274", "32756", "41035", "27701", "10541", "9787", "29529", "27574", "11509", "59642", "2647", "36635", "57372", "16190", "27009", "50040", "36826", "47466", "27835", "1588", "40494", "10697", "23822", "14962", "57096", "45961", "12678", "14772", "4034", "38709", "42589", "25779", "21713", "18086", "39472", "20513", "5399", "46967", "12580", "35385", "30629", "11539", "1949", "55044", "29241", "20676", "33242", "5030", "43326", "37116", "8987", "45834", "47290", "39525", "2349", "23341", "30548", "1969", "37610", "13215", "53088", "84", "23136", "31295", "49240", "37869", "2372", "56050", "17893", "21058", "250", "17412", "40963", "21691", "19286", "39034", "56386", "53957", "20803", "22680", "50569", "35783", "16057", "58763", "9804", "40071", "57826", "59957", "41103", "59946", "48433", "48058", "26549", "37179", "32823", "40254", "50786", "57034", "24827", "16172", "27187", "25387", "26514", "22222", "35244", "21948", "45645", "15403", "39317", "30617", "8936", "31319", "29302", "52842", "18666", "45658", "38222", "37318", "53373", "1989", "51786", "25539", "847", "7387", "47722", "7755", "36800", "58018", "2316", "23426", "46024", "29738", "45534", "50460", "14868", "25075", "21767", "23530", "50803", "46469", "58001", "51629", "27673", "36098", "24015", "3394", "59224", "46614", "54744", "25083", "31248", "58000", "29909", "30644", "27480", "16969", "12807", "57028", "24148", "49610", "47709", "10803", "28063", "51929", "58535", "37588", "6951", "19907", "33107", "27529", "822", "14005", "8045", "42069", "40934", "28805", "18730", "40700", "49122", "48214", "23423", "45700", "33726", "52513", "57676", "35802", "18657", "5197", "47791", "17024", "24487", "46720", "13793", "57796", "22989", "9216", "24948", "26266", "15028", "42352", "22831", "18485", "56207", "38404", "300", "908", "57563", "32314", "56398", "9985", "56994", "48641", "29675", "37182", "16976", "14365", "13930", "29068", "47484", "51219", "46776", "42331", "15972", "35962", "40062", "26249", "17887", "26119", "51575", "39574", "34942", "41867", "14391", "13965", "34657", "39023", "43826", "37874", "24465", "12301", "49132", "46241", "33891", "35999", "6298", "54021", "39327", "46941", "53643", "44793", "28638", "15049", "17035", "24039", "20024", "53883", "34406", "1674", "3879", "4027", "12251", "23895", "46930", "34606", "24175", "53764", "31157", "34646", "28775", "42282", "41696", "55957", "6052", "13995", "50041", "44864", "54047", "38642", "23878", "20129", "31787", "53791", "24814", "5523", "3388", "50557", "8379", "23999", "11872", "18978", "3023", "26894", "57857", "45750", "55840", "30686", "43105", "38106", "22812", "1854", "23", "57021", "5360", "30483", "36575", "54303", "45070", "4411", "15474", "35452", "41163", "20538", "31828", "56005", "12289", "45634", "8614", "2383", "32320", "19050", "42301", "25029", "33471", "43082", "29344", "50253", "18129", "14333", "49098", "40367", "39482", "17071", "15873", "57298", "57576", "21616", "7089", "32710", "15788", "17204", "16932", "18744", "58463", "10836", "7063", "26269", "30245", "229", "1120", "7178", "13506", "12453", "8464", "15554", "29063", "40365", "1376", "52933", "40744", "3536", "14410", "36533", "26178", "18763", "18375", "42512", "41741", "6462", "19782", "15338", "59122", "40032", "40946", "55760", "29641", "53444", "41761", "16808", "58336", "44144", "3323", "56141", "12415", "41549", "46944", "26221", "50503", "19875", "15964", "58098", "11822", "55693", "58740", "41700", "57324", "23347", "5345", "48948", "5642", "30908", "51372", "47757", "47277", "33251", "28901", "35183", "16176", "21494", "12494", "46691", "38632", "26818", "41994", "41400", "27077", "30189", "59821", "26418", "4397", "43966", "25761", "22740", "26159", "8830", "46114", "2165", "15850", "18637", "58117", "4403", "30325", "48228", "19044", "28121", "3982", "8765", "38055", "43086", "44532", "16789", "736", "27761", "12623", "13886", "8188", "4550", "42490", "25664", "5623", "14610", "56318", "47433", "45555", "53060", "34618", "42810", "19686", "58543", "44951", "33353", "8252", "16137", "20921", "41161", "54056", "815", "5080", "19402", "14001", "12255", "5591", "51583", "32820", "49597", "17303", "23585", "8267", "8543", "13228", "16085", "55757", "6285", "52518", "15944", "55308", "36375", "25413", "37980", "32962", "38134", "11219", "3331", "8059", "52558", "11304", "22601", "24849", "2118", "53256", "48960", "46029", "5252", "51487", "2180", "7238", "38892", "11875", "57279", "35432", "12447", "42855", "52875", "11205", "59533", "7649", "56509", "47828", "46335", "6846", "35492", "37627", "17306", "38714", "33140", "2285", "19792", "15748", "31663", "57471", "41586", "27878", "34128", "11632", "20690", "5584", "49287", "14908", "19219", "52264", "51311", "6902", "14673", "17085", "2757", "32433", "9783", "58577", "6086", "57624", "50309", "7068", "17433", "2239", "4142", "42153", "38149", "9908", "38239", "6584", "44542", "25577", "54106", "4054", "58924", "13934", "37354", "10581", "59117", "26870", "55006", "16145", "20004", "19190", "59041", "13222", "30630", "23162", "12644", "59298", "5601", "20617", "32687", "30172", "48303", "43500", "13511", "33898", "15512", "20655", "17857", "35533", "39582", "4149", "14025", "6556", "38166", "7127", "36332", "59576", "210", "12853", "10423", "18381", "3681", "43484", "43519", "46365", "31483", "24454", "37031", "32494", "59098", "51955", "33698", "6510", "50456", "2203", "15597", "32761", "24279", "2950", "30862", "19615", "57206", "7286", "4361", "16639", "2819", "56828", "9485", "42643", "37821", "55107", "42537", "7173", "225", "58884", "46784", "29242", "49688", "31875", "6802", "6984", "9748", "22765", "49163", "9185", "34714", "34741", "55272", "59961", "4194", "53907", "19575", "27013", "55085", "46884", "50050", "23899", "41563", "22088", "50375", "56861", "4757", "29049", "50380", "40438", "19606", "57580", "25821", "55493", "6647", "14926", "12326", "22519", "26640", "38614", "19383", "34577", "57013", "52449", "17394", "25198", "48903", "52444", "18861", "58960", "42317", "14786", "55419", "6822", "23897", "46920", "34782", "49946", "26431", "29036", "32779", "11403", "28056", "5468", "13324", "47042", "26182", "42417", "3961", "8225", "747", "51457", "2849", "15894", "39756", "48203", "52380", "47634", "15108", "56283", "37136", "54636", "6358", "11498", "15179", "57827", "11672", "55554", "31011", "25302", "19490", "15235", "45032", "8818", "52777", "48464", "44027", "32159", "51687", "19375", "8728", "23081", "8690", "53698", "58348", "24049", "15653", "17788", "55546", "44208", "23703", "48601", "11792", "312", "28621", "28999", "50464", "50810", "19794", "26163", "12655", "17505", "4986", "37976", "12237", "56656", "10748", "51656", "3721", "29167", "7615", "19492", "47969", "35623", "37164", "43697", "34108", "29421", "38761", "50038", "37453", "46494", "29801", "55406", "10917", "49915", "13287", "20958", "39624", "28707", "2918", "59249", "44816", "36634", "4740", "9144", "49313", "24568", "45385", "20446", "40103", "52687", "57454", "11910", "3440", "34143", "51557", "23272", "46427", "26945", "4946", "37978", "38373", "47978", "30038", "32246", "35924", "46095", "19659", "19731", "36746", "37556", "10123", "42315", "54648", "46404", "58691", "44181", "39095", "44941", "54323", "14612", "44585", "9974", "52567", "16954", "57698", "46975", "43753", "20444", "40374", "43603", "18974", "20968", "26953", "37493", "57484", "4680", "12459", "23303", "35940", "20264", "57137", "44201", "50120", "26947", "22441", "36902", "50226", "7020", "6063", "1660", "47921", "49231", "22284", "50403", "41526", "34341", "46343", "39521", "40795", "23704", "15170", "44121", "37168", "17134", "40683", "98", "41243", "34756", "38978", "59311", "52580", "58788", "11653", "47953", "269", "56590", "42191", "25148", "41090", "22826", "4451", "13817", "1245", "32330", "27254", "27245", "682", "24774", "51600", "41550", "30527", "55559", "47287", "46134", "47399", "42346", "41861", "57559", "32081", "31655", "21387", "47646", "18272", "19419", "12799", "28544", "57952", "5215", "12214", "55487", "48381", "8554", "7211", "37555", "42128", "36629", "30917", "33062", "8957", "7222", "33193", "36362", "47369", "28177", "28649", "13048", "21727", "10551", "32923", "51433", "14719", "51244", "52828", "13589", "3700", "25526", "83", "27527", "30225", "644", "44932", "4529", "32315", "29328", "4603", "9365", "55328", "37469", "47030", "34925", "22353", "49755", "57851", "21802", "5768", "8539", "24616", "23297", "55536", "46815", "14129", "9934", "3985", "32254", "42235", "43067", "53300", "6730", "45414", "31048", "9654", "7797", "59360", "21640", "6636", "8016", "51486", "36899", "45458", "43253", "28255", "46903", "49971", "54931", "59617", "37317", "21535", "56010", "54032", "35671", "57458", "2765", "14762", "7826", "54653", "58450", "54965", "21136", "30275", "37260", "54901", "43222", "41716", "8996", "40542", "48478", "17629", "18217", "1742", "36702", "17178", "59391", "48092", "7251", "6609", "52811", "39877", "34300", "37232", "52148", "44906", "16764", "48157", "21619", "10817", "16615", "10287", "57548", "45666", "32331", "5223", "50281", "4642", "8024", "59090", "57646", "36731", "17269", "2145", "28349", "34231", "17757", "38530", "27924", "47623", "43771", "21308", "45264", "41239", "23506", "27870", "49422", "2398", "3373", "52035", "43742", "55253", "36", "57188", "408", "6082", "49136", "27901", "40912", "590", "16732", "17967", "13290", "41887", "54835", "32465", "24242", "59039", "59466", "10201", "2576", "57898", "14751", "9857", "55182", "41072", "33801", "36859", "32579", "6897", "46445", "58204", "49857", "52786", "1016", "52566", "37575", "14515", "31633", "29711", "5817", "53556", "35945", "27236", "21003", "24082", "41698", "37711", "3901", "43019", "32486", "50046", "27119", "43003", "59345", "30366", "53880", "53846", "35283", "19339", "23024", "52373", "34612", "30721", "49945", "46416", "17041", "21181", "17765", "39008", "21888", "57783", "58755", "11375", "27767", "25459", "6759", "33368", "38775", "58919", "51885", "5705", "55667", "2019", "42249", "3101", "3597", "13234", "11642", "46593", "20200", "32220", "52793", "42634", "19230", "29834", "18822", "35110", "51255", "50110", "46600", "55580", "2456", "5668", "59823", "9831", "12260", "30190", "26004", "46730", "20143", "9880", "41868", "28084", "39857", "3348", "14630", "26689", "36078", "11195", "34150", "5639", "50537", "38962", "29133", "4772", "28952", "34199", "26909", "24444", "16502", "14635", "20521", "53703", "41457", "52100", "10157", "12049", "53777", "45860", "47474", "5959", "55346", "4372", "40937", "35115", "46462", "3434", "58295", "13559", "9153", "24238", "22895", "39839", "37304", "40", "18353", "6061", "39809", "37339", "3588", "21959", "10955", "20404", "40018", "48455", "27209", "13539", "47693", "11870", "7233", "17424", "41310", "59308", "56442", "41208", "4332", "26748", "7594", "15817", "11927", "2591", "41145", "36022", "20161", "44095", "4088", "57925", "54135", "59958", "49451", "31311", "13125", "58441", "35544", "53282", "57450", "17561", "30154", "5227", "20112", "37640", "55943", "50989", "32045", "28212", "36056", "45380", "9297", "10617", "44834", "23366", "54659", "36028", "53577", "58400", "15667", "10502", "51783", "40336", "32602", "15865", "26583", "28416", "35710", "9093", "5626", "57769", "34536", "49327", "54383", "56069", "21160", "47103", "20292", "44106", "46615", "41495", "58123", "45653", "30728", "26298", "50669", "56098", "15278", "6020", "34297", "45083", "50848", "48598", "5931", "50443", "49175", "44084", "38750", "28510", "7868", "26101", "12411", "26592", "18341", "1216", "4653", "55178", "1261", "43024", "39802", "38042", "45520", "20696", "57328", "4698", "26863", "16800", "15592", "37587", "33276", "59708", "5029", "50017", "43772", "765", "4309", "37034", "12806", "52336", "22371", "38093", "47318", "45113", "14323", "56287", "49524", "46951", "39912", "22963", "11009", "37545", "33548", "54366", "45595", "50316", "54645", "58224", "46020", "50898", "31859", "30418", "35439", "34458", "41486", "17268", "40181", "24368", "34179", "23378", "2215", "46281", "54818", "42254", "27349", "35592", "1", "131", "43544", "24420", "47305", "39452", "10830", "59421", "20635", "50352", "30890", "54051", "23765", "59443", "14524", "27648", "40811", "5580", "50429", "40838", "7960", "35891", "31471", "58064", "13184", "15185", "8739", "21000", "31057", "49403", "59349", "57149", "5843", "39137", "59791", "21002", "30135", "41626", "2987", "35227", "37261", "48633", "1600", "30184", "23130", "3640", "53780", "49630", "58903", "9094", "24230", "21685", "25893", "10665", "10279", "53724", "56226", "25344", "59428", "31482", "2079", "16604", "51203", "34461", "8264", "33679", "13238", "45164", "58338", "41268", "34975", "46477", "54803", "52258", "39704", "50128", "16334", "9204", "51855", "37883", "19638", "2389", "10963", "57534", "33492", "21690", "13701", "45329", "19159", "16992", "31665", "14", "47515", "55417", "26289", "13907", "2137", "51332", "23602", "28575", "17252", "5808", "32156", "1763", "51105", "45327", "35311", "5994", "3886", "27274", "45448", "57246", "15094", "57134", "30230", "48597", "24658", "4099", "26838", "41865", "49619", "33187", "5736", "30348", "43858", "23605", "15819", "26118", "39720", "17379", "26892", "7701", "18069", "47715", "2715", "49916", "6954", "27718", "7234", "11179", "29127", "40821", "9848", "38876", "41924", "15139", "37160", "37820", "16340", "10269", "2236", "29922", "47875", "43065", "7407", "57878", "17611", "55124", "23848", "51263", "40049", "23743", "17369", "17198", "3629", "35864", "51199", "31341", "40839", "2362", "24070", "16135", "53240", "15674", "44408", "59827", "40680", "33423", "43433", "10224", "1920", "36026", "49299", "8607", "24435", "36212", "18751", "29065", "51028", "15383", "58488", "3363", "27535", "11203", "1594", "53595", "7190", "15398", "25532", "27831", "35485", "38513", "57887", "674", "27183", "29854", "59442", "18196", "12210", "11493", "59714", "43566", "27623", "48293", "32043", "28019", "2146", "10474", "55221", "46156", "36251", "57120", "12319", "15122", "41773", "53210", "37851", "16037", "4813", "28040", "42124", "20937", "4131", "34075", "56301", "27632", "30839", "15365", "56721", "52846", "88", "28398", "2235", "25388", "52063", "55809", "16566", "48845", "25550", "23846", "16948", "53159", "30160", "30202", "54205", "23485", "24336", "48149", "42305", "32834", "56403", "42391", "51470", "43005", "51422", "47801", "27982", "12054", "2096", "55064", "47816", "30297", "9903", "27210", "46127", "20687", "51446", "58649", "43668", "3273", "10869", "48807", "5837", "21060", "45306", "35454", "30183", "10131", "55539", "13508", "43452", "14863", "49668", "23302", "27926", "34608", "52145", "8324", "39938", "39701", "11863", "1755", "42909", "30213", "39467", "19889", "8488", "24174", "35629", "59762", "2753", "48610", "16047", "52132", "38289", "5864", "19282", "40832", "20518", "19041", "30155", "13587", "46292", "47023", "52538", "26755", "50745", "45008", "7204", "27644", "16867", "8509", "23047", "31595", "12726", "22382", "52224", "36136", "23449", "36434", "14780", "47102", "57320", "33284", "1054", "48268", "59673", "14603", "55833", "30205", "33248", "2435", "22231", "45114", "51811", "19184", "3993", "53457", "30018", "59259", "44382", "38947", "19930", "26977", "10507", "40364", "52542", "53948", "41746", "45716", "43125", "7022", "40406", "14374", "15555", "37326", "55169", "7338", "59629", "49608", "5002", "14180", "39883", "23528", "42278", "29033", "47554", "16535", "10316", "21333", "36325", "35428", "43853", "55215", "20815", "20453", "48138", "47518", "54121", "48537", "53164", "29492", "6869", "15831", "49625", "16080", "24716", "35960", "26328", "15382", "22920", "29386", "44908", "37126", "30269", "38667", "13063", "32812", "40514", "36055", "44026", "40849", "17715", "32272", "48549", "19403", "55720", "4014", "34078", "34982", "31265", "17203", "59971", "1453", "55590", "23052", "44894", "12856", "27422", "24623", "32970", "52908", "19384", "43978", "18918", "33550", "59743", "31316", "44110", "4315", "53449", "47444", "56642", "3162", "33097", "46259", "21248", "34268", "57881", "22092", "47597", "52306", "1629", "41283", "36381", "9163", "11987", "37562", "55314", "17167", "22232", "50732", "54387", "59105", "15384", "27427", "37873", "30217", "50400", "29860", "5844", "10521", "36372", "45093", "53345", "464", "2246", "4133", "28290", "16876", "27837", "9962", "48701", "889", "33400", "48212", "23907", "32418", "38742", "3483", "46870", "7850", "18578", "30242", "41857", "8405", "14327", "44631", "46683", "15898", "32196", "13164", "21759", "36914", "19866", "58704", "22021", "9639", "6234", "3951", "26501", "54749", "20222", "18336", "37908", "9322", "45119", "57301", "26928", "16300", "14550", "46650", "32058", "8469", "36309", "33869", "17151", "44271", "29792", "34324", "29471", "52919", "21324", "22004", "23231", "4273", "45305", "29210", "29962", "2805", "36886", "24005", "48613", "49819", "59782", "55893", "45300", "52486", "15011", "55994", "5214", "414", "5436", "17422", "52718", "14214", "6467", "42239", "4687", "34505", "33704", "3606", "50155", "38851", "37144", "48502", "46264", "22267", "47794", "59541", "392", "16420", "16793", "22473", "32285", "1180", "759", "39523", "22711", "47594", "52420", "23265", "59826", "21016", "40738", "16515", "9754", "51275", "41076", "59711", "17892", "53185", "45796", "13170", "25844", "44969", "18849", "2990", "23772", "31035", "13435", "18327", "2200", "48158", "28188", "49912", "40600", "30334", "16844", "8565", "58351", "3589", "5241", "12325", "18571", "31314", "56792", "22423", "40012", "24953", "21382", "36976", "46809", "44018", "31702", "39830", "55050", "15118", "45615", "39366", "44066", "44435", "3754", "38262", "36312", "20452", "40830", "48102", "42045", "11926", "56737", "56475", "47321", "3437", "42258", "47222", "50590", "49471", "13009", "40900", "39859", "35256", "44555", "18332", "43774", "1248", "24580", "23260", "54858", "38743", "12110", "21931", "52609", "18683", "9531", "14502", "28322", "38020", "1572", "59369", "2514", "17047", "29432", "49716", "14800", "57359", "18581", "15872", "47159", "13622", "34285", "21873", "2320", "2048", "44798", "6660", "21932", "44775", "23165", "32488", "2725", "22117", "28218", "53255", "27511", "15457", "7306", "8104", "46641", "48440", "33760", "20051", "2011", "6733", "23419", "29035", "34120", "36305", "56851", "55735", "16971", "40290", "44912", "19343", "5528", "9284", "19179", "5716", "19895", "18318", "38807", "7917", "14642", "2151", "50743", "27850", "7904", "51049", "51067", "37462", "29888", "55697", "34014", "8278", "38747", "34512", "5742", "42958", "33233", "42220", "12340", "27263", "10633", "43035", "37641", "5235", "28274", "2564", "54726", "15770", "45148", "49104", "39307", "1282", "42774", "46489", "43319", "53408", "45228", "22187", "5672", "42267", "53978", "14295", "58149", "43031", "52631", "50303", "33183", "44389", "24069", "12449", "16054", "11449", "45614", "47790", "32183", "42286", "21089", "2515", "33507", "7330", "41756", "16086", "24064", "14825", "55214", "54961", "14145", "7508", "55379", "24088", "54126", "24366", "55507", "30059", "22326", "20464", "39213", "38973", "10527", "23038", "1651", "25257", "35741", "42917", "34118", "24741", "52212", "55003", "28950", "3978", "54607", "11215", "54842", "22612", "55709", "50995", "33982", "25968", "54612", "15313", "13411", "16158", "28016", "1668", "1713", "38132", "11917", "22042", "22511", "34042", "27231", "25866", "54075", "57055", "26181", "8459", "24292", "17886", "21457", "33563", "20290", "14450", "22538", "48891", "39091", "50905", "35988", "13957", "4720", "28624", "8548", "44511", "46892", "55008", "7741", "22736", "56263", "1303", "56122", "7156", "20209", "51050", "56116", "34178", "41511", "47915", "23862", "37148", "19993", "28234", "34193", "30222", "49480", "6230", "37718", "16744", "24073", "48096", "23820", "655", "24220", "49182", "14338", "6431", "4863", "34398", "15700", "13794", "9948", "36586", "851", "59497", "33927", "37319", "17498", "25072", "39217", "35348", "25697", "53749", "22949", "612", "14312", "47567", "15933", "21105", "180", "7067", "22244", "25502", "44752", "55606", "58531", "17772", "41527", "9207", "1890", "10266", "23607", "30251", "52180", "24956", "40590", "51302", "7148", "49850", "47802", "15503", "41818", "29191", "50872", "11770", "44380", "34408", "3602", "19656", "41236", "51790", "56043", "6689", "396", "44850", "34591", "32542", "59322", "20780", "30171", "1973", "7817", "21965", "16879", "43338", "25161", "14075", "24739", "27125", "58645", "14958", "7624", "56481", "6474", "31537", "13353", "48209", "51369", "39680", "41564", "44358", "21186", "25568", "12072", "49983", "50287", "18460", "22827", "47215", "27281", "19715", "33768", "43499", "23358", "39165", "38186", "5689", "46966", "41785", "3198", "21684", "38507", "19642", "52263", "49734", "24402", "93", "44220", "44795", "6265", "53285", "37677", "29639", "47338", "57519", "21660", "16752", "14837", "24982", "31794", "53979", "4016", "16000", "38931", "46347", "38538", "21784", "6925", "23309", "5670", "38731", "16838", "8622", "43392", "22330", "18291", "39689", "12767", "27613", "58955", "24166", "25018", "15920", "58715", "5412", "29288", "54896", "51986", "29181", "28007", "47555", "14913", "1669", "17392", "41877", "23427", "10910", "9839", "1953", "27948", "50422", "26910", "41874", "483", "39372", "6976", "37384", "52413", "40058", "14383", "23166", "4870", "44986", "49190", "46471", "53860", "47195", "2758", "16492", "33842", "26047", "14116", "35080", "16511", "39099", "17336", "39018", "12197", "21431", "9829", "17619", "1467", "45699", "45543", "18403", "14887", "46453", "48823", "55391", "15839", "8183", "14073", "29146", "16358", "8168", "7821", "13764", "33975", "4577", "28285", "39518", "59702", "21744", "22574", "40441", "55556", "58399", "57319", "39121", "58373", "31271", "25765", "29268", "55266", "45322", "43039", "19588", "38815", "39093", "31195", "54923", "53610", "5478", "39966", "35480", "18651", "28005", "46945", "23064", "14597", "53496", "34636", "37171", "45102", "31797", "20633", "1607", "33332", "57177", "39412", "30879", "8502", "7977", "14851", "50755", "16032", "50406", "41368", "33682", "13641", "15889", "9368", "1721", "35984", "46367", "8931", "12195", "31707", "18579", "47239", "16977", "1044", "101", "17778", "29096", "16788", "49306", "35092", "45646", "4395", "1772", "13243", "29323", "7464", "4439", "45473", "19561", "45156", "59707", "38185", "18517", "50197", "8272", "55495", "6232", "41869", "5418", "24861", "39276", "6329", "38297", "36529", "920", "12865", "36501", "33994", "33922", "54386", "1864", "14203", "6106", "57845", "55235", "55070", "8470", "4530", "12127", "3359", "37355", "49251", "27792", "5104", "6377", "5038", "22778", "23012", "38823", "2039", "53649", "45697", "18343", "17428", "35947", "37165", "14454", "46543", "2143", "42556", "26921", "17232", "30977", "23757", "17517", "48073", "59747", "16019", "27725", "26507", "43600", "3576", "50926", "14565", "54361", "30359", "3838", "46045", "2454", "25567", "31513", "57510", "55060", "25736", "56214", "43415", "4339", "49952", "33054", "23744", "3776", "19096", "25463", "5095", "57628", "51064", "43736", "44918", "11073", "28807", "58511", "1944", "25941", "39951", "16922", "15144", "22690", "38133", "54834", "29376", "25125", "28358", "48296", "49461", "33529", "24516", "57419", "39205", "28299", "14371", "15958", "8237", "49903", "40480", "4704", "9018", "446", "59007", "43012", "8512", "56324", "22524", "36719", "9919", "20945", "11331", "30349", "28528", "32551", "58431", "10751", "50832", "29794", "4648", "3224", "56762", "7219", "2517", "59273", "28500", "26820", "52463", "7115", "13779", "37062", "23452", "45314", "24709", "59100", "45313", "44632", "47306", "10338", "56372", "4120", "37018", "42480", "14170", "46110", "43241", "24825", "21539", "57333", "541", "11062", "30356", "21495", "24016", "10649", "6199", "10781", "41848", "29582", "44149", "3405", "35727", "56240", "23408", "31142", "44172", "17205", "51626", "52949", "52835", "58688", "27402", "9440", "56217", "39133", "35619", "1684", "38102", "57123", "49219", "13769", "18509", "17161", "39485", "2535", "8976", "51892", "46093", "37439", "50160", "50258", "14902", "54269", "26985", "11248", "19352", "55608", "16219", "31842", "40074", "36483", "1559", "11627", "51975", "31013", "55484", "27687", "32587", "4330", "48762", "35972", "51679", "39202", "44274", "18470", "795", "4066", "51197", "34190", "56038", "56907", "46431", "44288", "29559", "28988", "17263", "9707", "28171", "17798", "31123", "20515", "7656", "57201", "6273", "20050", "23833", "56189", "20463", "35852", "37645", "24611", "59210", "52138", "5206", "42694", "32374", "44038", "7876", "56013", "25627", "42269", "12575", "57430", "7147", "56084", "46278", "44928", "57989", "44743", "26920", "36391", "3920", "16759", "36347", "49817", "23382", "46492", "56389", "3671", "34465", "17929", "24985", "20250", "49637", "25056", "34772", "42620", "31573", "13203", "31377", "35678", "19790", "28831", "24055", "12013", "12576", "55301", "10410", "44153", "45282", "28631", "46575", "52598", "4818", "9044", "55599", "9242", "2835", "12721", "19019", "4020", "44914", "57376", "40041", "43597", "49698", "21007", "5924", "33460", "38769", "10794", "27597", "36733", "36596", "39820", "44633", "46764", "2470", "3397", "38683", "15427", "5168", "35504", "15772", "8348", "14652", "29722", "47041", "24048", "5688", "10288", "6180", "32948", "36622", "7599", "18417", "6131", "39504", "26310", "14647", "9990", "29113", "19788", "28347", "31643", "1922", "13368", "38666", "46486", "8890", "42952", "29769", "15394", "11021", "17780", "26397", "13931", "47882", "27973", "29534", "21261", "38291", "6667", "10815", "46700", "41425", "34962", "45987", "59737", "31602", "9039", "35225", "6257", "17489", "6943", "36606", "58618", "18667", "32441", "19992", "53570", "25586", "52430", "22233", "13046", "2479", "52529", "27206", "36152", "32984", "42710", "3685", "19801", "48218", "39971", "23536", "51884", "52938", "26923", "2275", "35362", "41438", "40557", "6793", "6049", "7001", "1058", "38559", "27905", "57889", "981", "32216", "7866", "36544", "52228", "32417", "17055", "22807", "51033", "57836", "37532", "13723", "9565", "12380", "1279", "37464", "11333", "40812", "6469", "27780", "53949", "52791", "3706", "23440", "35814", "35569", "33996", "12634", "47682", "3072", "59579", "53628", "56877", "31679", "41750", "11527", "38860", "23242", "7086", "2721", "56485", "26791", "33259", "9527", "18801", "32060", "53023", "36572", "25982", "441", "32864", "8475", "15000", "43199", "12506", "43609", "27300", "12213", "6447", "38917", "54722", "2307", "57600", "56012", "57828", "6141", "38395", "56925", "45806", "27015", "14883", "5840", "17299", "29186", "45590", "56264", "20510", "10941", "53051", "52183", "49435", "27418", "12073", "49448", "38600", "997", "34251", "2949", "57734", "910", "52689", "29458", "56501", "20952", "51093", "13983", "56830", "35455", "7726", "54381", "22483", "45253", "12961", "43247", "52775", "32048", "43009", "5947", "53182", "34596", "52988", "50775", "15910", "19892", "6394", "52316", "19590", "49785", "59231", "28863", "11048", "30397", "4501", "39456", "49241", "39702", "42344", "26320", "30340", "28675", "58949", "3267", "51638", "42742", "10540", "43538", "59643", "21627", "15078", "6178", "1421", "33637", "58515", "54721", "26066", "42263", "47262", "8585", "12386", "7384", "33885", "17117", "26616", "33402", "687", "27118", "7093", "53", "7129", "41638", "40641", "45226", "38553", "52986", "54190", "15857", "10080", "46085", "17733", "26222", "30816", "52294", "41999", "23470", "49788", "24976", "28428", "42171", "59166", "26811", "49334", "29826", "32657", "44228", "42410", "54411", "19406", "28705", "15088", "36104", "46052", "6093", "15217", "10726", "14105", "42528", "4095", "33256", "13166", "11476", "51677", "37399", "30255", "21814", "46080", "41026", "43723", "8313", "45062", "48567", "45918", "54144", "38755", "54441", "2677", "41763", "19474", "35301", "14110", "47674", "26339", "18639", "50701", "12870", "31938", "32966", "37667", "19763", "50111", "740", "5650", "4420", "38675", "15874", "44156", "24572", "47847", "4074", "56623", "25004", "7138", "45214", "28730", "1086", "33542", "1839", "42858", "57075", "34773", "49520", "6366", "28031", "32783", "8806", "26065", "5531", "6010", "21608", "21969", "38905", "4759", "9850", "48991", "57550", "34027", "38862", "12287", "51447", "25731", "39789", "12401", "7799", "59130", "48318", "59086", "53936", "42837", "28630", "29530", "28623", "9443", "54968", "57170", "14272", "40484", "41963", "20811", "4433", "22279", "10807", "23983", "32738", "28891", "45418", "57116", "58300", "46775", "38101", "11019", "34013", "17296", "29757", "51660", "53534", "15899", "9596", "37721", "38192", "11839", "38351", "30211", "26451", "34069", "19744", "31069", "6303", "48200", "12286", "7723", "55426", "7885", "58249", "45914", "27991", "41592", "55497", "46361", "57523", "50917", "29872", "26061", "43615", "27698", "52199", "53148", "41044", "6765", "14245", "7424", "37799", "1598", "33467", "56855", "11446", "6198", "50528", "1693", "57948", "5380", "8486", "41396", "41155", "23432", "7853", "23591", "8239", "30817", "10417", "15214", "30144", "35174", "33777", "45843", "43235", "7327", "57151", "13665", "13305", "22637", "3017", "49940", "11844", "10298", "10354", "6259", "44844", "47249", "32484", "12891", "18527", "10491", "8571", "31350", "36425", "59123", "19818", "53117", "32661", "17029", "52142", "25904", "15055", "34578", "18385", "35942", "55974", "16906", "54676", "24582", "5882", "47493", "26499", "50635", "19640", "8960", "42731", "51820", "58738", "47628", "45880", "30361", "10293", "6713", "59923", "18958", "4469", "22973", "15930", "22897", "59595", "52907", "33376", "46563", "16146", "34174", "38737", "32530", "9483", "42846", "59068", "50417", "44174", "6393", "16106", "12363", "53141", "42154", "34770", "29748", "4084", "33877", "42339", "2981", "11335", "25461", "8423", "19940", "6611", "11687", "42011", "13977", "25116", "6616", "39396", "2565", "21480", "10204", "5653", "58106", "3496", "6907", "41160", "38287", "33164", "9328", "34214", "9298", "39638", "27081", "18315", "58961", "34434", "27247", "18884", "831", "15646", "18091", "35884", "57616", "42330", "56299", "43252", "47320", "26560", "19026", "39595", "38555", "8929", "28364", "14148", "7470", "59995", "10509", "22332", "13803", "16173", "32711", "11221", "37064", "23653", "32553", "54099", "19692", "45067", "15993", "1703", "43851", "42058", "44264", "57872", "19734", "47251", "22292", "41206", "33832", "38466", "49007", "41453", "31805", "57904", "14090", "25988", "40003", "2139", "29078", "32606", "27739", "33158", "39142", "31691", "56056", "29733", "14690", "51669", "42126", "34624", "15775", "2697", "55023", "15033", "17945", "28961", "12928", "46676", "9479", "12889", "42902", "1135", "33576", "32960", "29439", "29472", "44108", "5189", "51174", "5553", "18501", "19554", "53125", "38650", "28073", "50566", "46942", "15516", "45347", "7487", "37145", "3519", "48406", "22965", "30552", "17532", "31340", "12750", "57268", "24323", "8504", "3930", "44457", "13900", "51191", "39097", "40349", "1960", "32891", "15335", "56884", "34716", "59228", "4377", "336", "3384", "21925", "9271", "1658", "15909", "54733", "41748", "25319", "48461", "58277", "46497", "43457", "25283", "19288", "54561", "52643", "39593", "20528", "21729", "22297", "46244", "13857", "33947", "21606", "13727", "26674", "22788", "51040", "31168", "33019", "33734", "54538", "20623", "26028", "5796", "39869", "22302", "47879", "13424", "27801", "20955", "5004", "48465", "50119", "31065", "50573", "6378", "9299", "46863", "44939", "46331", "21736", "37748", "16590", "38240", "9811", "23054", "20106", "18545", "44263", "32996", "49234", "43733", "44707", "9841", "14117", "32512", "23152", "42027", "26385", "32", "26505", "18411", "30015", "5341", "44268", "53887", "35808", "57849", "46217", "51138", "18893", "48848", "52127", "31440", "43345", "48888", "7695", "12086", "53351", "30024", "1262", "14805", "36691", "9861", "48270", "56329", "56528", "29299", "54250", "9583", "4617", "5618", "52752", "58706", "40563", "27186", "3670", "6668", "45840", "17678", "20445", "16813", "38523", "36730", "43902", "52320", "46827", "7768", "12757", "9022", "52879", "36615", "38328", "45539", "26170", "17550", "9842", "55174", "44400", "13851", "42476", "14209", "44667", "40265", "10499", "17785", "26475", "4486", "18454", "48413", "26835", "4093", "27297", "49266", "33385", "38017", "25469", "7996", "25293", "26023", "42696", "34359", "25611", "30998", "18726", "11689", "49486", "54337", "21905", "21350", "52028", "40297", "50386", "8560", "6978", "17549", "18394", "12056", "35483", "16654", "7734", "31641", "57806", "31593", "47487", "58374", "20287", "31060", "49604", "9287", "40628", "4076", "56083", "223", "57824", "6981", "28245", "27196", "11100", "16797", "22006", "50554", "25781", "45073", "46229", "27336", "49341", "37967", "26623", "15024", "31391", "56568", "44750", "49224", "54360", "49257", "16872", "1861", "48488", "56351", "48133", "17936", "26485", "37420", "47048", "14629", "53998", "2068", "14281", "16576", "44259", "37094", "27586", "41360", "55284", "21549", "36382", "54898", "34197", "27253", "23749", "8906", "33234", "8019", "56984", "33738", "1569", "39751", "9971", "14337", "55145", "17784", "42667", "16781", "56123", "19721", "7402", "40006", "29817", "18424", "24978", "52134", "24591", "22817", "25590", "20163", "15379", "51396", "28324", "25038", "53098", "20911", "49046", "26582", "39662", "16560", "9530", "30346", "18512", "8310", "56157", "59933", "33136", "52914", "33138", "54346", "15386", "57444", "14995", "6304", "33516", "10544", "45424", "27026", "57776", "24416", "14592", "56924", "16134", "27877", "48643", "45705", "20156", "12897", "49744", "42137", "52286", "1024", "26833", "1225", "32877", "29045", "27741", "15623", "28167", "30426", "25497", "42014", "528", "50991", "55612", "40553", "30315", "51590", "52853", "7153", "7364", "34537", "47979", "19255", "30570", "874", "33478", "59051", "25156", "10020", "15525", "3935", "57335", "13933", "28600", "24095", "22662", "17823", "29890", "13136", "38338", "37853", "12886", "39360", "5267", "326", "18703", "39487", "20149", "11127", "32181", "12508", "46360", "27891", "52982", "53654", "5177", "5855", "48009", "7152", "37389", "56360", "31164", "58498", "32690", "1190", "17562", "48793", "9541", "55371", "38021", "33901", "6278", "55468", "14263", "9600", "35442", "46198", "22801", "47748", "5234", "14384", "37069", "32228", "26897", "30814", "30830", "48276", "5838", "19270", "5022", "29538", "34908", "38842", "18576", "48861", "34455", "34934", "4916", "2154", "7473", "10993", "30101", "54502", "54324", "36549", "1667", "50629", "22512", "54891", "12485", "24144", "53412", "20034", "32131", "50084", "38449", "13949", "33870", "6203", "8228", "32794", "5535", "2605", "23025", "50886", "45832", "58873", "13953", "15791", "54720", "11984", "24773", "39816", "29300", "48995", "40474", "53479", "47193", "156", "5024", "40792", "19130", "45466", "10081", "20905", "33930", "4600", "52660", "40434", "46666", "26213", "27331", "33971", "7409", "55683", "57603", "57934", "43998", "40293", "59749", "6225", "9969", "24384", "36323", "36839", "58350", "35277", "42881", "53603", "42375", "17710", "17529", "35335", "23460", "48983", "22024", "57286", "16760", "16736", "27400", "29636", "58694", "13731", "40285", "37981", "14009", "14936", "53844", "4564", "3129", "12145", "39243", "39125", "38586", "52408", "50266", "41300", "8116", "27466", "2197", "58816", "32561", "59918", "16756", "56139", "2132", "36016", "39573", "31631", "58168", "27859", "31569", "25617", "27134", "25948", "49129", "9862", "14845", "7079", "15495", "13265", "37243", "21646", "57199", "51797", "51492", "2639", "5376", "18786", "3711", "21760", "32343", "9034", "29066", "5562", "31578", "16831", "3976", "13396", "58254", "56675", "37945", "38987", "32601", "52488", "49425", "35344", "29037", "56634", "18673", "55119", "52648", "44357", "8998", "4652", "16042", "45123", "11194", "32829", "8510", "1161", "55393", "18245", "29995", "47437", "58997", "6554", "28987", "58938", "28054", "44979", "13846", "32785", "43453", "1777", "39501", "46893", "5856", "36929", "30595", "59058", "16257", "57103", "26302", "16360", "23558", "43221", "53399", "54877", "53486", "44791", "49462", "32408", "58414", "48067", "57539", "19880", "28679", "48491", "8584", "2769", "47633", "58303", "21551", "39547", "44723", "26789", "29413", "44837", "11080", "10478", "43921", "31936", "49883", "23928", "50353", "24694", "17779", "1165", "19527", "27960", "7558", "30636", "17616", "56469", "46552", "18925", "11101", "12406", "26282", "58713", "38504", "6039", "10342", "10818", "13751", "34361", "57018", "34876", "34342", "54178", "57297", "30373", "39781", "48724", "13783", "3081", "58405", "26168", "16841", "30141", "59373", "45533", "10214", "27601", "9078", "3637", "9967", "54405", "52230", "53302", "55920", "33941", "8214", "33623", "46452", "42169", "4334", "29449", "52433", "17327", "18083", "6186", "49393", "47797", "9512", "20838", "21167", "43278", "49284", "33321", "16348", "33808", "5288", "33633", "59312", "37122", "53753", "43605", "39236", "12717", "43159", "23137", "19687", "24540", "40240", "19528", "15913", "20458", "47938", "38278", "26297", "34969", "1538", "25913", "6066", "40877", "8902", "16264", "25331", "15790", "42660", "31772", "11735", "57121", "48630", "16886", "42478", "17253", "24304", "47316", "10891", "31445", "24784", "50171", "51154", "54592", "10455", "44300", "31733", "7491", "1775", "10945", "57101", "5423", "53030", "40535", "35365", "40121", "49048", "23363", "1697", "37208", "43621", "17040", "26642", "12747", "8780", "47925", "30987", "57051", "10485", "42648", "56566", "59275", "19571", "38263", "32516", "11143", "9620", "54869", "59924", "31984", "59510", "20208", "23623", "30507", "4712", "36506", "14508", "28430", "18690", "58865", "8017", "58966", "45907", "53161", "4249", "31549", "5754", "45526", "36810", "22787", "11447", "56100", "43636", "2795", "53490", "3082", "24656", "38608", "58571", "59426", "34653", "57378", "41226", "33558", "38779", "33886", "23413", "1056", "11916", "14446", "7650", "931", "28051", "40052", "59500", "39894", "28253", "27164", "20971", "17699", "36338", "31845", "56082", "32234", "20994", "32633", "43385", "2230", "28934", "33465", "33893", "40104", "35527", "7616", "25709", "10488", "4179", "27053", "58453", "37494", "7954", "52050", "14722", "10145", "436", "37167", "17460", "28001", "17368", "58458", "33934", "12878", "40357", "50310", "41197", "43021", "3560", "12046", "56577", "17641", "35118", "44293", "12968", "56387", "4030", "58438", "57433", "13683", "9904", "39020", "38921", "25210", "4097", "8680", "5017", "3347", "41308", "41501", "55541", "5385", "57643", "43026", "17799", "34599", "18444", "30920", "28239", "33440", "13578", "49320", "56112", "54044", "23539", "12981", "46272", "25384", "44904", "43914", "29388", "17831", "23306", "23438", "11966", "5534", "44040", "6801", "16812", "7181", "35829", "55931", "33190", "56175", "56190", "14784", "8892", "5281", "10277", "51845", "7881", "985", "19833", "3463", "19954", "12345", "38712", "48271", "44499", "1827", "39712", "58398", "18577", "26502", "31215", "56630", "53081", "28328", "24247", "22369", "30419", "51752", "41760", "54812", "59269", "8012", "48873", "32092", "14480", "25168", "12232", "24539", "30671", "36173", "38094", "24003", "58773", "53371", "46253", "59494", "5798", "46231", "32190", "14943", "40562", "7998", "23840", "32708", "56271", "11053", "20181", "30100", "56550", "44897", "22319", "53508", "50521", "7804", "39711", "217", "7512", "19411", "47394", "5355", "41179", "50877", "52491", "3408", "38944", "59431", "20299", "57095", "58677", "29986", "7292", "30707", "18304", "54839", "51588", "51962", "22500", "48267", "12842", "47348", "27010", "28191", "32914", "14057", "39767", "53579", "3912", "30837", "30867", "49482", "40516", "20023", "20979", "34719", "59671", "38878", "59746", "25146", "4070", "19119", "14240", "28351", "35964", "45759", "28099", "28269", "12707", "18251", "26415", "20552", "30772", "3679", "20548", "25721", "17626", "6034", "43512", "55718", "30267", "43769", "31892", "18774", "2468", "41673", "17708", "25917", "133", "41685", "30820", "18600", "41652", "1705", "38604", "54844", "15835", "47593", "47168", "31243", "5224", "49605", "50275", "5684", "25794", "53322", "36280", "11963", "25670", "31731", "54843", "45225", "24835", "52140", "18640", "28289", "14438", "28336", "23283", "55480", "1131", "32012", "32112", "46648", "4453", "7693", "24715", "43292", "51673", "38474", "18299", "15754", "25292", "1786", "31148", "14684", "790", "33330", "15538", "33361", "6491", "45237", "2609", "2333", "58334", "9825", "11290", "14038", "2227", "13688", "29138", "32790", "42811", "49550", "43403", "59926", "55425", "35855", "55233", "19622", "9295", "37885", "14850", "51682", "12518", "21820", "33639", "16287", "50713", "53338", "7239", "47187", "44365", "22367", "20120", "51726", "13923", "41005", "4415", "11091", "10030", "22344", "26873", "50969", "7062", "3375", "31624", "31099", "23327", "54055", "11188", "29966", "25392", "16818", "7913", "23533", "8233", "34810", "46698", "25809", "8681", "24119", "32142", "53084", "4040", "16399", "49458", "50868", "54004", "4960", "37759", "56289", "12437", "4941", "50433", "34081", "56921", "50055", "25804", "13127", "40758", "13695", "32651", "50272", "51546", "45578", "33505", "43269", "18414", "44886", "47149", "20245", "48499", "10256", "56218", "4606", "1746", "5042", "57050", "3849", "47450", "8343", "47531", "48265", "26363", "46066", "1354", "52221", "23747", "20993", "9036", "7864", "58802", "41435", "41978", "16092", "17544", "13143", "53214", "10798", "42008", "52042", "28466", "20801", "47987", "35763", "51134", "48870", "40326", "39155", "23622", "22847", "40977", "42189", "41951", "56681", "16916", "30169", "46011", "37375", "24670", "14876", "36609", "41274", "17982", "26420", "44251", "37382", "57700", "53555", "4507", "10913", "50933", "9542", "35922", "46677", "46014", "42240", "57099", "29919", "56254", "2744", "17693", "14409", "45828", "16706", "2252", "20567", "41947", "8401", "3376", "7037", "23610", "10320", "42492", "19284", "30535", "22238", "57725", "35033", "13881", "57147", "49993", "7964", "10029", "9965", "56969", "33878", "53788", "53915", "7633", "37856", "12662", "46538", "22019", "11559", "27083", "11826", "56545", "50513", "36407", "20992", "27750", "23663", "4912", "4726", "7934", "14726", "3222", "19921", "32174", "52157", "34548", "25213", "39066", "51454", "48908", "48563", "40042", "59230", "27800", "12482", "15374", "38594", "14056", "45585", "24357", "8181", "43333", "37990", "50365", "15357", "1269", "37241", "10858", "5630", "21978", "19618", "56342", "7279", "28464", "5003", "22185", "10774", "53588", "54230", "30880", "6287", "45362", "41595", "42511", "43545", "33411", "48993", "13341", "39651", "22960", "43660", "4362", "35897", "59457", "665", "26526", "57178", "6405", "17237", "23450", "48014", "41766", "55925", "13902", "58729", "5579", "12026", "43151", "52186", "31413", "35824", "32034", "4676", "33290", "37301", "24789", "9230", "20064", "7600", "10216", "44733", "43984", "51930", "35949", "7573", "47637", "20763", "45572", "11370", "53624", "9589", "52204", "51189", "41496", "16880", "28422", "43473", "49008", "24008", "24185", "51516", "59655", "51077", "50592", "42293", "23399", "18328", "30542", "59034", "53261", "39707", "13122", "30815", "51564", "5949", "25000", "4261", "58503", "34517", "19071", "43846", "26938", "27122", "49673", "24270", "47745", "37755", "13322", "4890", "3012", "40718", "8080", "30448", "40650", "36639", "17378", "56587", "55152", "46769", "3335", "28717", "51674", "23211", "29539", "54139", "42597", "7916", "52447", "7862", "38071", "1002", "4782", "34232", "3572", "9246", "56876", "24320", "28059", "31298", "5772", "51466", "8496", "14335", "37021", "36068", "33989", "46646", "50102", "12142", "13310", "49156", "14614", "50014", "33572", "2727", "55361", "31487", "55645", "29953", "42080", "37662", "53414", "49944", "26590", "51267", "13678", "53172", "12109", "26495", "1470", "3796", "40872", "58075", "22202", "29271", "37831", "31748", "24225", "58273", "20374", "9259", "30781", "635", "475", "39587", "12989", "18532", "10814", "37609", "18994", "4763", "56361", "51158", "3034", "30601", "12535", "2820", "31768", "49017", "48973", "50322", "58972", "7541", "40581", "3590", "16833", "50904", "16215", "28744", "2551", "55980", "24769", "324", "23131", "1901", "37900", "8429", "25914", "34571", "52692", "47525", "35303", "51716", "36430", "17157", "41019", "41454", "49202", "38674", "55299", "18010", "33754", "46308", "6345", "10299", "27363", "25561", "1894", "56546", "11593", "24843", "6795", "15321", "44431", "47253", "8872", "16163", "31113", "51151", "1945", "27652", "30244", "6315", "46836", "59613", "17181", "4779", "42135", "17152", "31233", "55498", "35842", "24844", "43973", "17189", "57583", "2471", "28704", "19542", "4295", "3826", "9107", "30311", "58", "52815", "57604", "32685", "23920", "3667", "56963", "54467", "36201", "35664", "55289", "27070", "4311", "5218", "9345", "16596", "20121", "49293", "14591", "26245", "31943", "59003", "12585", "13679", "5411", "37802", "12102", "8092", "37176", "4670", "6606", "3616", "12279", "36092", "6146", "7831", "10688", "13377", "56868", "4867", "29196", "58836", "40841", "1274", "49752", "22012", "53918", "57667", "8177", "46062", "41709", "42883", "56256", "10548", "53271", "41075", "33775", "40538", "12820", "18857", "39398", "458", "30833", "17523", "32643", "134", "36956", "29951", "50420", "6031", "6029", "33399", "39642", "8184", "42104", "5515", "8589", "10652", "3757", "14452", "36412", "20924", "3301", "51482", "8499", "33099", "59830", "13309", "7346", "32063", "13141", "58270", "51387", "30605", "49562", "10102", "55784", "51207", "12474", "5517", "52724", "54622", "1631", "7059", "56706", "38245", "59397", "33744", "56645", "12956", "22719", "9231", "35944", "37854", "36053", "18311", "16096", "11938", "8898", "1344", "55532", "10880", "17504", "26701", "29571", "21902", "1649", "30098", "35877", "17391", "10523", "35069", "10755", "8803", "38752", "50133", "2751", "56840", "15531", "44987", "43592", "33289", "28918", "52146", "1781", "27399", "48796", "18574", "8097", "886", "47334", "39220", "56498", "53027", "58039", "36333", "39889", "9469", "5072", "4889", "27170", "6353", "40674", "12172", "40014", "33863", "10347", "32674", "40759", "48392", "24862", "54927", "39407", "10218", "36386", "4090", "23140", "53658", "54436", "25820", "33096", "18729", "16017", "35737", "45126", "5633", "14638", "54492", "56152", "33848", "4508", "52882", "34589", "37692", "5588", "44280", "56643", "3379", "17481", "14063", "49854", "40742", "37044", "33517", "32223", "13550", "12254", "23154", "556", "42668", "11040", "28761", "25673", "27516", "47235", "56359", "46181", "4285", "35550", "43494", "32595", "40511", "16417", "47520", "46003", "43444", "20829", "2736", "2922", "2635", "28866", "15654", "4523", "31628", "31337", "18159", "42697", "11816", "3441", "26980", "30729", "31647", "23307", "39842", "49840", "12440", "19051", "2930", "31291", "47445", "8895", "26533", "8695", "32178", "37893", "746", "17618", "10749", "49900", "24362", "34497", "20534", "50212", "31929", "46047", "13780", "11944", "34725", "58969", "17752", "3925", "43788", "38479", "57445", "56687", "3874", "7859", "48454", "9950", "46522", "49379", "29160", "49236", "59087", "25076", "4630", "10663", "15431", "13607", "54872", "127", "24725", "56327", "59318", "37002", "25435", "50749", "129", "49493", "12462", "13868", "45707", "45043", "20077", "36968", "20304", "57780", "50556", "16622", "37952", "31325", "51485", "8046", "24850", "32164", "21941", "39017", "30401", "24096", "36187", "5420", "28786", "12038", "40196", "57052", "12228", "14008", "54777", "55738", "29595", "1197", "57996", "406", "19879", "29561", "38653", "35738", "56703", "27699", "39530", "5310", "23788", "13633", "10469", "14589", "59414", "44717", "38562", "49804", "25465", "54963", "40929", "13954", "52124", "54836", "28666", "51932", "7905", "32078", "2903", "15124", "28300", "22551", "22548", "23578", "38295", "50990", "54855", "41203", "37123", "32000", "51421", "29202", "9751", "19067", "38627", "59960", "34807", "36063", "39263", "21096", "30861", "54678", "31190", "31296", "8049", "31409", "28997", "49987", "43229", "31696", "29363", "23146", "3207", "56538", "41557", "23454", "31846", "28179", "20520", "43809", "22767", "58857", "17934", "38574", "21990", "34400", "22770", "54261", "15546", "33006", "55010", "52177", "53150", "15965", "1113", "28308", "46234", "35827", "36594", "49801", "930", "54709", "52553", "32510", "11328", "29894", "46621", "6268", "11254", "1778", "1430", "3219", "10163", "37407", "45517", "41678", "17734", "26234", "50988", "13534", "34972", "15781", "9477", "59885", "54880", "43232", "36006", "18679", "46608", "34340", "52246", "15070", "51226", "26009", "11940", "44270", "38818", "55978", "15812", "33012", "22395", "56473", "15974", "18585", "8775", "6028", "17667", "10014", "3795", "9103", "25918", "19154", "15182", "6261", "19172", "41697", "15951", "55675", "9445", "6347", "9143", "23869", "25717", "34431", "1158", "56192", "52269", "54300", "40244", "3881", "6776", "55184", "40899", "2913", "48389", "10129", "34922", "50501", "7770", "33802", "19048", "41437", "43797", "30327", "3189", "47260", "39113", "30967", "43775", "29229", "42629", "40614", "57862", "10829", "37716", "1636", "42877", "1206", "59250", "54512", "47362", "24382", "40783", "4737", "10598", "35481", "12417", "51649", "31710", "53522", "182", "1800", "19714", "34513", "12549", "17066", "23019", "5649", "8465", "28325", "1739", "41421", "17435", "32032", "56015", "58888", "39773", "25943", "17274", "9988", "17677", "20289", "39153", "29772", "13371", "2768", "6181", "20985", "30692", "34635", "29650", "20491", "43042", "35426", "27766", "40981", "54793", "6763", "47584", "24681", "35199", "50941", "38872", "48438", "37650", "39460", "13451", "27607", "24191", "53352", "19093", "57221", "43251", "50870", "9474", "8922", "35655", "32170", "35791", "50899", "59361", "39855", "41410", "36664", "17225", "38983", "17771", "27052", "57316", "58032", "5999", "52403", "19386", "47543", "35642", "5702", "44019", "17417", "17396", "45837", "54293", "38286", "29453", "50326", "4013", "52794", "50997", "4984", "10394", "17925", "34953", "24281", "50657", "2888", "20681", "52796", "38330", "15950", "13929", "22632", "42347", "46261", "49410", "55179", "47658", "37803", "58184", "21745", "11739", "20729", "31807", "30081", "27714", "13627", "15853", "56554", "37075", "39878", "29755", "5995", "9859", "36293", "8620", "17650", "45817", "1401", "55676", "59449", "5428", "6087", "18933", "18061", "12446", "41986", "50312", "28550", "59939", "35508", "12698", "18858", "49315", "43812", "15453", "30660", "53257", "36502", "58378", "43334", "53839", "50613", "35089", "18957", "27337", "26377", "36446", "52160", "10399", "39972", "9579", "53003", "41295", "14045", "42654", "44060", "8583", "22377", "15084", "12240", "1048", "53205", "28087", "11676", "22781", "57469", "19977", "33372", "26793", "752", "24373", "48580", "44640", "5685", "8366", "54061", "5398", "37535", "58087", "52917", "146", "39669", "15328", "21144", "22248", "32666", "48616", "44780", "28029", "54394", "32447", "3914", "45903", "45676", "12735", "52685", "40532", "5627", "1257", "18819", "37095", "13315", "52409", "54857", "56054", "28691", "41380", "58962", "37247", "57396", "11807", "58615", "10079", "27139", "56532", "49229", "47455", "24361", "16432", "7959", "31400", "58860", "25370", "2341", "59253", "13448", "50267", "55395", "21816", "7727", "23178", "1025", "41708", "11037", "37421", "40213", "46825", "54754", "54488", "29227", "56290", "22208", "25545", "58928", "42764", "7796", "34692", "48650", "13636", "41876", "12011", "45741", "35287", "30486", "27985", "59459", "10460", "30635", "32839", "51857", "39344", "51620", "21943", "10257", "47549", "703", "51627", "53657", "9406", "8114", "52626", "6205", "51577", "1240", "33470", "49617", "56723", "48222", "29462", "55686", "49191", "7731", "31632", "48379", "21945", "59574", "14362", "19441", "50206", "24781", "25970", "30055", "19369", "43318", "4374", "9326", "16098", "58107", "55856", "52078", "21700", "39661", "47197", "11115", "51867", "54100", "49824", "28343", "19183", "18602", "48063", "57703", "34940", "38713", "854", "28874", "9097", "25662", "25891", "934", "57111", "41431", "42724", "9973", "26907", "1633", "41475", "39081", "55441", "38636", "27743", "39244", "23151", "58055", "52608", "51883", "10615", "9628", "48939", "2092", "9167", "1372", "47600", "45976", "45994", "50607", "35314", "27555", "21602", "57985", "23722", "6293", "1934", "52969", "15101", "10714", "11988", "9694", "10176", "12944", "59164", "16495", "31042", "54473", "7763", "10928", "46018", "22583", "12760", "57036", "16559", "25862", "38897", "14492", "9148", "47485", "5467", "45171", "30539", "34065", "23645", "43527", "19757", "43514", "25637", "3196", "30826", "17331", "3088", "12444", "55202", "42940", "9087", "25359", "52094", "12288", "8971", "22320", "49481", "56064", "13050", "49446", "57748", "15066", "19244", "5536", "46886", "7970", "42360", "19409", "47771", "5686", "41427", "21315", "58560", "53028", "26231", "50937", "57065", "13393", "42851", "10427", "39838", "24037", "38589", "2386", "37409", "11778", "31585", "26093", "15472", "32896", "11388", "55132", "32997", "16346", "7291", "15318", "20749", "11521", "18596", "8573", "12720", "32725", "13124", "28530", "39471", "53184", "26591", "46230", "39426", "32047", "37454", "35179", "7887", "39406", "40092", "35407", "34913", "29632", "33819", "2213", "53783", "53944", "27175", "3775", "18475", "36414", "38890", "39608", "55725", "26021", "12117", "43806", "29658", "19103", "42428", "34997", "50346", "47340", "51684", "59554", "14216", "44465", "15280", "5899", "13263", "15003", "34273", "56095", "51173", "9139", "54924", "57256", "7477", "137", "2261", "32888", "36748", "38756", "10304", "18136", "26941", "2318", "39930", "39410", "25852", "9360", "35554", "13097", "24525", "28115", "38430", "12300", "20590", "51057", "52967", "43811", "17948", "12143", "18895", "58591", "40549", "25691", "52392", "24898", "36450", "28181", "3412", "34665", "6786", "46474", "29943", "59297", "51680", "3893", "34991", "53057", "2016", "13716", "34196", "40980", "59263", "21400", "603", "54280", "29905", "43118", "50964", "59882", "26137", "18621", "56471", "19633", "11707", "50012", "18875", "33597", "46357", "8909", "41251", "27057", "40381", "48775", "52163", "59590", "40258", "46949", "49697", "18283", "40335", "28583", "19707", "16380", "5822", "30381", "2250", "2883", "6807", "27519", "45619", "33076", "4839", "52779", "39021", "12964", "28175", "17808", "199", "25294", "34321", "47082", "20663", "166", "53585", "32376", "28458", "892", "13973", "12609", "9096", "9591", "8247", "38371", "4280", "53841", "6211", "36583", "21091", "2932", "44778", "4348", "20355", "12434", "51292", "52563", "24796", "42091", "24498", "49565", "18031", "15073", "52918", "54849", "43535", "7010", "39785", "9461", "46512", "30780", "15704", "8941", "51760", "18971", "33436", "9656", "3247", "23350", "50571", "1857", "15140", "49006", "10321", "12450", "38292", "11404", "30696", "3181", "39549", "48836", "30741", "21095", "13852", "7193", "23731", "3641", "20928", "10539", "25501", "42341", "13336", "20931", "26039", "33403", "4868", "25648", "44879", "30089", "58845", "37832", "49508", "33594", "56478", "8521", "42048", "38402", "52857", "7700", "40206", "54741", "52666", "28053", "47702", "53332", "20847", "6438", "51479", "22060", "50986", "6829", "23050", "11813", "32981", "42718", "55810", "37709", "30436", "37198", "37661", "23431", "21258", "5356", "38605", "27150", "23961", "46712", "2361", "9121", "48015", "16999", "21053", "44346", "11931", "29868", "40994", "24063", "47194", "39149", "30435", "18164", "40328", "18716", "42544", "32425", "39199", "1699", "13484", "42630", "24745", "18859", "38388", "15523", "21470", "57597", "45735", "36658", "40002", "51020", "39527", "20326", "24824", "17385", "23058", "6551", "40086", "1661", "7363", "23208", "32093", "11865", "33195", "12920", "22133", "49502", "34955", "4490", "34149", "35202", "49409", "38341", "9828", "12139", "49723", "6777", "4761", "33364", "52061", "9229", "23886", "4071", "23344", "30186", "2159", "46732", "43564", "20478", "20300", "50827", "27142", "8391", "52439", "15521", "50676", "32492", "91", "8854", "51551", "48661", "42923", "48912", "31094", "52701", "16629", "52375", "18401", "10317", "51697", "37534", "45433", "12267", "56355", "25120", "34007", "54657", "52514", "53404", "28326", "58568", "34715", "15510", "31017", "1957", "30843", "55622", "13014", "49896", "37440", "31457", "24249", "35511", "45724", "54814", "55066", "7460", "16291", "16321", "8123", "50534", "3727", "46006", "20041", "38419", "28516", "24215", "58934", "14736", "52880", "51118", "13165", "9612", "13655", "20417", "33683", "35127", "14193", "13735", "39815", "20260", "3409", "45422", "15671", "42033", "10576", "47308", "16929", "36528", "19376", "26260", "3607", "33223", "27720", "43272", "59769", "4498", "31932", "59042", "5677", "13668", "43340", "7671", "6414", "58223", "46310", "16942", "21287", "7828", "21005", "33170", "3322", "41871", "18783", "52630", "12456", "35859", "49077", "21409", "36073", "40020", "13346", "55200", "42363", "49144", "35121", "43669", "26063", "19173", "46035", "38181", "38399", "9917", "57504", "35869", "13729", "4656", "53676", "37294", "36917", "54811", "30935", "31570", "25833", "53523", "56216", "41370", "35887", "4366", "22959", "33018", "37015", "41715", "1643", "8439", "425", "52826", "30128", "50638", "22311", "41325", "56865", "59908", "43543", "53194", "5537", "30685", "49424", "13512", "28964", "29018", "59726", "23167", "57601", "42303", "13232", "58534", "17914", "32100", "57712", "23860", "15498", "888", "7713", "2328", "15979", "43299", "15452", "59056", "37193", "12711", "10980", "45157", "9775", "22566", "35150", "31577", "53504", "23482", "14905", "6175", "28673", "11386", "5242", "24979", "56756", "58766", "39631", "2568", "18085", "42772", "20229", "19626", "19709", "13196", "26637", "8596", "44137", "43692", "12845", "46039", "45475", "47402", "32913", "30882", "28150", "44239", "46707", "23736", "7762", "25517", "52114", "47587", "45753", "48667", "26876", "22528", "17101", "47114", "1152", "57540", "20255", "9563", "46263", "38803", "27028", "32020", "44690", "44197", "34887", "51882", "21352", "13582", "34664", "21231", "17093", "8770", "14669", "3140", "17729", "39306", "10824", "23026", "53826", "39308", "8369", "53274", "2746", "30091", "25192", "52819", "18669", "46147", "7587", "58349", "19157", "48173", "23018", "13502", "8717", "33307", "42408", "39138", "28316", "24975", "25484", "23925", "3722", "21332", "10564", "43237", "37312", "57704", "11918", "2032", "20197", "40369", "10700", "48693", "20141", "48833", "56686", "44810", "16143", "43052", "56248", "10205", "49742", "43850", "55343", "42034", "55404", "27043", "10842", "27913", "51147", "14888", "12999", "35119", "55247", "28897", "30919", "44412", "4521", "26786", "30120", "37348", "47295", "38334", "37950", "20721", "56863", "21676", "44385", "39968", "9458", "45045", "17717", "2067", "48774", "36395", "18119", "3759", "12302", "7236", "53482", "24006", "51060", "53509", "42120", "47993", "30353", "58572", "49985", "36520", "41841", "53935", "12715", "59155", "29832", "31245", "51876", "56772", "16316", "53771", "28642", "37146", "29220", "23563", "55963", "22210", "2087", "57029", "59838", "25486", "53278", "1091", "36231", "4431", "30134", "58226", "21634", "47135", "28260", "26000", "26241", "7893", "34315", "50888", "44530", "45939", "20196", "35993", "50545", "57317", "57560", "23908", "38208", "10260", "28994", "58701", "52774", "44374", "51900", "38679", "49019", "11542", "49164", "16211", "32858", "11155", "41718", "35925", "17726", "59410", "23728", "13912", "39089", "51647", "28013", "43616", "9521", "1378", "16633", "27678", "33345", "17834", "8561", "3972", "138", "42515", "40845", "20199", "23592", "42059", "55443", "6642", "59282", "24447", "39316", "37060", "58071", "40132", "13075", "20481", "18873", "17874", "57104", "52926", "7480", "44964", "20159", "55354", "20707", "44921", "20291", "12985", "17319", "30432", "42839", "51623", "19732", "33587", "15330", "2763", "30818", "15201", "34024", "51037", "44112", "38977", "22099", "36301", "15540", "26613", "14432", "41181", "14473", "30761", "59540", "1022", "6737", "276", "27213", "45266", "36646", "23148", "22720", "1167", "17062", "4378", "43296", "30959", "18035", "56900", "21500", "22855", "1382", "27326", "56811", "30829", "42142", "47891", "48824", "47483", "44636", "40682", "9860", "11314", "21009", "18546", "39000", "43033", "18987", "35968", "16240", "19674", "41966", "38308", "36202", "30319", "3651", "29681", "35645", "5336", "11481", "46056", "43006", "23268", "46579", "17264", "29814", "46339", "57142", "58479", "3562", "23182", "24534", "53376", "57798", "48760", "6209", "24686", "1464", "42219", "36410", "3753", "41991", "45679", "695", "46548", "12118", "11044", "28154", "25234", "3549", "48450", "54253", "34372", "1170", "3974", "22090", "18573", "16299", "7901", "59112", "34479", "2370", "3630", "39127", "26565", "54773", "19819", "57332", "39227", "31626", "15826", "32691", "17566", "35076", "44751", "58996", "12101", "55983", "9159", "51007", "2898", "22743", "48512", "21518", "36698", "57238", "29293", "56464", "28226", "7628", "28122", "11288", "41322", "16688", "36551", "31097", "16165", "21356", "52105", "55869", "3216", "38066", "41500", "4195", "27591", "53602", "1396", "48883", "33960", "22999", "33943", "57269", "16370", "59435", "52092", "10731", "48778", "31402", "48866", "9165", "46290", "26117", "55635", "55953", "6900", "21913", "51294", "21584", "54770", "29016", "53405", "11007", "31994", "6843", "53937", "12859", "30945", "50683", "7060", "48526", "48294", "56169", "33130", "44372", "41589", "6975", "11086", "20679", "17227", "14381", "35875", "42591", "21926", "16727", "3270", "919", "15271", "59900", "22128", "29427", "59608", "32833", "7525", "4729", "12090", "52324", "54646", "9525", "27585", "42813", "49791", "19841", "28344", "9901", "21063", "55216", "43971", "21142", "50691", "27936", "37192", "21219", "1065", "57027", "50408", "26433", "21987", "35392", "46328", "35672", "39503", "33045", "34051", "13240", "46800", "40161", "42861", "59381", "7991", "28081", "28413", "20167", "30385", "27038", "23088", "267", "39718", "9352", "35954", "24325", "46374", "8260", "7738", "12445", "2636", "46995", "47890", "29482", "47409", "20902", "6640", "53267", "14324", "4548", "18148", "766", "45052", "20908", "2790", "23868", "42854", "40973", "38550", "18349", "23364", "30111", "6720", "57513", "51517", "24674", "34926", "13611", "30637", "29457", "44162", "52574", "28235", "44041", "52219", "5590", "49407", "15991", "33028", "4925", "55751", "30841", "34279", "17150", "30377", "15443", "14058", "42859", "25690", "13497", "57477", "18588", "56494", "56823", "19887", "23718", "28746", "39395", "38854", "9814", "15389", "21379", "10801", "13594", "17510", "38175", "3272", "41528", "27094", "30102", "18620", "47422", "52616", "48386", "26670", "6051", "34656", "11780", "57682", "49394", "27581", "7657", "54956", "38611", "29422", "15052", "30019", "14349", "31552", "30192", "78", "8692", "36722", "58139", "51725", "25162", "47494", "46748", "39840", "5524", "26646", "50766", "8685", "20103", "25795", "55548", "46115", "37173", "43339", "30215", "14018", "14395", "25528", "45601", "30909", "26931", "757", "32197", "56144", "29735", "58702", "157", "49270", "40920", "16384", "1398", "52592", "41703", "1454", "45358", "37373", "43721", "31559", "33569", "5819", "31511", "59035", "21399", "18350", "49430", "15305", "24581", "41392", "18944", "11722", "18961", "52981", "47788", "5883", "40999", "13039", "41106", "57662", "17103", "36094", "16379", "22011", "10085", "49022", "42790", "52374", "44130", "1277", "18354", "11603", "12610", "32233", "27945", "29281", "36599", "29326", "20276", "25297", "42627", "13649", "31273", "25335", "5549", "7511", "4215", "34382", "40836", "48605", "55165", "26401", "56363", "1655", "49710", "3337", "52944", "15515", "32079", "41351", "12368", "70", "57629", "32050", "55031", "35813", "42379", "1810", "19668", "14549", "47276", "42516", "15448", "53108", "22880", "50880", "35147", "32659", "26098", "21270", "41189", "901", "35082", "3465", "6821", "37814", "29849", "43079", "59226", "49842", "22843", "8570", "35543", "31121", "52175", "26033", "31915", "9238", "9816", "2089", "4452", "10850", "31535", "37518", "17121", "8831", "23906", "15046", "13672", "44622", "23638", "38869", "17233", "3016", "2558", "25613", "1089", "25950", "26624", "42475", "50702", "2681", "12883", "32169", "14330", "5299", "9226", "10647", "29341", "42751", "52653", "50130", "56328", "14463", "21663", "9035", "15387", "52189", "54825", "50340", "50010", "52889", "45124", "238", "33396", "23900", "19685", "9704", "36602", "52754", "52302", "53366", "45757", "8236", "53006", "57893", "36601", "1663", "18033", "7121", "10782", "7342", "13581", "51476", "7568", "21042", "40550", "11293", "13997", "5155", "12931", "20219", "28818", "1231", "13944", "38191", "34428", "54746", "1680", "29245", "22691", "6397", "19881", "3311", "2597", "56167", "23699", "43471", "30520", "7322", "17845", "59616", "15723", "52789", "30545", "57276", "27158", "8634", "244", "52011", "12333", "39077", "28000", "45577", "17256", "27805", "10302", "32230", "10908", "19878", "37325", "54723", "41706", "15348", "13781", "33048", "12962", "44331", "28610", "56369", "14968", "12239", "8522", "43060", "24110", "56813", "20849", "55435", "14359", "2659", "57706", "34255", "53163", "37643", "9369", "26194", "23280", "13190", "19431", "15739", "43089", "13188", "5167", "33147", "38770", "20472", "35621", "59843", "4460", "18386", "2021", "19698", "58353", "41669", "15888", "36313", "22214", "13981", "55140", "43088", "15566", "21076", "3152", "15807", "55882", "36652", "46528", "4909", "10991", "57848", "1062", "31714", "24104", "47962", "23513", "26012", "21012", "44826", "18264", "10072", "52184", "9653", "35103", "39987", "7593", "10627", "17183", "57804", "11587", "10458", "59913", "53811", "2128", "40157", "17470", "12832", "10972", "5861", "33959", "43111", "52227", "57634", "35025", "22015", "1976", "1182", "56679", "43982", "16637", "1591", "3673", "26314", "3831", "44861", "30203", "50368", "42661", "18956", "57076", "11960", "16661", "47040", "15292", "11625", "34498", "28208", "18765", "2368", "48041", "50731", "11102", "21677", "11505", "48065", "28992", "45808", "40425", "26584", "37226", "34102", "35350", "5275", "26663", "1762", "13391", "5091", "26424", "4381", "45489", "3486", "40827", "6311", "22592", "53128", "28887", "28120", "28690", "38244", "21436", "19202", "56379", "44703", "37340", "8390", "8503", "55932", "59320", "4233", "58797", "21903", "32403", "51602", "57606", "42998", "45744", "940", "37206", "29144", "25875", "36959", "8308", "31908", "40168", "25903", "52337", "16790", "42641", "8238", "24693", "21339", "10739", "46660", "23589", "28644", "47093", "8846", "4906", "51376", "6137", "216", "45421", "45800", "58648", "11169", "57344", "58138", "2552", "50776", "8859", "7249", "43146", "37664", "11890", "36479", "59370", "50953", "12881", "43618", "46046", "14727", "595", "41023", "55790", "2348", "4557", "9832", "17011", "34151", "37104", "38406", "28738", "19713", "35618", "45019", "27283", "15380", "18274", "3099", "6511", "15767", "14829", "19323", "2498", "8277", "58654", "33841", "45954", "6096", "37697", "53020", "16016", "42292", "27793", "20752", "58592", "1371", "39908", "44326", "40597", "53388", "25252", "6567", "3285", "37906", "18626", "42822", "59594", "27672", "56698", "59008", "8658", "5158", "29829", "1495", "53925", "24462", "9800", "45813", "54575", "23124", "44015", "52255", "48111", "20666", "29190", "35745", "27518", "9851", "13208", "42015", "42572", "40638", "37674", "25874", "39196", "48516", "40073", "46169", "11077", "15446", "11530", "35725", "9607", "5722", "37133", "44354", "31486", "2150", "7183", "54067", "5638", "10039", "25496", "28219", "44885", "38545", "46611", "32905", "26639", "21765", "9786", "46304", "47267", "3917", "1121", "8450", "2803", "1935", "30722", "13740", "6453", "22814", "11025", "49771", "31708", "43238", "54349", "22864", "18422", "8676", "15408", "2221", "41126", "36097", "21540", "55376", "41265", "48025", "44439", "59863", "56429", "19754", "32518", "10389", "13974", "54568", "3691", "17676", "48408", "37621", "42969", "40342", "47566", "248", "3497", "4058", "7485", "49656", "53456", "7646", "38469", "56001", "39224", "52174", "30918", "39537", "48094", "47917", "22741", "56315", "4810", "15969", "32808", "10489", "3870", "35792", "29881", "55791", "27500", "43365", "9436", "25815", "37775", "38057", "50863", "43101", "46210", "37459", "2037", "36774", "29898", "31034", "47533", "7578", "21675", "45953", "50922", "13588", "30544", "14159", "50632", "59860", "38547", "20974", "47292", "56183", "11197", "4270", "50867", "10552", "49953", "27364", "8282", "7395", "28425", "2534", "42332", "24482", "9426", "19347", "34732", "37965", "58958", "17516", "12427", "28968", "7841", "20875", "35457", "29440", "34696", "39362", "3163", "28473", "10448", "41343", "9529", "486", "59106", "50516", "905", "12568", "39428", "16908", "19252", "29998", "30302", "56702", "42992", "24044", "9028", "41751", "52634", "41906", "56569", "1806", "15942", "59741", "32798", "20381", "29497", "52658", "50601", "9285", "42245", "51634", "37515", "28963", "15481", "12571", "18469", "30118", "19796", "50586", "44328", "15169", "35998", "53893", "23776", "22014", "50512", "16398", "6797", "12499", "22398", "44952", "24206", "18491", "48901", "26208", "6868", "43999", "43075", "32945", "1156", "8980", "32513", "30655", "42328", "25109", "49128", "45642", "40772", "13456", "4296", "20063", "33417", "23108", "21496", "17774", "23711", "35644", "44216", "5203", "39035", "13566", "22178", "43931", "11953", "59278", "30588", "57083", "8490", "47007", "29972", "13452", "57325", "11545", "38622", "32594", "32501", "40957", "6412", "52733", "27967", "41411", "14579", "23913", "33462", "34622", "33120", "37428", "49242", "24235", "34090", "54841", "9700", "24877", "50568", "55262", "34629", "16820", "14099", "59356", "57189", "12983", "15957", "38810", "24181", "5675", "35116", "18767", "29112", "45107", "51918", "1432", "56948", "38183", "52983", "46542", "58911", "24732", "33691", "23299", "39735", "47439", "12212", "11728", "10612", "17295", "34145", "4694", "56417", "16225", "58421", "56466", "33166", "57002", "48159", "43743", "8944", "33412", "57220", "39805", "17168", "32499", "36107", "37199", "46883", "54404", "21835", "23468", "20341", "56438", "32068", "22550", "1209", "16877", "20837", "58594", "38152", "23261", "28320", "17136", "2490", "42143", "46655", "1069", "41892", "25725", "46197", "21394", "47691", "47221", "57341", "13880", "4860", "31143", "34399", "12751", "7595", "39179", "23562", "58454", "38655", "32908", "17800", "54601", "5119", "53797", "15064", "21311", "54235", "27861", "43472", "30531", "21130", "46505", "22205", "55586", "19728", "12770", "5717", "31199", "24046", "35754", "25415", "18410", "33537", "56632", "32229", "12667", "36591", "10967", "37623", "28093", "40120", "10835", "27029", "26299", "40684", "3904", "9900", "33338", "46849", "47874", "45423", "46794", "5729", "49250", "52885", "41447", "2634", "29052", "53873", "9235", "48840", "13217", "7472", "59723", "24218", "11399", "40889", "13915", "53836", "34487", "16854", "16243", "19582", "33278", "54284", "48171", "17063", "41512", "5136", "29484", "34100", "55641", "32350", "3481", "41974", "15856", "56062", "33519", "27858", "54255", "26293", "56572", "27486", "39567", "36279", "48951", "48865", "40603", "34228", "57611", "42777", "15786", "44600", "48085", "23218", "19599", "47092", "42383", "44768", "45279", "9234", "10226", "39795", "10922", "40607", "14720", "35243", "22392", "54593", "41220", "39586", "24651", "10434", "52868", "5465", "38879", "9925", "6128", "20743", "59867", "18457", "33504", "37108", "5526", "43638", "41742", "7971", "21391", "54528", "36116", "23811", "43956", "38304", "25755", "6886", "19489", "11173", "13445", "45355", "37847", "32737", "19700", "17234", "3092", "33064", "17501", "3697", "52225", "32521", "4466", "7434", "16403", "55888", "42809", "33715", "2807", "34319", "44312", "26334", "856", "33004", "12913", "31931", "43095", "27394", "8420", "58044", "8039", "50236", "23476", "24260", "40275", "21544", "51834", "20001", "11148", "58708", "11483", "33782", "19454", "44575", "33845", "50011", "51136", "22264", "53283", "4083", "32554", "50057", "1124", "2989", "51896", "5958", "15661", "57461", "37114", "48487", "59614", "58778", "15161", "41926", "6157", "18882", "30491", "12930", "34590", "8354", "55471", "37737", "40354", "54638", "39246", "51153", "21692", "43741", "51395", "23219", "47940", "46786", "3805", "33697", "39941", "56835", "50661", "53965", "57681", "46397", "45147", "19975", "42676", "43176", "36771", "16411", "44588", "6762", "39911", "47696", "8075", "4176", "56556", "55412", "51202", "19390", "9222", "57373", "21916", "15454", "44369", "6407", "332", "19027", "41691", "17730", "55524", "40227", "53938", "29024", "10639", "41156", "2872", "24483", "22924", "43575", "32585", "21257", "28047", "52307", "57411", "15086", "43411", "20343", "57379", "9767", "22916", "22045", "14308", "16463", "42880", "34854", "4089", "54775", "40401", "3381", "46401", "43231", "92", "26963", "41908", "32478", "37841", "42506", "12461", "36298", "8983", "39376", "44433", "8322", "36336", "48561", "49980", "14421", "35422", "43687", "11649", "33584", "18175", "54808", "32971", "33161", "32413", "52836", "20166", "40170", "47689", "41284", "24296", "10328", "44397", "28616", "8221", "19736", "49986", "30022", "35065", "32211", "38465", "22305", "55183", "24398", "26572", "17922", "50044", "9106", "34186", "44096", "6305", "49778", "53673", "12775", "1477", "13139", "50176", "35894", "4115", "51481", "21473", "13724", "27433", "30369", "14256", "34277", "35403", "8466", "16597", "32132", "26220", "7355", "54734", "22773", "59982", "34793", "49154", "18936", "54665", "33973", "1794", "11161", "35948", "15044", "6446", "36808", "19885", "7466", "52923", "2229", "51052", "5881", "972", "5340", "37676", "43863", "56514", "53072", "53497", "10947", "15460", "20150", "26749", "54520", "36640", "56873", "54145", "5748", "41591", "26831", "45836", "22822", "40139", "20794", "12869", "17727", "25846", "23249", "40512", "2268", "47037", "11067", "7689", "45091", "58904", "54860", "3875", "53896", "59647", "54736", "49081", "21974", "33015", "51815", "42533", "22303", "6841", "30257", "46498", "31618", "13998", "59544", "58998", "17669", "13179", "52659", "57980", "52722", "51047", "52751", "51256", "49765", "35356", "53000", "35371", "37729", "763", "57348", "13714", "25802", "38958", "34795", "6313", "59221", "17941", "42770", "51727", "23781", "7054", "5001", "15681", "35633", "22819", "23924", "2640", "36071", "31781", "11398", "53823", "15109", "42023", "27748", "29948", "39759", "1073", "35927", "13282", "51917", "15765", "56321", "23630", "4063", "33398", "54700", "42987", "31466", "55445", "29667", "46479", "44790", "5007", "18562", "29285", "43061", "8647", "33794", "15596", "46409", "39193", "49615", "8903", "22048", "41808", "53369", "17506", "4509", "7865", "12922", "53709", "58956", "54291", "52086", "14613", "35175", "3015", "42744", "54976", "37505", "12169", "46076", "40485", "1946", "52298", "57167", "46211", "46508", "11592", "35026", "51437", "48841", "27452", "39478", "521", "27216", "16549", "44769", "31694", "51688", "10297", "5982", "41270", "18096", "37051", "47464", "57627", "52547", "24111", "34066", "49321", "19117", "7816", "24708", "3682", "1456", "27109", "53633", "52980", "14484", "8050", "10533", "34267", "50317", "13959", "25666", "54169", "58833", "10152", "40808", "25954", "12432", "8968", "17432", "34006", "1191", "30231", "228", "10482", "44888", "34814", "21015", "43876", "33129", "47344", "26450", "58495", "37200", "59632", "38855", "32199", "7499", "58125", "18982", "17638", "38816", "46664", "6062", "37984", "55160", "21328", "14055", "39710", "42793", "34930", "13086", "48368", "48255", "26808", "22047", "962", "55317", "39599", "16609", "25493", "53970", "18699", "34070", "47259", "22746", "25399", "1414", "33672", "42", "47115", "50231", "15761", "44133", "24800", "38433", "36926", "29312", "26107", "54438", "56295", "10863", "40280", "32326", "30314", "54384", "51000", "36466", "52170", "32809", "13492", "19484", "59902", "12837", "11912", "14071", "6686", "9959", "49418", "46962", "20305", "32612", "1351", "36711", "26145", "25249", "30453", "32104", "26823", "53329", "11168", "5582", "8970", "20206", "42438", "25254", "59856", "34600", "10268", "37166", "1639", "3401", "4208", "27171", "55977", "38665", "52975", "12606", "16878", "58165", "3857", "11438", "8921", "28385", "43226", "33752", "51056", "14559", "6545", "56185", "5386", "43352", "58545", "27988", "30412", "31688", "6857", "47590", "23813", "36449", "14651", "59287", "1312", "8001", "14062", "27367", "41798", "52735", "22935", "20090", "55672", "37654", "30736", "40933", "37048", "35847", "31856", "3834", "17758", "54973", "24834", "1296", "8520", "22940", "34453", "41303", "39529", "56000", "46701", "55378", "30174", "35043", "36287", "57940", "36870", "18137", "10285", "25228", "23890", "53015", "11755", "52963", "45400", "41461", "4715", "59334", "38320", "41916", "6317", "17090", "34385", "24941", "833", "53231", "3645", "32671", "18523", "41968", "3414", "30927", "41383", "23783", "13853", "6727", "2107", "52740", "9053", "58172", "41107", "24481", "49652", "24870", "17445", "16070", "2455", "31752", "14781", "1023", "18052", "2994", "14623", "56967", "22703", "8425", "56233", "36827", "24684", "25676", "8400", "22322", "55285", "24643", "12081", "3334", "23659", "53176", "21028", "34718", "52561", "34836", "38052", "24092", "28215", "42816", "33642", "43874", "42864", "12670", "57693", "32615", "17854", "32937", "18845", "2573", "30755", "40043", "16799", "27553", "18793", "36665", "29109", "7632", "50412", "22098", "58758", "37961", "31208", "42716", "46550", "32940", "22962", "3494", "47592", "34084", "18168", "50956", "41359", "31857", "44217", "50095", "19827", "8013", "44822", "48792", "33602", "57957", "34338", "3071", "35347", "50107", "45116", "39653", "41007", "14196", "8609", "27240", "22545", "52272", "15306", "10415", "2167", "41067", "39856", "19644", "18892", "35620", "6270", "44997", "53451", "12490", "18725", "34309", "1856", "18072", "58619", "27225", "6160", "11478", "53156", "11716", "55518", "2722", "44083", "11794", "48160", "53313", "1678", "53930", "31927", "27266", "56542", "56362", "48449", "22983", "46059", "41759", "46237", "650", "45933", "7557", "45842", "36122", "5137", "35360", "49137", "25226", "50297", "56376", "21664", "7631", "24610", "27722", "38746", "4724", "12991", "7429", "14208", "13314", "8254", "15163", "4123", "21368", "56294", "33165", "37846", "58900", "24228", "41027", "21410", "55843", "51724", "18768", "57449", "10670", "16494", "36138", "12900", "32416", "52120", "38758", "5967", "26901", "45017", "28577", "4434", "11889", "59060", "8157", "50298", "21581", "29593", "3865", "55170", "29778", "24404", "39194", "29662", "9438", "49391", "22101", "43712", "12583", "49659", "52703", "6671", "17835", "14798", "52788", "29613", "46174", "46", "59014", "50018", "48323", "56097", "36633", "46874", "52635", "40458", "958", "14665", "54433", "31107", "3783", "42390", "33831", "13911", "11108", "25184", "33303", "2124", "352", "53293", "10724", "12679", "5150", "14425", "33809", "45617", "30357", "51385", "53886", "35126", "39552", "35088", "42524", "23942", "1970", "16710", "6292", "33615", "5733", "14181", "56849", "13033", "21756", "9555", "49864", "58289", "25551", "20320", "59534", "2877", "10981", "58906", "2206", "12822", "36045", "38105", "36256", "5342", "56586", "36981", "20897", "11239", "53950", "51536", "5732", "53804", "44749", "30464", "7336", "8615", "6930", "24736", "29391", "4350", "7610", "25857", "18466", "4217", "21195", "46100", "23643", "9423", "15994", "8689", "19110", "23799", "49830", "8993", "59427", "19198", "39451", "30911", "48481", "42155", "3949", "43532", "25744", "18548", "42954", "25368", "27334", "59220", "40059", "9780", "17541", "7642", "31818", "29213", "20627", "40264", "27079", "39814", "39145", "26190", "13772", "25975", "58940", "12527", "17697", "1108", "46183", "57982", "31614", "20072", "41143", "34547", "54181", "55264", "14470", "49120", "58558", "53862", "26720", "17235", "4423", "46934", "48432", "5648", "29120", "23388", "27628", "46963", "23216", "4539", "27832", "58851", "20689", "5390", "43062", "49052", "57577", "41294", "18880", "10247", "32160", "7348", "47448", "6872", "6392", "55527", "59844", "30153", "3240", "9936", "30786", "7608", "17528", "18188", "4307", "33121", "48001", "26362", "51317", "689", "29651", "22979", "6067", "46668", "15191", "44035", "2737", "1923", "31609", "38715", "49573", "27840", "14288", "33351", "56764", "53309", "52829", "39152", "8632", "3957", "45998", "54694", "23975", "47491", "59725", "5096", "59704", "39604", "53670", "49419", "1232", "47028", "42640", "38061", "17075", "45831", "13883", "21051", "52097", "10999", "9415", "36428", "50096", "45375", "23555", "45141", "50910", "44876", "27271", "5025", "26140", "50261", "21797", "17917", "28697", "10281", "37511", "59403", "23114", "30468", "26268", "32861", "52297", "29264", "2796", "43366", "53311", "15436", "22840", "32845", "58923", "52321", "28585", "5193", "17910", "32066", "7164", "53581", "23394", "39583", "59898", "12334", "14505", "44613", "34762", "4766", "52640", "5297", "49992", "4146", "45963", "54947", "49950", "19241", "51518", "15533", "39511", "10241", "32084", "8875", "53971", "22715", "36607", "43611", "59396", "25615", "28162", "33942", "33605", "18840", "5992", "25748", "50480", "15117", "21137", "24857", "37936", "11710", "39272", "42164", "6590", "21786", "1642", "28740", "44725", "43935", "413", "36744", "34259", "55713", "52948", "1942", "45506", "13248", "13111", "45895", "17100", "51618", "10411", "57001", "23053", "49666", "30429", "56507", "55815", "8633", "17049", "6750", "5163", "34206", "54231", "19135", "35524", "23013", "3214", "38463", "14020", "26703", "35317", "12861", "10466", "542", "52030", "44710", "25204", "44490", "53287", "5220", "55294", "41976", "40001", "12403", "27298", "49115", "38477", "35862", "18595", "23252", "39420", "4981", "2761", "56787", "7052", "2828", "7924", "44151", "15291", "22659", "19583", "31619", "32077", "55022", "43163", "57717", "36234", "53911", "5037", "25445", "52850", "44936", "59772", "35973", "15316", "14808", "30392", "22218", "52639", "23139", "50572", "39871", "35746", "11117", "59792", "46473", "52767", "20863", "11343", "24269", "19516", "11356", "38019", "34439", "32087", "3821", "11416", "42062", "9453", "22527", "29478", "13303", "36182", "23059", "16382", "22912", "46721", "7894", "46257", "15927", "5530", "7576", "18538", "7993", "5929", "26843", "32614", "27772", "29933", "40656", "44673", "53354", "56996", "1974", "3992", "41955", "59073", "31045", "19900", "33524", "11329", "25239", "43962", "37503", "34031", "23456", "25984", "39105", "32713", "19341", "10695", "29434", "3880", "26104", "15936", "46330", "47695", "4722", "25383", "7379", "41754", "31176", "28941", "2968", "39228", "18812", "39169", "43202", "49131", "22525", "8202", "32617", "1985", "53738", "31841", "16220", "39499", "59302", "24959", "42094", "24967", "5750", "6483", "2375", "31616", "5889", "29298", "2168", "57879", "36036", "57026", "50315", "57265", "37552", "28397", "7103", "43953", "9243", "48189", "42675", "38154", "40524", "1112", "14096", "1127", "45556", "48735", "52262", "15985", "15591", "46067", "21610", "31345", "33806", "4046", "48083", "9980", "9888", "53321", "40891", "53921", "59624", "31378", "950", "34996", "9217", "27865", "17386", "11774", "55329", "12613", "42519", "14985", "58679", "20607", "46525", "24987", "51993", "15116", "9164", "16983", "42180", "21828", "31791", "49955", "53929", "18777", "18712", "39337", "46924", "54086", "24423", "40595", "16933", "16254", "54615", "20332", "54210", "12125", "16539", "26370", "16582", "25154", "2027", "3068", "46030", "39403", "8756", "46135", "58985", "34207", "5272", "53718", "15561", "16184", "52968", "35498", "21024", "474", "3164", "22665", "32802", "46885", "53664", "38316", "51223", "27928", "2689", "11615", "23061", "42536", "57749", "32109", "6416", "16409", "52818", "40551", "22687", "10228", "51621", "22031", "23853", "41992", "37052", "5055", "58539", "46914", "34973", "37653", "56129", "5323", "31526", "33366", "43820", "59394", "35476", "38124", "8175", "10449", "37706", "1729", "39059", "33914", "5614", "31957", "40241", "46112", "18675", "49295", "51241", "37238", "30474", "28702", "12710", "14127", "42972", "33707", "41389", "3085", "44819", "42377", "33586", "56806", "43327", "8364", "44298", "47807", "33795", "19013", "57343", "51568", "57159", "35167", "50209", "59727", "9720", "17464", "30077", "51563", "36522", "26850", "1302", "50936", "55926", "42645", "7648", "4992", "58939", "30767", "35316", "34140", "51068", "12618", "56574", "35920", "3053", "16890", "40439", "10088", "26227", "41354", "57063", "44193", "43495", "25618", "26940", "39900", "8644", "44873", "51006", "6833", "36183", "8215", "45947", "22730", "27791", "41157", "1526", "10494", "37335", "6177", "36005", "44785", "17844", "19197", "12951", "4681", "21599", "2663", "21723", "17177", "5886", "49687", "55846", "38631", "46186", "19141", "3320", "16894", "40262", "44009", "58844", "16911", "47950", "30076", "35339", "12587", "15283", "51282", "40190", "38282", "15248", "38708", "31247", "48688", "55238", "11579", "44422", "5806", "49930", "8389", "38561", "43732", "44002", "36281", "35373", "6896", "30581", "29570", "11377", "5550", "38936", "56710", "9120", "49805", "39076", "42103", "59701", "26429", "17716", "22370", "43350", "41390", "38910", "26577", "21529", "9467", "8969", "19189", "51364", "58002", "40520", "54273", "17513", "37929", "6571", "42327", "15582", "6509", "7872", "18024", "55797", "49749", "404", "45042", "20604", "4064", "9146", "51644", "43961", "46640", "17349", "20016", "28895", "31913", "6451", "30408", "22970", "19508", "33241", "2301", "10706", "741", "51441", "43405", "44034", "50499", "34534", "19830", "3701", "49406", "5632", "19009", "11310", "28374", "37393", "35446", "8771", "23446", "55574", "39365", "52048", "27569", "40938", "41128", "20377", "34602", "25342", "26116", "39354", "15536", "1099", "14264", "22139", "44360", "50436", "47850", "45044", "21919", "51749", "36970", "18029", "883", "46141", "11051", "31904", "46763", "23635", "48926", "17332", "5374", "52807", "17322", "40947", "25889", "13388", "43556", "11699", "12231", "54929", "22869", "25747", "37591", "59990", "41178", "34008", "51280", "39812", "54862", "15239", "18241", "46350", "41747", "52709", "5378", "49146", "42905", "35425", "19929", "56404", "57302", "54440", "11607", "42918", "20843", "34299", "57599", "14378", "6825", "57439", "11006", "43401", "53580", "6009", "37353", "44071", "46601", "44445", "34074", "58890", "46976", "21327", "42456", "21365", "15045", "56965", "5879", "35573", "21761", "38548", "12801", "18062", "38206", "3994", "694", "40766", "25926", "59580", "52364", "37236", "53802", "54903", "45105", "22419", "29417", "42279", "11146", "55278", "16009", "50895", "22151", "54725", "428", "14770", "31503", "10761", "18022", "1837", "31518", "25074", "45132", "19569", "23308", "18991", "26423", "24571", "24132", "26965", "17955", "32669", "8974", "4428", "49839", "38700", "7168", "43451", "29399", "29151", "36146", "34966", "4202", "22390", "2153", "44138", "16161", "29352", "38918", "16155", "57514", "14899", "517", "22097", "30371", "30344", "24594", "7324", "31305", "49067", "35970", "16801", "29372", "27366", "13442", "37394", "18915", "42826", "3921", "47335", "36062", "48734", "16555", "22071", "5761", "26906", "42805", "36784", "23511", "18986", "19168", "25132", "29053", "26905", "4390", "35595", "32739", "52895", "27782", "46628", "22986", "16466", "10238", "46480", "1234", "20101", "18192", "32065", "54177", "50218", "49140", "29887", "48443", "40971", "11334", "32290", "4015", "11824", "13140", "26073", "30899", "36085", "30212", "58422", "3592", "24915", "53424", "18400", "38146", "44479", "9451", "9262", "44818", "48625", "52769", "17975", "14369", "57718", "58362", "45143", "34856", "18854", "35000", "57652", "19512", "10902", "55368", "50478", "26329", "6905", "21863", "43461", "47861", "49674", "47059", "1157", "10937", "11318", "47223", "15152", "11494", "35135", "47228", "58520", "29255", "10725", "37063", "32450", "8704", "2463", "53684", "12810", "9693", "29126", "5343", "22617", "7081", "48907", "6958", "27463", "15154", "14040", "59666", "50838", "27276", "36247", "41895", "9534", "2619", "41433", "8145", "59286", "58497", "10973", "47451", "36817", "3170", "18631", "34691", "39025", "31900", "42170", "30934", "24927", "57896", "53815", "57894", "39575", "48", "46733", "59412", "9497", "54098", "49881", "13370", "5944", "40875", "56222", "28822", "4623", "11004", "27520", "47524", "10390", "43152", "21265", "19683", "226", "23656", "50628", "58342", "57368", "55231", "51320", "38434", "1954", "58557", "47098", "55928", "22673", "28165", "21444", "45296", "46607", "8643", "17624", "14311", "45152", "28842", "673", "3558", "46882", "19980", "41503", "41641", "39888", "52456", "26130", "7349", "39974", "58603", "22049", "41851", "31130", "8642", "40345", "53090", "43213", "57971", "8416", "14032", "34729", "32098", "50643", "1998", "29409", "47694", "50445", "26303", "37217", "1812", "24622", "23904", "1184", "53809", "45849", "11972", "14655", "48470", "9621", "2171", "40263", "25581", "56055", "7725", "55098", "9633", "43726", "7944", "21310", "2507", "52700", "54465", "35122", "46771", "4316", "55795", "733", "6264", "38725", "55106", "58088", "35494", "95", "15277", "10715", "40195", "53109", "54752", "37592", "20662", "8945", "31902", "35208", "4819", "58121", "54155", "55499", "44387", "35274", "51843", "46532", "21116", "49858", "36540", "17393", "15692", "30165", "11257", "51585", "39927", "43434", "43838", "27584", "11379", "26746", "25229", "18909", "34905", "26383", "4647", "22595", "20029", "5851", "40856", "52079", "28890", "40625", "12866", "16269", "20486", "33110", "25138", "44957", "32400", "10565", "8232", "53576", "6120", "46449", "27228", "31052", "59066", "23250", "26090", "45597", "46507", "20710", "34080", "50961", "10566", "39092", "15247", "31580", "52757", "24414", "2947", "35844", "6919", "46258", "6355", "41053", "21851", "29429", "17956", "54442", "28075", "45324", "9830", "16404", "40250", "25883", "5830", "47291", "25059", "43381", "47087", "45159", "5204", "1244", "57022", "49810", "29132", "28128", "40257", "25106", "3625", "41596", "9747", "32468", "56208", "20756", "6322", "11640", "6597", "19075", "36991", "30454", "41210", "51628", "19898", "44085", "29485", "56812", "43181", "4050", "32951", "47586", "2967", "51665", "11128", "55330", "8441", "43814", "3100", "8856", "11289", "17484", "21438", "38321", "55453", "29396", "40644", "41852", "31332", "54288", "33415", "7139", "36937", "27909", "5917", "38352", "2839", "13626", "55150", "42927", "57915", "23745", "52750", "51666", "33986", "10788", "32349", "6854", "31754", "19983", "28365", "58003", "12619", "57801", "51578", "38669", "46026", "14050", "20533", "27189", "46657", "36888", "51142", "36992", "54801", "50242", "7158", "41668", "13096", "40882", "19101", "2606", "58987", "34971", "32022", "43928", "38464", "1046", "30923", "56435", "45289", "41481", "10812", "12465", "51759", "848", "49167", "43342", "42481", "19455", "37102", "5910", "31678", "43477", "26497", "771", "12708", "32556", "16990", "40070", "13042", "37311", "54035", "48286", "33559", "28824", "18610", "23891", "23498", "3000", "38992", "23478", "31356", "30095", "34366", "51234", "51753", "10360", "17519", "38532", "51936", "20479", "19338", "38438", "24062", "27742", "21477", "19483", "57722", "12748", "43744", "42012", "41353", "50308", "50442", "19498", "4139", "50562", "53010", "35297", "17007", "26103", "9272", "55276", "57941", "53892", "51488", "16154", "7829", "2665", "40292", "41100", "36035", "46678", "24675", "45011", "43558", "53996", "48272", "53634", "7561", "36254", "34720", "50019", "59668", "37377", "47929", "24524", "39560", "15659", "21407", "7301", "21963", "25888", "23441", "13993", "20869", "26864", "32017", "23391", "6746", "19372", "13975", "55779", "10878", "15507", "29152", "39570", "27711", "10041", "11709", "28568", "11757", "9618", "31910", "47034", "35022", "57960", "54875", "50907", "30551", "20115", "26427", "34965", "49149", "19082", "35711", "8110", "23323", "34667", "39442", "36013", "13815", "24896", "43794", "24390", "38276", "49343", "17045", "14500", "5489", "16354", "28020", "42079", "57714", "16416", "6756", "50508", "53251", "28180", "53319", "19148", "23343", "9858", "24743", "24317", "25623", "17742", "28570", "2401", "4874", "5474", "3970", "7015", "1424", "1959", "11167", "38159", "11001", "33347", "21318", "34493", "46539", "23221", "59017", "2906", "10968", "43284", "27230", "18114", "21883", "26711", "5434", "24282", "14321", "23700", "33105", "51529", "18525", "55365", "10144", "48026", "56106", "55225", "46773", "34601", "59710", "7871", "49479", "19838", "5527", "5981", "38996", "5494", "28855", "40797", "10607", "16488", "29574", "51547", "52084", "45638", "6152", "15607", "32856", "40503", "29883", "11843", "16742", "15423", "42321", "5370", "46926", "4579", "29774", "16470", "9398", "37635", "19281", "28427", "8523", "23664", "33669", "8384", "39188", "18310", "2054", "28565", "28069", "9130", "35951", "10498", "26135", "58901", "57283", "31321", "3941", "58673", "36495", "59075", "39400", "54172", "40176", "30159", "19677", "52473", "19479", "32283", "57575", "30103", "25221", "13138", "54266", "30505", "51309", "20508", "51571", "27356", "56515", "55209", "32489", "37106", "46591", "49176", "28105", "10759", "19684", "764", "25393", "15174", "22613", "42500", "7400", "57931", "43323", "44765", "18420", "13055", "39819", "32049", "10156", "10646", "57308", "32537", "9675", "22299", "55254", "55217", "3712", "4575", "23224", "10994", "57237", "5603", "12315", "8700", "57204", "3166", "30555", "57954", "7565", "30848", "22678", "57950", "38490", "29566", "151", "24893", "9716", "57472", "589", "27391", "17357", "18351", "27014", "26111", "7321", "43922", "59759", "56896", "45950", "23143", "57728", "43748", "25079", "18030", "19548", "4356", "8937", "41912", "15652", "27728", "47133", "44062", "13540", "51403", "28875", "18584", "25661", "5871", "23641", "43513", "36086", "7493", "57257", "18837", "39381", "18441", "59652", "20512", "5236", "3527", "22346", "21036", "29135", "5077", "15891", "48753", "47746", "26409", "25879", "5010", "47064", "47264", "35172", "1106", "28954", "20705", "50915", "1000", "33847", "54368", "15320", "5751", "10187", "55563", "59479", "30219", "34621", "54992", "29674", "4919", "32590", "13360", "32993", "48482", "1657", "45239", "43016", "8742", "13187", "50648", "58035", "52226", "30204", "26244", "34674", "33828", "19842", "39329", "1318", "22888", "214", "14167", "52905", "9305", "8448", "11928", "47710", "18831", "28813", "28508", "41597", "55899", "43804", "14755", "34011", "49171", "30528", "32497", "6289", "26045", "51733", "28633", "48789", "49408", "41937", "39083", "46159", "56992", "25253", "1411", "38760", "33579", "14313", "41071", "57088", "16649", "2511", "36177", "8068", "11134", "28078", "42384", "55149", "24920", "28185", "49436", "43938", "40735", "41531", "6487", "47572", "17455", "2574", "13785", "5746", "10957", "33389", "10363", "21720", "24632", "12269", "26619", "5194", "14707", "53029", "19547", "47713", "43216", "43371", "45610", "47095", "22124", "49232", "48590", "39730", "59698", "48348", "33325", "11857", "5953", "59650", "1880", "6479", "28369", "35514", "40729", "40706", "2607", "47714", "43563", "12868", "24654", "53314", "12003", "46495", "11942", "14328", "35871", "2840", "29584", "701", "8726", "6471", "58464", "17643", "21568", "36952", "38519", "44401", "49217", "26186", "26173", "41299", "12768", "18007", "21426", "12860", "27986", "24027", "36209", "25243", "3891", "49542", "21080", "34113", "6918", "56805", "28251", "55475", "36587", "21519", "25895", "18609", "6973", "11280", "17182", "1192", "37222", "14878", "47145", "19761", "1335", "38718", "3307", "18404", "13230", "59984", "49556", "39466", "55123", "10685", "54634", "106", "22262", "36163", "30740", "54677", "46063", "53638", "59095", "5699", "56533", "55653", "45551", "26286", "4995", "5405", "51341", "46406", "19996", "5301", "45252", "47453", "38329", "54625", "51180", "33323", "39084", "42359", "560", "11663", "25013", "51762", "44655", "56670", "22701", "26128", "29693", "45667", "42844", "52569", "39389", "16022", "4239", "34554", "35773", "10407", "13916", "55244", "13625", "4386", "31153", "42934", "6575", "21111", "5450", "36710", "41219", "17334", "51994", "47897", "867", "15773", "3535", "5774", "21157", "17431", "45235", "12035", "18483", "54003", "28582", "52133", "19451", "29338", "35375", "30735", "44745", "37264", "31134", "881", "23905", "7080", "23887", "47284", "56269", "29089", "52745", "20346", "9014", "42247", "12901", "1822", "18313", "56775", "50121", "53760", "3843", "37496", "19171", "28318", "38001", "59005", "41653", "41866", "30825", "58586", "49311", "53428", "41672", "45911", "24558", "2291", "28517", "57323", "55913", "1123", "29877", "31587", "52243", "7438", "56480", "51530", "10709", "13199", "10628", "35384", "11492", "6053", "14828", "34675", "31576", "8447", "29965", "5071", "52959", "25882", "1479", "47213", "27455", "3417", "37280", "16691", "1126", "48765", "7100", "53179", "52966", "22628", "51301", "42242", "48153", "34499", "51873", "775", "37551", "45751", "23509", "35154", "22618", "22994", "12657", "23448", "27934", "53529", "37548", "4291", "29317", "14558", "9906", "32422", "22290", "55823", "20788", "36750", "13654", "27292", "18780", "25233", "59889", "43574", "27968", "15212", "37968", "16837", "37974", "8023", "44497", "37191", "27239", "49106", "6015", "32520", "49856", "48462", "2982", "55252", "18555", "57235", "6284", "11058", "10669", "37995", "25215", "58631", "13733", "43714", "38392", "17690", "49601", "33283", "8981", "22515", "21127", "42819", "40560", "56684", "49583", "17625", "50857", "8878", "23087", "26578", "37154", "1205", "35739", "34318", "21573", "27530", "23710", "41285", "46043", "56831", "20527", "35624", "26036", "31422", "36057", "49786", "25556", "58634", "28602", "49814", "15252", "12478", "35570", "50035", "32042", "55189", "11725", "59811", "47233", "42935", "29039", "6437", "19250", "22261", "13297", "56260", "57586", "21416", "31205", "46705", "46806", "54523", "27799", "22355", "26743", "4858", "9810", "15687", "41546", "38139", "49516", "39519", "39131", "50109", "34920", "4532", "1819", "50889", "16985", "41379", "58274", "57020", "42368", "44131", "4425", "34758", "14152", "51171", "51949", "38299", "26382", "10203", "28026", "16052", "54598", "7765", "21737", "28658", "51950", "44424", "37495", "39513", "35608", "23031", "52450", "10008", "55077", "35155", "54350", "25318", "47755", "58771", "41553", "19176", "27930", "13747", "362", "6964", "53632", "56878", "50125", "29284", "45775", "43718", "44386", "47649", "56547", "53955", "30719", "23717", "13219", "29642", "50290", "11275", "23455", "29121", "50286", "44427", "54590", "28127", "40884", "17858", "51051", "59118", "45613", "19771", "43210", "34864", "47077", "40269", "24374", "42465", "9091", "54714", "52673", "42904", "36697", "9140", "7300", "28589", "12559", "40629", "7611", "2910", "16899", "46318", "12025", "55151", "9373", "53805", "37103", "54176", "31928", "14316", "42316", "46400", "8284", "14562", "43840", "22166", "35104", "15595", "29038", "32469", "58811", "13395", "7782", "55573", "19006", "41269", "18391", "45522", "55350", "31873", "29747", "42172", "15406", "1388", "29222", "39147", "34798", "6100", "50882", "13440", "13885", "20759", "3813", "37771", "1171", "43529", "27618", "36198", "31745", "18103", "25195", "28662", "56330", "1247", "13475", "21225", "49278", "59848", "58213", "22383", "15476", "47021", "37924", "47508", "52181", "14351", "7596", "59543", "40486", "42732", "34330", "45639", "19537", "40807", "46225", "15345", "45349", "18700", "16617", "51694", "41406", "17399", "23582", "46816", "34843", "19591", "39051", "38000", "7526", "34210", "9943", "9147", "34647", "39239", "37320", "30609", "51780", "54654", "35767", "49340", "11126", "28261", "43790", "4811", "15017", "12588", "52655", "36938", "1909", "33836", "30012", "2604", "11348", "30892", "58047", "37937", "26019", "31656", "39737", "44025", "53695", "29725", "55363", "36548", "25772", "57146", "6741", "12924", "46151", "19340", "38733", "30451", "13338", "31444", "742", "31098", "8479", "40703", "10290", "6523", "25175", "54263", "28254", "50494", "2759", "58219", "47655", "16835", "9475", "25365", "50331", "11736", "51842", "24463", "59585", "40308", "26262", "148", "59266", "9416", "23952", "32307", "19994", "2391", "56543", "35190", "42448", "11512", "20637", "24973", "44536", "51890", "4783", "19718", "25680", "40886", "49087", "44495", "36703", "22802", "16507", "4587", "34060", "18265", "3663", "1981", "16192", "3061", "25582", "31866", "48243", "51992", "2035", "58768", "30975", "33953", "2332", "2875", "52276", "2403", "27078", "5156", "14518", "52366", "28797", "47196", "20044", "1181", "33521", "9963", "10840", "25269", "57661", "40888", "1458", "10640", "59464", "7531", "25916", "51469", "39267", "18398", "301", "57421", "39368", "3205", "17796", "4134", "33900", "34099", "49748", "58689", "21384", "53171", "35695", "46356", "11225", "50560", "18068", "57838", "1487", "28622", "13259", "19853", "53008", "46206", "24331", "24271", "41781", "53661", "45583", "1293", "13461", "8537", "32435", "27100", "38688", "15988", "22022", "45261", "14910", "31800", "473", "37324", "39119", "41686", "12441", "20890", "18750", "43097", "49661", "45449", "34435", "39343", "14819", "21620", "39872", "34630", "27144", "1576", "15948", "34900", "42174", "24865", "16100", "1536", "6088", "27631", "53525", "21971", "57390", "32434", "4613", "48533", "3591", "58086", "40667", "14611", "1930", "8982", "6862", "44420", "58205", "6256", "46023", "33304", "48483", "56797", "29874", "58253", "26207", "30347", "47958", "29754", "40011", "22064", "36115", "49598", "50117", "15801", "33172", "30238", "55117", "10661", "10516", "37770", "28035", "55560", "6724", "38995", "34869", "48755", "21198", "32192", "34541", "40035", "578", "48079", "19810", "19020", "13713", "29734", "4683", "44185", "50551", "1460", "48417", "28383", "8718", "48288", "26875", "15715", "2775", "26464", "54627", "50029", "11786", "16062", "21579", "11193", "25431", "37954", "51974", "15344", "36100", "40044", "35541", "22611", "37120", "22893", "55874", "17889", "9827", "53856", "4931", "40462", "13065", "54330", "38964", "1795", "53201", "5144", "26579", "9548", "53795", "43848", "48013", "30284", "22217", "6819", "38025", "49211", "28071", "15530", "56419", "48234", "29918", "6703", "30138", "35218", "33392", "15658", "40862", "14688", "47736", "5848", "44607", "43926", "43488", "50377", "15971", "13943", "39933", "33057", "16093", "31074", "58780", "58885", "40397", "37797", "22553", "57786", "10133", "3556", "37836", "39459", "5810", "27552", "32067", "30956", "27834", "20322", "28774", "34488", "57702", "49244", "65", "19701", "5026", "41506", "2231", "46693", "15349", "13533", "18997", "21029", "23119", "38117", "4443", "18025", "25397", "22941", "22542", "42434", "40216", "39250", "31173", "21796", "16111", "54014", "8531", "55457", "939", "12272", "45428", "6928", "44098", "41613", "48980", "553", "1595", "16284", "59088", "33835", "25466", "22408", "23207", "26257", "48505", "21528", "6521", "50100", "41532", "3567", "18732", "1557", "29655", "14423", "3030", "16050", "50526", "10783", "54461", "5284", "32102", "32837", "32930", "57144", "34727", "3160", "53516", "9695", "51565", "21159", "12312", "43720", "43098", "6906", "50784", "53067", "6674", "13337", "27651", "25447", "29931", "43478", "58403", "52073", "29583", "31698", "23453", "56945", "37688", "51872", "49263", "28070", "50463", "30783", "48282", "16405", "17594", "58293", "31249", "50809", "52309", "12177", "7362", "29906", "10479", "33336", "55356", "52402", "24662", "17407", "46466", "27462", "14474", "51758", "54557", "34592", "59274", "819", "17298", "12381", "31075", "59851", "27355", "22602", "53952", "49290", "20798", "36241", "33464", "11408", "47686", "41248", "5825", "20327", "19233", "21017", "37508", "39913", "54159", "15923", "58148", "44363", "15678", "29793", "51168", "24442", "47536", "31312", "7468", "57819", "4700", "29878", "41416", "18477", "47254", "14695", "42472", "32472", "21064", "45687", "8320", "16296", "438", "46537", "24734", "29445", "7523", "23859", "49042", "3634", "9634", "11204", "11172", "20768", "35926", "7547", "12400", "11268", "16635", "48683", "18736", "7372", "57270", "32149", "30462", "32186", "8397", "31288", "39965", "54762", "6566", "28803", "4184", "12632", "3316", "3937", "5977", "41191", "25422", "252", "41477", "52996", "40692", "55102", "8031", "3220", "1553", "21288", "54589", "51001", "9334", "17587", "26993", "14202", "54751", "15409", "3468", "23652", "5814", "26553", "25332", "45535", "33626", "5373", "22923", "29476", "41176", "24084", "20152", "47104", "9878", "42618", "56559", "1519", "51428", "7161", "28910", "43249", "45040", "18026", "30168", "2546", "23586", "28698", "26788", "15784", "55316", "28462", "24194", "19948", "59281", "35014", "39390", "45191", "5358", "12793", "24782", "9240", "56662", "9611", "27244", "15638", "18216", "4212", "10585", "21486", "15395", "21638", "47119", "42875", "56750", "8279", "30013", "29973", "54845", "25385", "18395", "58644", "35", "4534", "51812", "11727", "29514", "53855", "36049", "17535", "2063", "4855", "54278", "9759", "31672", "27272", "30261", "40670", "14809", "10986", "38959", "26051", "16290", "48365", "28327", "8029", "32527", "15895", "51098", "47778", "19248", "10590", "37723", "44305", "47331", "37601", "18189", "58823", "34152", "12972", "47831", "45519", "488", "52436", "12959", "5445", "10110", "2814", "14960", "47011", "10270", "6324", "5247", "58199", "6888", "42439", "13340", "28922", "36884", "25972", "14939", "55638", "41809", "44796", "12189", "58963", "22605", "18973", "30553", "15346", "33315", "47418", "23721", "27451", "55007", "55917", "4942", "5804", "494", "55971", "51490", "3079", "16461", "15019", "56281", "3742", "914", "58709", "19494", "35478", "6064", "40471", "27510", "1815", "14257", "59096", "22742", "42284", "29569", "39108", "12364", "40790", "37447", "27291", "15963", "58057", "588", "19380", "46554", "45562", "41727", "28276", "36321", "18134", "28554", "42289", "53596", "39643", "42932", "57864", "56443", "50313", "8131", "6174", "59223", "15493", "15221", "19760", "11393", "13680", "6472", "52178", "31234", "22116", "23922", "50892", "36367", "8942", "19955", "17515", "7908", "15908", "32821", "4618", "54670", "24288", "624", "21534", "55131", "16089", "54236", "23128", "13787", "41525", "52070", "47848", "9947", "55776", "42963", "10398", "57345", "47076", "43103", "24024", "58426", "10003", "35038", "5901", "54321", "59377", "15769", "16324", "30198", "13120", "37539", "45142", "36802", "59620", "50553", "38083", "32338", "37699", "32855", "864", "26984", "25406", "20636", "24560", "16753", "38645", "58094", "22061", "53969", "47519", "6401", "37964", "56667", "51540", "19872", "21175", "18816", "56125", "3746", "21289", "38933", "6890", "16575", "53031", "49554", "41135", "38689", "33126", "42445", "23965", "35589", "47298", "29047", "26531", "14632", "59490", "20104", "39745", "608", "24432", "6601", "36753", "15462", "12030", "18256", "47029", "57196", "1363", "7071", "27550", "11508", "15215", "5081", "35542", "55465", "22213", "33547", "14121", "41177", "36681", "17890", "29649", "54640", "32847", "57086", "32718", "42892", "37840", "37988", "44533", "41373", "43702", "20757", "15416", "15112", "40075", "53710", "6675", "15492", "49030", "2161", "25861", "51390", "10763", "34386", "24771", "218", "11312", "44734", "34347", "50373", "30871", "48395", "22856", "4398", "2421", "45548", "3491", "6808", "35647", "8007", "11802", "40350", "15657", "45826", "23626", "47066", "15229", "58946", "40392", "16385", "42729", "12821", "32815", "32900", "55961", "44974", "39665", "6155", "13921", "16546", "7515", "27693", "40878", "47892", "19648", "51559", "39594", "2175", "791", "25750", "52475", "45765", "34786", "25062", "58776", "29930", "37466", "45202", "13876", "45274", "30036", "57589", "38318", "26939", "39945", "29122", "2660", "48208", "18580", "20134", "14172", "47737", "56503", "58430", "13767", "25897", "43050", "9212", "34125", "22782", "39684", "30271", "1647", "15665", "10594", "50301", "45020", "17872", "59470", "9920", "51026", "23490", "56558", "48332", "3257", "7033", "15932", "58541", "49607", "27022", "37270", "16344", "5216", "44184", "26372", "33928", "44571", "14898", "48394", "59676", "23946", "57114", "30389", "51100", "41854", "26367", "28378", "52544", "12635", "40164", "13264", "5432", "50215", "47014", "18", "18963", "27765", "28991", "27796", "45", "40288", "6111", "45683", "10825", "1621", "25128", "28342", "39824", "23794", "34157", "3735", "29143", "54152", "53631", "19861", "27080", "43899", "8904", "43957", "13952", "8082", "32334", "54760", "19813", "57486", "49764", "55009", "30726", "34132", "13522", "45464", "55129", "16029", "9092", "23179", "16728", "49835", "16987", "11845", "42508", "15016", "9785", "9750", "36151", "44576", "18852", "36864", "35413", "24881", "4954", "3226", "44020", "17083", "10376", "40260", "51072", "32292", "8545", "3176", "44644", "14804", "41304", "8862", "37600", "14004", "33388", "54518", "23803", "27165", "53520", "12513", "33025", "34709", "14796", "59037", "39774", "50339", "1300", "42160", "41705", "28060", "42910", "33666", "35512", "31795", "34961", "7685", "55349", "23842", "10584", "45600", "33299", "10253", "51141", "47739", "32055", "21877", "31246", "50203", "23168", "22664", "25606", "6239", "57303", "47578", "4929", "49124", "49292", "1865", "17323", "54338", "57360", "3019", "27107", "51743", "28709", "14791", "58308", "22448", "28937", "44473", "24605", "46749", "2837", "4565", "34394", "9702", "49770", "52428", "17612", "31786", "52526", "1461", "56461", "4572", "8513", "5641", "47271", "35267", "57567", "7092", "8897", "30985", "15672", "40249", "14978", "9079", "17896", "27484", "30642", "59383", "55600", "29768", "16203", "34276", "39042", "7730", "24517", "17019", "15730", "12643", "6489", "8820", "10753", "27790", "52489", "58033", "3923", "13038", "20715", "31617", "39622", "22587", "41146", "58234", "12918", "1292", "31112", "54809", "25069", "18186", "32695", "56333", "47377", "33218", "30123", "9508", "53397", "33780", "51430", "18890", "7082", "47862", "51901", "59151", "527", "25236", "18817", "57293", "34797", "34018", "29156", "19246", "2149", "5821", "27726", "31477", "37822", "12540", "53269", "20298", "1134", "38717", "56358", "28259", "6497", "29046", "42520", "19641", "59103", "9258", "40631", "36164", "35803", "26462", "40948", "36232", "29557", "39104", "59904", "42803", "21342", "48697", "45120", "39534", "2521", "22173", "31810", "40964", "4559", "18520", "20469", "2811", "49498", "40665", "52679", "49260", "37500", "2797", "54577", "7939", "5583", "16378", "20568", "37693", "31500", "7013", "1452", "29744", "18252", "21468", "14238", "14088", "48747", "18380", "23838", "38323", "21204", "35551", "2953", "55079", "8535", "14034", "46422", "1706", "35997", "39462", "58633", "21267", "39542", "22250", "10183", "40320", "38197", "40323", "51065", "51373", "14194", "48048", "10147", "33522", "30025", "49037", "42514", "22345", "41677", "36012", "17511", "57267", "20939", "13631", "12919", "53328", "3987", "26139", "55782", "10712", "55801", "57553", "46555", "53094", "26041", "20388", "22454", "46001", "34878", "36610", "29648", "26725", "2750", "56592", "9410", "37881", "57309", "6410", "11852", "16884", "5130", "59873", "36612", "34679", "21087", "21250", "50409", "58794", "46867", "46910", "53135", "8295", "42974", "49009", "45756", "31536", "40255", "23519", "17325", "22111", "34093", "36556", "13201", "30216", "3907", "46275", "22943", "49414", "2116", "35406", "27895", "30715", "37488", "30846", "28419", "57287", "16785", "28456", "1075", "38806", "23727", "59959", "23614", "33843", "51770", "23248", "58203", "16674", "34227", "49253", "55436", "53277", "44641", "6656", "41623", "49043", "46639", "27543", "10723", "7846", "19800", "48736", "29551", "46338", "59627", "4314", "49314", "48529", "11285", "11782", "28657", "58188", "18652", "7618", "14574", "8093", "38668", "16680", "47636", "14089", "11223", "38625", "15557", "31200", "43583", "13770", "43501", "21463", "59949", "29473", "31911", "21326", "22340", "15173", "46438", "4824", "31286", "45990", "45863", "50177", "19592", "714", "26673", "30615", "4072", "40519", "37868", "44566", "19806", "41523", "8474", "1309", "49631", "57112", "21232", "5192", "45382", "22853", "44977", "21004", "55137", "44508", "53158", "33690", "39656", "29433", "57135", "36300", "26387", "14776", "23365", "22882", "663", "31965", "31833", "27111", "57875", "42547", "23648", "32204", "25498", "40427", "56231", "8460", "15240", "33717", "42095", "38285", "37438", "2933", "59568", "22531", "30600", "23588", "51464", "37321", "33545", "53960", "11265", "16014", "53850", "14443", "45720", "59388", "42691", "59025", "53056", "46297", "38188", "32771", "19963", "43389", "36048", "54727", "47934", "36292", "31128", "26948", "22065", "25607", "59121", "55688", "1510", "46456", "43297", "24136", "36539", "802", "53668", "25442", "19289", "43813", "10325", "45717", "18307", "54240", "37227", "26540", "28527", "5296", "49415", "5047", "55761", "38994", "57886", "2643", "6805", "22571", "48441", "22688", "3461", "37558", "38601", "18742", "42652", "9439", "5595", "18927", "47896", "6698", "35139", "44937", "29209", "58312", "27696", "7938", "11582", "51378", "15890", "49346", "13372", "47995", "31437", "17712", "35759", "47991", "37506", "35974", "36117", "45546", "8949", "11537", "45640", "58569", "46811", "31764", "53061", "37333", "50901", "53226", "49880", "37982", "11183", "54713", "38415", "57820", "40414", "19389", "27873", "50235", "8847", "36015", "54482", "20642", "55186", "24330", "43565", "53792", "17097", "13694", "7545", "34520", "35503", "17608", "38085", "32011", "7571", "30214", "38819", "30054", "3361", "43959", "22858", "41419", "58146", "54990", "23271", "9872", "55855", "14223", "32073", "5676", "49447", "48931", "54546", "41855", "55501", "34162", "40802", "22780", "55080", "42789", "37109", "15590", "40331", "15534", "8172", "31081", "8199", "37602", "29101", "9303", "52427", "31598", "31171", "5291", "29893", "49667", "32387", "31775", "4923", "17698", "35801", "34246", "25805", "27954", "28903", "36903", "59626", "39423", "51220", "22442", "49157", "13466", "5169", "50520", "27344", "41707", "49402", "8686", "34202", "46889", "22522", "171", "46852", "10619", "40259", "42539", "52798", "32252", "32629", "9602", "20790", "26032", "45513", "11540", "26323", "15634", "59757", "47936", "46090", "17321", "37023", "37292", "43467", "178", "38606", "8850", "22626", "9464", "11861", "3073", "27719", "13650", "29707", "30968", "52407", "58629", "13871", "11558", "7935", "43705", "25934", "47359", "1809", "34613", "54379", "12245", "53362", "56194", "52739", "24521", "29594", "4163", "36727", "24695", "10506", "22041", "31521", "52171", "20781", "31639", "632", "44624", "44317", "58550", "42913", "37901", "55900", "18521", "29385", "51089", "22568", "3244", "38038", "4731", "31434", "8431", "40822", "36737", "58482", "37400", "23898", "40361", "11818", "16610", "42785", "20822", "20540", "44666", "16757", "29321", "55450", "18019", "42553", "42576", "28360", "37882", "56562", "30995", "43160", "11697", "9389", "54567", "13705", "30180", "38078", "26796", "1673", "53463", "24493", "10629", "35928", "5141", "47946", "34572", "53118", "32964", "51224", "12882", "17838", "16696", "25962", "48733", "36478", "40583", "56671", "4282", "59950", "37810", "36393", "43580", "5171", "31110", "2396", "22799", "20557", "28909", "58843", "39231", "43627", "46148", "29688", "2222", "46496", "55040", "59944", "28184", "56641", "50504", "5954", "24942", "33925", "47199", "28804", "57092", "37892", "53994", "38613", "22507", "10303", "49803", "38256", "28426", "18972", "50459", "42213", "54586", "36805", "43169", "20423", "37656", "18836", "59869", "22172", "20886", "46358", "34771", "31328", "37815", "49275", "32955", "14757", "56437", "34168", "17364", "59196", "12359", "37218", "24318", "53786", "45767", "6804", "55241", "32232", "31623", "42076", "27381", "4096", "23499", "2657", "49531", "16669", "33188", "51016", "24133", "50939", "13181", "45795", "31684", "49069", "45494", "16196", "5511", "44404", "33493", "40602", "9016", "21611", "44630", "47012", "16446", "19604", "24029", "54942", "27237", "48054", "46762", "46710", "37046", "30549", "32677", "27658", "6056", "22160", "16477", "41233", "8880", "11885", "28158", "27976", "2884", "56976", "12336", "53946", "57326", "53474", "59687", "43555", "36444", "39326", "44001", "43925", "28592", "17522", "642", "21177", "35451", "48889", "17807", "15260", "27323", "17720", "10414", "57353", "9781", "21078", "35125", "13749", "675", "55508", "21998", "42420", "2427", "21020", "39323", "40651", "34288", "47471", "880", "59766", "29752", "45838", "35726", "56008", "55081", "7706", "50358", "19826", "34371", "18843", "38296", "32263", "36764", "50960", "53096", "52765", "35269", "15287", "26347", "10745", "5459", "3281", "42052", "35270", "7739", "50846", "36220", "50999", "8204", "2847", "24297", "33359", "16786", "1767", "69", "39803", "58313", "14543", "273", "53473", "50202", "25113", "23435", "50278", "22860", "39446", "44783", "47607", "39647", "42623", "49453", "42888", "59022", "30633", "10075", "14914", "43877", "4094", "22175", "51954", "7048", "15714", "50145", "54153", "24691", "54239", "51027", "29856", "18115", "21214", "17186", "11374", "6231", "45197", "49353", "38721", "30672", "32539", "14341", "45943", "19747", "21423", "38724", "55942", "14069", "43760", "51580", "16053", "49113", "3689", "38091", "34840", "37299", "20816", "43276", "42695", "30060", "21604", "3633", "57562", "8328", "15976", "2098", "19131", "8043", "4220", "44860", "4609", "7778", "51003", "16529", "47602", "50531", "50974", "23487", "45784", "47565", "8437", "38991", "34345", "915", "43432", "37946", "2218", "5646", "1250", "37153", "24943", "57961", "42832", "10089", "340", "12039", "46315", "28549", "58411", "13992", "57651", "19319", "6929", "41671", "33593", "20184", "48548", "58190", "35516", "27844", "29010", "23416", "39576", "8569", "8337", "43190", "4283", "59137", "8828", "21924", "27868", "31179", "3981", "10680", "1788", "18791", "28281", "3623", "11610", "9397", "28278", "55540", "32947", "30084", "47513", "6506", "32631", "58559", "38514", "44942", "9641", "41382", "13877", "52434", "41015", "34540", "18337", "41522", "26899", "5188", "54719", "1047", "13427", "30898", "14728", "48740", "5225", "33716", "23693", "12001", "19503", "40867", "37437", "50024", "51232", "25702", "31086", "32593", "34198", "6859", "3900", "28889", "15911", "51276", "5644", "49243", "27340", "51086", "50714", "41083", "52480", "33270", "36488", "11771", "59483", "27357", "32198", "36961", "20820", "13879", "21615", "56621", "11177", "33823", "35879", "21956", "55831", "29296", "58285", "48162", "12187", "54514", "31372", "47873", "39788", "16870", "58984", "43388", "2669", "49806", "3344", "20731", "51924", "42237", "2905", "49036", "38050", "45736", "52283", "49086", "29903", "27900", "17534", "57556", "53466", "35505", "9898", "12277", "58490", "48263", "15870", "8901", "53483", "401", "44786", "17342", "42736", "58875", "18364", "44377", "12687", "47866", "37096", "55042", "16980", "22663", "28972", "39493", "25707", "16955", "49362", "28558", "16028", "57729", "52576", "56285", "28142", "52995", "20407", "55753", "13245", "51434", "39085", "14766", "18865", "55909", "7840", "22967", "44653", "57198", "45396", "30006", "51533", "58867", "31646", "58167", "22857", "21035", "39340", "42262", "36377", "9815", "16726", "35394", "30324", "43280", "27362", "59135", "46313", "33613", "58245", "29902", "27304", "11296", "51004", "50398", "34110", "13888", "50660", "19737", "20428", "58492", "40711", "32257", "43153", "7563", "42603", "23970", "15532", "9605", "9193", "36093", "53747", "3772", "52667", "39058", "9358", "40662", "38348", "37963", "11063", "42662", "27409", "50931", "59404", "38551", "28870", "18935", "38762", "9460", "1259", "1437", "14290", "28229", "36359", "26523", "16298", "23003", "13175", "49456", "54765", "43259", "23990", "9286", "44155", "56085", "45992", "32378", "42900", "23222", "59424", "58562", "35996", "50984", "54306", "47310", "15559", "50249", "33125", "440", "41873", "58272", "32082", "36670", "1310", "59840", "24606", "14469", "4101", "4560", "34988", "41458", "26857", "12388", "40440", "48217", "2601", "24012", "43450", "25101", "21538", "24556", "55982", "4042", "23778", "24255", "50966", "11706", "2088", "28233", "51198", "58537", "3360", "30788", "58132", "25016", "36294", "20915", "48673", "15351", "38095", "35113", "3330", "49887", "21809", "27178", "33263", "44574", "16514", "8716", "41543", "9015", "23795", "48183", "25936", "4747", "2735", "50404", "43801", "20932", "10492", "56906", "12211", "1383", "52125", "47478", "19086", "36346", "4644", "30021", "17214", "3321", "54470", "18102", "34298", "36047", "28977", "28518", "8954", "35707", "38472", "59061", "8618", "51274", "51392", "48387", "20940", "47237", "42071", "30874", "10055", "5438", "12224", "39336", "12536", "39923", "23945", "22365", "7776", "22432", "17353", "15442", "33776", "51572", "32712", "54042", "9366", "48326", "42545", "49665", "46307", "33261", "371", "53726", "28646", "49926", "56644", "28595", "26216", "24405", "17632", "50333", "39074", "24177", "19373", "12393", "50224", "21696", "7332", "10682", "19275", "33745", "34867", "26422", "58781", "970", "20089", "33873", "19099", "57437", "18665", "26950", "5120", "28930", "35266", "38381", "58214", "9772", "5755", "7529", "19831", "47275", "11551", "1333", "42205", "50337", "39752", "16451", "17437", "164", "34473", "53540", "48277", "8073", "46314", "22446", "44564", "40076", "50967", "49181", "18126", "13142", "2276", "48748", "34938", "6691", "13128", "40389", "46481", "499", "54509", "22067", "53091", "46753", "21915", "17723", "2097", "47470", "16112", "3172", "37746", "13458", "16347", "28981", "18271", "17309", "23998", "47392", "30682", "20888", "33382", "33892", "53620", "43658", "43635", "10624", "49428", "33608", "34490", "34538", "55611", "30199", "14465", "18619", "20447", "5905", "12372", "46285", "18426", "23444", "34747", "21533", "48669", "2691", "52841", "4792", "3959", "18769", "45048", "2575", "22119", "48246", "47190", "18465", "48620", "24875", "23570", "21478", "42371", "11576", "50131", "41701", "37877", "37467", "21654", "39641", "17713", "13948", "48030", "5771", "52245", "10775", "7772", "6520", "35713", "38654", "12590", "47906", "26804", "16395", "11887", "57803", "54167", "48979", "52534", "12321", "14599", "40037", "16369", "13494", "13700", "29216", "18785", "25850", "46079", "54391", "12225", "52045", "13146", "50163", "33784", "17635", "2917", "49694", "54043", "30964", "18633", "47227", "21190", "41472", "50483", "28417", "34737", "26981", "56126", "56391", "12294", "29737", "39048", "36925", "45087", "33555", "28538", "2444", "40968", "51254", "20714", "57598", "15881", "32581", "22677", "8022", "50305", "28836", "57888", "24602", "47651", "4177", "50575", "23071", "58507", "863", "7066", "26187", "42990", "38478", "7431", "10193", "59272", "38076", "26659", "39278", "23825", "55520", "6503", "51160", "54041", "7937", "8160", "8187", "24701", "8613", "2944", "4232", "7017", "36135", "11373", "52784", "49690", "24378", "298", "31920", "17124", "12630", "25081", "44539", "9737", "14128", "37362", "47033", "39440", "10998", "47019", "26389", "32398", "45486", "40975", "43440", "20712", "40987", "44990", "26015", "56708", "11251", "52535", "6823", "15347", "5005", "2018", "57186", "4364", "38887", "26518", "14125", "11378", "12472", "26580", "58077", "19382", "54446", "176", "13726", "48681", "28723", "39122", "4051", "32910", "59407", "47758", "18498", "57535", "51826", "38006", "44171", "16997", "50794", "42067", "2585", "3947", "35919", "27786", "1965", "55191", "37617", "37025", "24685", "9949", "17868", "38831", "55832", "38156", "48618", "58803", "1983", "10114", "44898", "45596", "41590", "58897", "16282", "7714", "16636", "41720", "31884", "31808", "27288", "4853", "27557", "13703", "5674", "42096", "31634", "10651", "54049", "46797", "28163", "37302", "14979", "41973", "1805", "16950", "22179", "24472", "16274", "59364", "26807", "14605", "28337", "50618", "23580", "8511", "55513", "20914", "3109", "26174", "50705", "51130", "35640", "4809", "55851", "58302", "11799", "34376", "16847", "55374", "35646", "56510", "18308", "31122", "33210", "7039", "11654", "5251", "12960", "45684", "45009", "13386", "26376", "8713", "38048", "19073", "51323", "48750", "11906", "120", "58611", "11055", "9730", "33902", "15399", "45507", "20868", "48257", "32697", "27432", "10517", "4751", "13226", "56414", "33224", "29699", "50005", "8798", "36308", "33963", "4069", "41554", "8148", "4345", "58136", "23504", "59978", "13107", "34388", "55201", "8070", "48809", "23787", "21439", "25481", "48337", "44253", "33319", "53100", "36684", "45686", "51461", "32498", "8646", "2367", "35051", "55836", "876", "4081", "18345", "39399", "40429", "48224", "54970", "43531", "27135", "43645", "206", "23870", "43317", "38682", "7975", "8550", "11056", "24276", "54919", "7427", "13062", "6033", "36669", "33952", "53548", "575", "48007", "57710", "43778", "37463", "38637", "21413", "54270", "33198", "59180", "32138", "55777", "14430", "33014", "800", "45258", "21876", "36833", "39763", "5744", "14760", "56408", "33699", "53425", "42166", "51927", "11202", "51345", "37418", "1664", "32977", "42721", "54603", "2254", "55108", "30288", "49258", "17753", "46993", "18688", "44109", "52016", "38785", "50304", "55038", "16653", "46898", "49221", "16421", "46122", "59879", "10868", "33168", "7861", "32683", "3678", "41791", "22394", "23896", "4238", "21975", "32954", "47760", "15602", "32793", "27695", "35653", "31020", "48273", "20900", "13034", "36437", "10412", "52308", "22196", "13890", "47379", "10052", "43177", "20839", "46696", "51597", "51370", "45165", "75", "9645", "37911", "38460", "9778", "57853", "31428", "8261", "13761", "3310", "574", "24819", "3393", "33480", "2808", "57679", "17397", "45027", "25049", "7263", "34409", "7981", "11974", "40100", "30249", "25983", "6593", "24649", "22467", "14198", "52352", "37361", "53086", "32632", "29863", "41790", "51691", "20789", "37204", "4650", "40022", "13732", "18907", "52523", "4649", "5765", "9098", "48266", "32748", "46549", "59099", "44016", "50970", "31933", "4573", "13306", "48097", "6783", "58519", "52838", "9768", "55665", "32004", "58831", "32172", "10890", "39195", "59438", "37127", "4325", "2494", "40038", "54606", "3237", "51499", "54539", "21046", "8666", "36282", "54867", "16261", "10767", "18413", "29448", "44368", "54401", "52046", "11284", "18201", "168", "3609", "13267", "48700", "4224", "32535", "47856", "4833", "20919", "36900", "1635", "24054", "12120", "29957", "54790", "33053", "50332", "7583", "14933", "42953", "45192", "47734", "19724", "31417", "11141", "33387", "47330", "34533", "2283", "12787", "59849", "5961", "56155", "50940", "27794", "43426", "34212", "56844", "29605", "1365", "2298", "14287", "44923", "49931", "56247", "48118", "58661", "47352", "50708", "46002", "58198", "30737", "35231", "46348", "22655", "10882", "55274", "58623", "22808", "10939", "24666", "45365", "12108", "8150", "25524", "10421", "5634", "28801", "10111", "49168", "4861", "26206", "35479", "39728", "37660", "22622", "28605", "37875", "16738", "16864", "48538", "15677", "48168", "24672", "45579", "32362", "28220", "14219", "55213", "48186", "56850", "44831", "13608", "31997", "37055", "42817", "48646", "21123", "56447", "36909", "46334", "43429", "49417", "58376", "44772", "3692", "56557", "25868", "30209", "13708", "16930", "56960", "10871", "50602", "31766", "32260", "53306", "52909", "49434", "18784", "46214", "56131", "30762", "19088", "50491", "42750", "45243", "42659", "11332", "11110", "21867", "59238", "55121", "55049", "14965", "40664", "39985", "40488", "59679", "30924", "43923", "2457", "6870", "35330", "25859", "54013", "50300", "2237", "7744", "57720", "28089", "13872", "12990", "13239", "53228", "3575", "891", "13059", "33369", "55643", "49834", "54712", "31716", "59896", "59589", "50296", "4474", "18219", "26962", "57953", "4661", "21207", "49596", "56326", "882", "56774", "19693", "43633", "10440", "57106", "59797", "55297", "47610", "38028", "18654", "8836", "18698", "54833", "19905", "4703", "48940", "17028", "16427", "37805", "24901", "9650", "9894", "40959", "32916", "53041", "47057", "53905", "48543", "21657", "50715", "19667", "52311", "41478", "41607", "23164", "1374", "35036", "22456", "34022", "48569", "36061", "12007", "46038", "10940", "39071", "52472", "57766", "23109", "37971", "19518", "14302", "49332", "17718", "55684", "1360", "18378", "50954", "41345", "21514", "5727", "6069", "54421", "51337", "40658", "21280", "32766", "2303", "57900", "32767", "31058", "4209", "48716", "31447", "47137", "52359", "51258", "52641", "2742", "37584", "6244", "59506", "10819", "38174", "22246", "37544", "10588", "19519", "10103", "1889", "51808", "16479", "55660", "7860", "16130", "43594", "14284", "9149", "3296", "35834", "40655", "45916", "4710", "53326", "21909", "31874", "10575", "20753", "59941", "58674", "3040", "25837", "15719", "51650", "45469", "25543", "42582", "10093", "13204", "32550", "14941", "25119", "50233", "58660", "6745", "52319", "31418", "18309", "17592", "43834", "47773", "53914", "31980", "4547", "59712", "17630", "2120", "57273", "12646", "21808", "23978", "5311", "51175", "16090", "47419", "17833", "11856", "21726", "57016", "11170", "27817", "15650", "43064", "45500", "10252", "22498", "33428", "31348", "44275", "2622", "7298", "32660", "43045", "17356", "41173", "56460", "11027", "2053", "55134", "54565", "7445", "16643", "20947", "57906", "33500", "3835", "36019", "13693", "53320", "40337", "54985", "31773", "21567", "1417", "32491", "27602", "14706", "32480", "18122", "17598", "36158", "16044", "16115", "1377", "52253", "26312", "54289", "31138", "43831", "18827", "35334", "30341", "28406", "43002", "59239", "20808", "2779", "47626", "25963", "44931", "39581", "40435", "34850", "4268", "37364", "3411", "22197", "44101", "31505", "3113", "49848", "36154", "46864", "15319", "17649", "6169", "36490", "27566", "39028", "58105", "5320", "2832", "3687", "15062", "30616", "9846", "13018", "45537", "875", "31804", "52521", "28563", "45710", "11317", "5964", "50269", "21798", "57223", "49298", "52672", "7476", "44105", "53012", "12203", "10831", "24154", "16266", "13667", "36794", "11767", "45719", "6781", "26890", "15740", "7754", "8853", "42468", "7398", "40858", "44548", "2395", "51946", "8173", "55907", "44722", "34337", "44332", "40083", "2561", "34502", "19572", "36265", "16160", "19799", "54771", "31390", "44205", "21199", "49203", "2311", "51070", "478", "48213", "44843", "37793", "21094", "45178", "48953", "40094", "47685", "29188", "43030", "40876", "832", "10738", "56063", "41429", "2658", "14822", "23134", "38732", "1107", "29668", "45275", "25949", "9812", "21082", "4328", "9625", "19699", "25855", "55062", "40949", "10833", "40572", "27120", "10958", "11883", "37629", "47853", "5328", "2901", "22937", "8579", "44543", "5290", "59979", "11915", "45923", "32668", "55063", "42117", "58051", "1381", "8293", "16352", "5869", "37658", "21621", "22769", "42070", "19849", "27097", "2556", "7567", "35292", "12156", "29563", "11954", "50689", "57584", "55116", "53607", "2418", "697", "39607", "29974", "59532", "49305", "39282", "53629", "50920", "48221", "29077", "55716", "6197", "44224", "33363", "50716", "38647", "522", "3168", "23200", "48360", "30526", "25185", "59161", "55857", "14098", "27226", "5935", "56678", "5407", "37733", "55323", "49138", "39115", "5741", "46363", "43756", "7811", "42181", "43054", "41738", "19220", "24121", "52771", "56904", "33293", "35584", "31238", "22604", "51735", "32182", "33815", "22606", "18242", "45516", "27489", "49528", "49548", "31916", "48535", "10143", "27172", "51368", "57062", "24101", "49725", "54766", "18659", "28270", "6697", "29452", "23685", "34740", "18240", "9487", "24409", "1893", "43970", "51495", "53236", "1238", "51451", "25090", "47980", "36000", "5758", "2678", "52719", "52891", "57831", "25848", "14861", "2587", "1617", "177", "59203", "37024", "498", "1148", "11270", "3529", "48403", "29517", "17022", "46126", "15190", "6046", "29324", "39334", "3044", "10061", "49034", "13499", "49412", "2826", "15332", "16216", "19902", "45398", "41664", "29228", "15193", "56738", "20903", "31344", "33010", "21173", "54249", "51277", "2975", "5259", "32504", "7123", "44023", "54831", "25472", "7968", "7134", "14122", "13439", "35843", "32728", "18902", "54117", "42272", "44551", "13112", "51789", "58310", "33541", "58048", "9335", "16914", "25238", "52942", "44556", "23062", "54797", "28781", "39993", "18897", "20147", "24507", "33026", "24753", "11434", "10027", "26506", "58807", "59729", "37549", "7699", "36715", "56320", "11065", "59864", "19243", "29260", "50971", "54768", "27605", "51313", "51893", "28693", "54431", "58038", "57465", "44036", "44830", "41819", "4338", "41961", "59393", "10718", "19186", "22189", "17025", "35157", "14650", "33506", "55434", "33770", "23210", "4794", "14701", "16518", "52551", "47537", "27498", "54957", "46336", "59553", "36623", "11306", "22685", "53238", "4900", "38202", "38535", "7205", "20613", "26493", "5011", "42118", "15810", "46208", "17828", "54679", "33280", "12523", "40541", "15132", "7053", "6514", "11978", "6810", "24376", "34165", "23637", "54553", "47240", "1336", "47984", "14498", "5070", "2624", "12966", "48154", "40127", "25214", "43266", "41661", "16139", "46120", "52715", "40506", "9010", "50740", "43083", "45790", "57850", "16606", "10154", "58169", "18804", "15065", "28250", "20282", "36438", "17590", "39892", "36721", "3210", "2224", "10284", "40278", "27185", "21373", "44187", "57457", "55724", "23817", "41221", "21124", "57994", "54483", "9064", "25877", "34111", "24946", "56130", "22489", "9577", "38215", "25194", "47700", "7667", "33625", "51524", "14242", "28799", "41884", "36735", "28773", "33968", "26991", "3471", "30684", "28214", "36857", "39944", "44311", "19146", "26120", "12259", "16910", "45864", "54364", "46603", "7891", "5016", "38354", "30481", "24577", "12489", "6798", "55701", "55482", "22627", "56237", "37720", "42177", "55880", "26226", "39600", "36537", "33798", "20119", "21679", "8940", "31133", "6095", "10650", "54581", "33539", "55792", "8606", "52884", "25915", "26772", "46578", "31398", "369", "46985", "47094", "5860", "22083", "48898", "42218", "16250", "173", "18179", "46410", "12204", "44975", "36316", "11965", "22372", "42605", "37879", "50027", "11957", "27713", "34047", "37414", "48434", "7465", "26359", "34233", "40340", "2520", "24816", "35431", "58333", "11742", "33094", "12738", "18406", "1611", "18225", "13847", "21121", "53344", "29364", "42025", "39049", "38593", "43321", "16268", "5625", "59227", "28656", "11433", "10977", "23801", "9556", "23400", "14326", "11464", "20603", "29697", "47499", "21995", "41049", "5116", "7542", "36555", "18661", "31592", "58989", "38692", "25262", "42701", "42139", "36687", "54635", "34828", "6989", "23259", "55204", "19623", "56802", "44048", "27727", "16415", "48405", "2664", "15838", "13168", "30926", "25892", "27610", "56795", "44494", "22645", "29840", "7", "12942", "50733", "43835", "2988", "25129", "18496", "16113", "39293", "26783", "57010", "10109", "51014", "41338", "56339", "12253", "23762", "10250", "5608", "10019", "2852", "22270", "16202", "33904", "8306", "8162", "21345", "21701", "18288", "11935", "50047", "43665", "1186", "4828", "19414", "17292", "59452", "34058", "24396", "37563", "49616", "59390", "54389", "10681", "6778", "8396", "19309", "41246", "48486", "11535", "4926", "47503", "10487", "37785", "35140", "44582", "192", "19541", "30027", "46517", "11385", "51312", "59156", "973", "1743", "52322", "24512", "25989", "54185", "53716", "22517", "54228", "51816", "42849", "56135", "1252", "12631", "58154", "49445", "20274", "59045", "40571", "6433", "19539", "23572", "15992", "31989", "28301", "40360", "280", "39953", "56433", "44913", "33993", "55046", "15213", "23634", "20991", "7415", "4119", "23294", "45662", "21343", "30237", "20929", "31149", "27335", "36125", "23256", "27854", "48570", "39706", "29975", "17128", "55985", "36795", "52921", "89", "21443", "42443", "39579", "44086", "3546", "27777", "47476", "36197", "26096", "54571", "17154", "15955", "21754", "11488", "187", "44054", "37417", "45367", "42119", "44079", "33037", "6149", "59504", "27641", "28435", "52539", "14708", "27675", "53239", "21008", "26654", "24544", "18215", "21341", "24923", "7758", "45989", "8476", "32545", "27933", "20376", "29351", "7064", "10359", "13090", "24629", "49861", "47236", "57410", "11621", "11645", "32819", "13527", "22726", "58566", "16915", "1443", "19394", "29366", "27036", "18170", "18110", "45565", "29521", "15901", "32094", "6341", "4621", "49699", "10860", "58187", "5421", "52029", "19147", "59013", "32814", "51698", "27997", "49179", "5417", "46393", "38121", "58657", "14657", "11153", "54576", "43662", "3052", "3648", "20717", "19843", "12290", "19697", "51761", "19417", "20719", "56322", "3069", "48289", "46216", "27636", "21855", "32354", "32686", "35178", "32773", "42029", "8506", "5229", "46817", "17897", "39605", "12062", "4730", "50588", "22873", "16428", "24565", "47650", "12496", "11733", "38203", "32139", "20660", "49342", "56791", "19170", "21125", "46567", "50229", "9019", "19036", "14155", "49526", "4595", "37111", "22321", "55136", "26809", "49586", "9353", "3906", "50217", "9399", "38576", "24506", "9510", "23929", "54846", "4005", "43265", "52614", "46974", "51304", "18320", "2762", "41571", "33762", "28515", "4082", "57442", "6041", "506", "44122", "13790", "12243", "30742", "19291", "14477", "56388", "17655", "15192", "32276", "12244", "4612", "20474", "40456", "28125", "12592", "5780", "10240", "35681", "4031", "25021", "28613", "6017", "45734", "51059", "13374", "39919", "14334", "4907", "56250", "32248", "50826", "56399", "23077", "2895", "36881", "51133", "29182", "25669", "14024", "11741", "35389", "14250", "51111", "30293", "51801", "59539", "24593", "53572", "24126", "33204", "34116", "54724", "16956", "43080", "51771", "46782", "7915", "47733", "42201", "40577", "42145", "45373", "17457", "1369", "8446", "36793", "57140", "57544", "57507", "1924", "56957", "42683", "54686", "35059", "57533", "28535", "17645", "15797", "18020", "12676", "41387", "33237", "11330", "27249", "28110", "86", "31387", "13696", "2380", "19356", "29781", "13689", "31014", "34098", "45463", "21526", "23101", "2078", "48731", "28584", "58741", "562", "33029", "22995", "24682", "7352", "54206", "59690", "789", "50032", "46299", "50252", "8639", "30161", "56422", "4241", "19884", "11250", "34559", "53286", "24056", "21596", "33788", "35469", "50619", "29404", "24872", "30492", "16611", "58143", "27256", "6016", "35923", "26903", "57497", "5018", "14832", "46150", "35506", "9156", "36713", "41223", "58581", "22862", "51442", "4373", "30699", "47108", "6676", "49064", "43656", "12641", "22094", "20451", "47620", "36087", "15135", "48896", "27526", "26146", "48756", "28237", "23480", "16698", "40739", "57446", "32563", "39562", "13235", "42469", "47052", "18456", "23370", "11079", "5282", "4337", "44856", "37441", "37559", "58359", "11810", "22821", "19768", "9394", "24962", "40528", "1434", "52590", "51658", "29698", "49518", "22278", "58147", "50159", "4458", "58636", "8146", "15776", "30667", "35911", "29843", "33598", "42655", "7275", "34133", "47396", "24033", "31711", "24143", "20755", "3313", "44805", "36020", "28943", "11469", "59089", "29575", "4430", "44925", "36573", "58175", "9822", "27992", "15640", "59570", "54668", "31520", "59189", "30900", "33386", "56075", "42757", "51866", "22833", "19035", "274", "4544", "42144", "5491", "36818", "12065", "23349", "24192", "15183", "22411", "37490", "31789", "5920", "34812", "16873", "3550", "19531", "14309", "8341", "6183", "11360", "51922", "35473", "26697", "59358", "41609", "54631", "54120", "31759", "30487", "4416", "7680", "19981", "306", "37029", "42599", "54729", "20561", "44509", "25529", "6501", "4144", "16810", "9478", "21665", "30723", "58164", "33518", "14183", "21093", "38541", "44615", "45589", "32189", "39359", "27469", "2071", "1518", "32557", "55219", "29808", "57674", "10295", "40363", "21499", "6361", "37611", "21800", "2680", "1142", "45399", "18928", "51659", "8842", "58457", "3303", "23078", "23473", "29779", "24906", "55111", "19212", "37138", "10524", "29585", "8532", "51942", "13538", "42808", "57670", "20175", "51409", "38529", "11208", "59600", "1441", "32765", "36453", "55402", "5327", "35987", "50184", "33103", "49112", "57174", "2964", "56567", "43448", "31241", "24859", "28484", "50164", "56935", "29248", "51252", "5394", "53181", "10132", "16658", "10477", "55572", "48571", "49439", "35716", "33646", "16301", "34738", "8301", "38786", "11874", "363", "20148", "40445", "40179", "52468", "51592", "37022", "24634", "1811", "33227", "14060", "422", "35220", "52241", "25973", "30875", "16213", "6349", "51330", "29547", "23153", "35561", "52231", "31596", "9442", "9104", "45999", "36811", "372", "27124", "51768", "53175", "45530", "31085", "59018", "24059", "52288", "8158", "45487", "22263", "59901", "2894", "37769", "18524", "38534", "989", "3755", "4499", "53252", "28086", "41371", "36431", "42308", "12178", "30504", "136", "38584", "23532", "35958", "3639", "2972", "43456", "4771", "8155", "51417", "44774", "47530", "35594", "4057", "25751", "33445", "16536", "18365", "14097", "58026", "7412", "36226", "52454", "2504", "32460", "57165", "2748", "54548", "14514", "11596", "48202", "42028", "57228", "25430", "36780", "5147", "14345", "42891", "20859", "28894", "10059", "40005", "49743", "20000", "6", "50170", "17525", "6457", "31574", "5102", "37795", "39065", "59836", "19632", "34728", "42326", "9012", "43055", "5219", "22087", "5897", "44388", "32335", "22811", "7503", "38382", "49635", "29900", "8855", "3203", "23766", "41282", "26200", "7815", "24696", "34130", "215", "59456", "13643", "965", "25796", "46842", "42825", "33659", "16412", "21804", "1584", "59023", "59216", "35969", "55075", "1390", "35101", "48010", "480", "28242", "15308", "24468", "6210", "53573", "15795", "34609", "19040", "33000", "6629", "4008", "58074", "57374", "14447", "56733", "26082", "53719", "13126", "25724", "19276", "2773", "59983", "2321", "516", "39729", "54188", "18325", "6001", "10542", "270", "24874", "9250", "53204", "29367", "11753", "10196", "10911", "19497", "39925", "47350", "24919", "9391", "38034", "44618", "4742", "2874", "8640", "36718", "34226", "34677", "48116", "4799", "12299", "27025", "45779", "42112", "13326", "52720", "49281", "2611", "55677", "20873", "19242", "45778", "34995", "56053", "48872", "58367", "32999", "721", "27851", "53324", "47684", "38563", "4322", "14340", "18331", "11291", "1801", "53242", "34879", "13745", "24009", "3702", "15573", "40023", "41432", "8169", "38517", "21228", "9004", "5663", "43156", "57084", "31139", "35609", "49119", "4222", "54447", "26552", "14017", "44704", "279", "37017", "57385", "34129", "37149", "19262", "49347", "53507", "13584", "15120", "38799", "8065", "3387", "57727", "12420", "29838", "40185", "39470", "27181", "39057", "14732", "43106", "51500", "14957", "5046", "17855", "51238", "33156", "35721", "23291", "41603", "14355", "18776", "37425", "25620", "1858", "30691", "32365", "37090", "47479", "5665", "46602", "31703", "49873", "40053", "49984", "59044", "39846", "42406", "43945", "48274", "21209", "34348", "29604", "54125", "6370", "40721", "289", "15637", "43287", "18017", "8036", "56987", "55736", "16363", "5098", "57912", "22033", "4865", "32569", "41953", "52922", "42948", "21045", "14156", "36725", "53990", "38455", "47768", "21728", "42044", "18458", "37595", "22520", "42840", "32702", "43174", "51166", "44737", "47595", "20213", "34413", "48091", "25420", "37894", "45604", "18879", "58759", "7927", "37492", "46201", "10971", "38413", "2867", "56745", "11750", "48028", "32107", "1476", "6436", "25345", "9211", "59773", "31282", "4765", "47043", "7592", "14358", "15014", "45868", "26733", "32240", "13356", "56612", "19597", "41634", "13083", "58993", "30220", "42389", "10300", "50425", "37401", "21488", "30650", "59615", "54063", "22368", "18764", "59672", "42115", "47769", "19759", "41770", "38377", "1214", "11305", "17746", "35656", "16943", "33910", "54116", "44784", "47692", "37749", "50649", "42758", "5195", "56854", "6589", "1574", "35786", "37352", "58425", "39950", "53418", "10403", "36837", "28496", "45685", "26044", "44438", "14530", "5509", "2008", "59048", "20597", "32842", "11131", "53187", "48900", "13400", "34998", "59857", "39676", "18369", "16709", "14375", "30674", "52282", "3284", "56356", "45417", "53471", "6768", "11971", "13909", "4250", "33678", "31290", "8242", "386", "20598", "55491", "11580", "59131", "52371", "14509", "4758", "24491", "56119", "5458", "38738", "28522", "12594", "19710", "58070", "39947", "5969", "41923", "4784", "19433", "41357", "42611", "33266", "16229", "49477", "25364", "11543", "15832", "33075", "29686", "1364", "22894", "4616", "31029", "51328", "13969", "26620", "33589", "19553", "3737", "1436", "15904", "22118", "13737", "761", "16975", "21917", "8422", "24057", "48313", "14855", "59502", "50156", "41792", "1281", "50654", "9137", "27261", "52906", "1863", "2308", "17546", "13077", "50090", "49754", "51355", "5657", "26324", "28683", "14577", "7550", "44605", "170", "26110", "13426", "12724", "13850", "42898", "29377", "47907", "24348", "26994", "11783", "42775", "26355", "13158", "9197", "55034", "32340", "42915", "10372", "40033", "13984", "13382", "52824", "4230", "36473", "36499", "123", "20160", "38060", "21290", "39996", "18428", "23617", "3105", "55243", "54904", "24386", "29177", "29002", "4464", "22521", "27145", "8578", "17970", "30321", "26765", "48521", "57299", "21323", "42726", "42521", "12995", "40188", "16060", "38726", "23192", "46450", "20562", "7803", "25942", "15434", "20420", "42378", "14931", "13317", "27797", "13403", "36140", "18685", "8990", "3083", "37828", "48914", "38526", "34220", "43129", "29720", "45140", "48945", "56357", "43573", "45739", "34638", "49336", "8682", "31004", "57615", "52031", "16010", "24233", "29932", "54406", "29197", "1996", "59546", "58832", "58104", "11302", "11472", "35296", "45318", "16678", "42313", "12916", "15415", "23274", "34872", "30622", "30950", "3747", "40754", "1271", "34639", "40347", "17681", "30470", "48489", "47627", "47383", "39588", "16936", "31412", "39387", "55604", "8542", "19055", "28482", "59817", "345", "11189", "6325", "52729", "15243", "15002", "10034", "5803", "7341", "30590", "53541", "39253", "56411", "4914", "48051", "10769", "8734", "50091", "11350", "5419", "58418", "41687", "6806", "31307", "43053", "15458", "51550", "21474", "32604", "39508", "32838", "40660", "22155", "4787", "27302", "46364", "20399", "26694", "39975", "52708", "23733", "58124", "31206", "31903", "57213", "12356", "4026", "54009", "43634", "40618", "12926", "46303", "35597", "16313", "28731", "46862", "52117", "39370", "12405", "2485", "22223", "82", "45297", "21404", "28599", "18518", "40234", "13773", "2108", "22578", "31992", "58468", "40828", "56986", "31530", "26225", "41897", "3507", "32969", "7462", "46659", "10756", "35210", "59065", "52904", "56622", "13795", "36621", "12504", "15259", "26916", "3246", "52955", "32846", "15456", "26918", "55232", "35343", "50079", "48607", "23338", "57361", "3884", "4807", "49139", "36089", "32243", "15206", "37577", "11682", "6573", "20074", "47761", "8115", "42397", "59694", "24950", "24035", "51362", "46649", "48776", "20083", "35893", "56901", "48038", "43896", "25050", "46105", "3271", "28966", "59824", "5563", "10179", "38405", "47123", "8406", "24327", "18992", "16665", "13786", "44266", "8014", "42937", "59047", "49427", "10329", "57768", "18065", "10272", "43777", "52816", "9998", "21520", "57281", "5043", "108", "39328", "8650", "22889", "48879", "20187", "58160", "25378", "43188", "1544", "6342", "47126", "56748", "57408", "9557", "7833", "44281", "12604", "18280", "59778", "11877", "49757", "43409", "6513", "14102", "52027", "37522", "48415", "21891", "45129", "2085", "31192", "39601", "43132", "866", "38955", "58986", "24250", "44973", "50319", "34244", "28597", "48431", "9210", "51781", "45325", "11668", "55956", "30784", "42679", "57964", "23941", "47374", "1027", "34287", "58283", "55787", "1423", "26785", "10495", "56767", "14160", "15734", "43180", "31467", "24804", "19764", "1273", "34437", "46139", "28504", "8275", "1628", "55578", "6000", "59582", "20138", "23135", "4472", "21819", "21320", "7350", "25511", "57381", "30404", "56470", "37657", "3622", "28946", "32064", "19214", "7107", "56977", "22762", "36545", "51640", "20220", "2595", "53765", "2670", "12374", "14639", "48997", "44757", "49227", "46161", "5462", "40608", "25608", "42446", "57825", "24853", "48396", "28025", "44812", "41568", "35441", "19778", "21893", "39150", "45764", "53897", "6499", "42814", "26188", "16423", "16237", "48847", "44853", "21493", "7009", "8568", "21958", "5587", "42749", "16561", "20042", "2374", "1253", "12663", "26285", "34722", "27050", "38130", "32958", "36896", "50144", "55220", "4847", "51870", "52710", "45924", "34175", "17966", "56474", "47406", "22852", "1695", "24151", "328", "35249", "51625", "26926", "19317", "33144", "19056", "686", "18914", "21707", "48227", "13339", "8754", "12729", "14994", "58690", "16339", "56383", "48955", "2319", "3013", "53814", "44339", "19645", "48534", "57979", "50900", "43655", "46738", "32007", "13823", "11428", "3282", "20254", "42674", "48581", "4604", "2802", "30766", "55161", "57998", "2957", "29499", "32135", "6463", "17079", "39090", "1521", "34158", "46954", "8003", "53443", "6206", "35526", "774", "31774", "56415", "3232", "34262", "46596", "20323", "33684", "45747", "22797", "10894", "57125", "35687", "33852", "59989", "16260", "23171", "2631", "34384", "34644", "38396", "45663", "5948", "9766", "33840", "10528", "8702", "8762", "18474", "15718", "50567", "51631", "53851", "48193", "18790", "42574", "9552", "57869", "37259", "59478", "15272", "28533", "51493", "21753", "23065", "37412", "35075", "54420", "50739", "13174", "59998", "14893", "47356", "12899", "36397", "2602", "54986", "43571", "47250", "44729", "11033", "5877", "3950", "3251", "44807", "16478", "58874", "10294", "28548", "24226", "24488", "54385", "34963", "8801", "38047", "26568", "27829", "52594", "43076", "58544", "56983", "58010", "4497", "53307", "33988", "46544", "23981", "5501", "57626", "44069", "58905", "58062", "46853", "17879", "7535", "30659", "30652", "19844", "53644", "42874", "20738", "37527", "20694", "56353", "12424", "18980", "44567", "8844", "33740", "22759", "13224", "27246", "27311", "19690", "35840", "52588", "55451", "9763", "57225", "17343", "59218", "53830", "24735", "59324", "50816", "30440", "24527", "8851", "42323", "14671", "44983", "3325", "8667", "12994", "47845", "21435", "52618", "17276", "14266", "9061", "11430", "39713", "37612", "21406", "40925", "24229", "57955", "44659", "18693", "40407", "47280", "49259", "45850", "48373", "2006", "56170", "28942", "30008", "38500", "50685", "27147", "45175", "4085", "27303", "18645", "12395", "40313", "28785", "11671", "52075", "20093", "19835", "40893", "50453", "49890", "21215", "25522", "17685", "44584", "12841", "58109", "42110", "33461", "23176", "51352", "57787", "22737", "56132", "15494", "25067", "12439", "16966", "50645", "34903", "28587", "8750", "21440", "41724", "55420", "2580", "20814", "348", "30571", "57612", "4125", "51367", "48233", "35132", "32552", "46783", "32437", "45384", "24422", "5805", "49590", "15412", "24295", "26675", "19951", "37848", "51031", "42870", "37767", "52295", "3966", "38041", "8032", "27221", "34072", "15970", "9201", "8837", "36584", "8782", "2430", "8812", "46162", "49159", "17148", "34278", "4598", "29108", "35459", "9048", "25460", "10637", "58067", "2526", "36946", "32724", "24280", "51799", "23515", "57407", "35068", "39126", "14881", "33935", "23628", "4405", "9836", "46442", "200", "3339", "9663", "39416", "31006", "47547", "1445", "31354", "12653", "35704", "3503", "37296", "36014", "48401", "22397", "23253", "21563", "55056", "23295", "44504", "17087", "33371", "4997", "37253", "37865", "16323", "39013", "8103", "27220", "42250", "15527", "12569", "24713", "54868", "21025", "27852", "10439", "25053", "57375", "48863", "18453", "56560", "58485", "7042", "6376", "51406", "27085", "6224", "48098", "31998", "53776", "57837", "51985", "12298", "34449", "25873", "15849", "17423", "22351", "49825", "139", "41836", "14275", "53913", "44596", "2142", "6075", "3260", "23096", "14161", "36178", "42199", "44496", "29700", "32753", "46065", "49628", "39473", "36695", "17560", "12572", "28264", "30017", "31693", "43102", "56002", "48093", "11832", "45731", "10822", "9223", "29864", "53899", "24042", "2939", "55187", "9938", "54245", "53301", "6740", "10071", "59771", "26219", "7214", "36772", "2345", "55959", "22029", "46057", "57921", "51228", "8495", "1159", "29237", "52032", "47382", "47138", "41046", "29029", "6156", "3699", "59153", "52292", "38333", "16141", "34433", "9987", "11734", "42526", "45452", "17487", "39297", "42459", "54373", "23099", "26233", "9317", "9288", "24198", "5512", "54313", "9913", "51121", "41264", "56428", "56143", "9710", "27270", "27669", "8635", "26978", "29148", "45807", "57126", "32655", "51407", "37397", "35939", "33407", "23290", "24961", "42834", "44919", "497", "59353", "52056", "10416", "16522", "12627", "31194", "36882", "54163", "52772", "8540", "20201", "32438", "33285", "21102", "10425", "10997", "54033", "6389", "57364", "13561", "37159", "10190", "48393", "20583", "51073", "16468", "28893", "6262", "43092", "52356", "37313", "52939", "39135", "11686", "47238", "42002", "40396", "5598", "19543", "10168", "35720", "15333", "40518", "49975", "2286", "37475", "5849", "17977", "24210", "1887", "51526", "56137", "34253", "20862", "10887", "56704", "13604", "24253", "4592", "59208", "33635", "1748", "18014", "12811", "26638", "53876", "4820", "15624", "25323", "10015", "1912", "660", "54582", "47365", "5085", "20784", "12980", "13618", "14745", "39401", "21869", "25250", "45089", "53467", "54168", "24494", "56241", "18284", "58128", "11517", "27393", "39678", "54244", "43522", "53420", "57793", "21847", "17944", "33520", "56936", "39937", "19654", "32345", "46957", "37402", "55812", "10558", "58880", "18492", "44665", "54978", "51303", "256", "25019", "31324", "54941", "18285", "37309", "38868", "47823", "53567", "3887", "7226", "48637", "8119", "54832", "10852", "3087", "59384", "15771", "964", "41404", "2853", "22315", "43830", "40237", "51135", "18125", "1851", "56147", "25634", "35639", "15794", "26859", "41990", "13914", "54981", "57687", "14269", "5114", "3837", "31757", "1792", "29014", "26162", "23914", "52004", "23049", "29516", "38358", "9080", "17403", "7733", "12129", "16330", "39976", "7369", "2499", "35851", "51903", "21041", "20164", "59853", "27131", "3787", "40193", "20684", "47416", "33612", "15352", "56605", "3217", "12849", "20483", "26774", "54503", "32072", "55993", "58396", "55058", "37435", "49639", "28928", "913", "10561", "7903", "50136", "44030", "19809", "22581", "16039", "58145", "59115", "38753", "10056", "26888", "40418", "38575", "52499", "16540", "31852", "42824", "11964", "44050", "51242", "9979", "55728", "14448", "26304", "17731", "33207", "57707", "8600", "54945", "50885", "25462", "31885", "57166", "8783", "29727", "30387", "51745", "9741", "44637", "41211", "6452", "17193", "49588", "48343", "22589", "23320", "27627", "788", "46178", "17493", "3385", "40142", "24720", "29", "9251", "3898", "15603", "15162", "35956", "39780", "1306", "13523", "58628", "51970", "46932", "56625", "26621", "42402", "29511", "22325", "34712", "34805", "7241", "49265", "37223", "57711", "37485", "46653", "42233", "6161", "4983", "59304", "28395", "18423", "8505", "50465", "26589", "43072", "53481", "37806", "58330", "59399", "47003", "51099", "34444", "19546", "21425", "17340", "56350", "47756", "51291", "29023", "47504", "3014", "11192", "37067", "16569", "47833", "31770", "31100", "40768", "27143", "47596", "44903", "37372", "59188", "12256", "57592", "6060", "17888", "42416", "58735", "35181", "24995", "43335", "14427", "58968", "159", "47317", "56044", "45788", "55759", "41793", "20130", "22127", "23080", "33005", "54327", "24798", "12690", "1174", "38063", "57657", "46131", "50724", "58229", "54776", "52690", "22089", "18743", "45037", "49549", "25316", "35106", "53824", "2111", "12525", "46760", "26517", "47405", "19755", "13064", "51357", "26567", "22845", "45810", "9537", "457", "30157", "38826", "4422", "28770", "4777", "46349", "15574", "32511", "15258", "29325", "28481", "38677", "44414", "27", "36448", "35271", "4806", "14482", "49675", "3004", "23836", "30993", "46189", "49928", "10861", "29319", "15459", "21781", "38673", "59935", "20500", "52130", "15962", "59808", "21937", "49070", "9932", "47346", "9999", "15793", "19703", "3807", "47954", "36481", "40737", "29149", "43220", "23073", "10117", "30061", "21558", "29214", "53906", "927", "44292", "2274", "59529", "31539", "27630", "25001", "50514", "4129", "33574", "50806", "30046", "57871", "54229", "41171", "17605", "36708", "37996", "7295", "28384", "47764", "13563", "1315", "51152", "52237", "15804", "59514", "13474", "31824", "15056", "52624", "5045", "51452", "32410", "41104", "11513", "40796", "28686", "6877", "18356", "27931", "15783", "8776", "34492", "55561", "57949", "23852", "5820", "26300", "8967", "43595", "47704", "30805", "0", "23579", "56180", "31983", "49968", "3150", "36901", "38837", "37351", "55950", "5363", "47036", "53870", "32978", "41662", "22795", "8886", "15618", "6319", "53186", "36892", "42973", "59520", "9559", "9380", "28960", "14439", "46107", "45563", "34895", "40017", "1587", "43983", "45728", "29114", "46839", "1457", "10505", "966", "2572", "37902", "41936", "51998", "37619", "33979", "25380", "35789", "42946", "6512", "5400", "8035", "48309", "12827", "25630", "806", "10666", "7065", "44482", "12840", "25525", "1039", "32312", "4998", "13195", "36958", "37646", "47148", "35060", "26407", "24876", "23289", "34469", "58871", "203", "25549", "12987", "6417", "31855", "40710", "59441", "28601", "17143", "52989", "38503", "5049", "14991", "12144", "43579", "592", "12061", "47747", "33724", "12084", "18015", "51593", "44770", "22839", "1980", "2506", "41141", "17902", "15425", "10699", "59657", "1428", "26232", "45004", "52182", "25008", "45929", "8799", "29402", "4563", "18140", "23561", "6456", "16533", "16723", "57264", "27611", "32901", "41831", "36677", "12666", "6038", "29267", "15635", "25563", "13021", "28846", "27259", "43489", "43207", "42821", "48959", "37", "3357", "14998", "32729", "57054", "47811", "5799", "23377", "43533", "53834", "43131", "7393", "58028", "1326", "8675", "14709", "18123", "20615", "16821", "34699", "15758", "53299", "48145", "8827", "21732", "15041", "57908", "38254", "35308", "39835", "12893", "49489", "10230", "31746", "36373", "6725", "46695", "45712", "50666", "58671", "36775", "33439", "28088", "46521", "13531", "1211", "18362", "5153", "57173", "48297", "9806", "20026", "3145", "19552", "55769", "44318", "15699", "31330", "30638", "49899", "45622", "39302", "48767", "46808", "59002", "44252", "44403", "6243", "47677", "27943", "58112", "44067", "2830", "45076", "47717", "51334", "17958", "9490", "23631", "30534", "37163", "18163", "32145", "27218", "33131", "15737", "40778", "16191", "25177", "46478", "17005", "39110", "38111", "53593", "58451", "35448", "9723", "907", "44466", "2091", "53136", "11217", "13570", "26010", "45146", "8248", "28663", "46667", "46032", "19301", "41814", "33318", "36236", "57578", "2060", "29617", "50854", "13133", "34211", "46605", "50582", "59720", "52652", "20496", "44813", "39053", "47559", "44405", "13549", "9083", "38298", "31303", "50419", "32600", "243", "14229", "18447", "376", "53751", "1843", "59", "41493", "34816", "37162", "46019", "44417", "1656", "41802", "48206", "33631", "51719", "53365", "16542", "20488", "29553", "35701", "12984", "1562", "18825", "6986", "16767", "4426", "6532", "7176", "39634", "40935", "31395", "53427", "39353", "17588", "50957", "32599", "45553", "2683", "7307", "3110", "19421", "23510", "32508", "6050", "23902", "1579", "12537", "40184", "56966", "17567", "47090", "38945", "18028", "12946", "50814", "35797", "1410", "50598", "45014", "56729", "36985", "39726", "33665", "54418", "1566", "5832", "18439", "43727", "12252", "2114", "36109", "32358", "17711", "51394", "9976", "38482", "27019", "11646", "2649", "1328", "1196", "43672", "48627", "13087", "22475", "23313", "57416", "53037", "41538", "16958", "38135", "11337", "4915", "6012", "19030", "17313", "44836", "45816", "347", "6519", "18046", "3878", "3396", "23263", "43246", "1017", "44390", "50363", "48510", "57183", "5544", "28545", "35184", "23501", "2732", "52899", "56245", "54237", "39212", "35198", "52651", "13304", "12076", "52082", "13802", "18867", "50773", "25639", "51412", "41694", "37315", "27492", "49884", "8006", "7109", "48075", "38235", "54633", "46207", "13821", "19573", "23175", "11831", "53987", "45233", "31728", "40279", "40013", "41095", "13941", "56606", "18598", "10889", "36904", "11746", "36997", "45435", "23491", "56212", "16230", "37110", "26602", "4545", "31705", "8517", "39191", "39380", "31231", "51869", "8838", "48596", "53111", "24023", "9261", "9561", "34790", "4852", "2518", "8120", "52593", "18810", "29064", "35538", "6871", "9070", "47015", "46209", "52732", "4445", "29782", "4502", "53945", "47680", "54937", "54569", "28819", "52112", "34889", "10560", "53358", "10237", "12638", "50723", "26609", "34415", "36761", "44746", "42473", "13408", "38939", "5435", "21590", "5709", "11717", "38739", "11320", "30437", "34744", "45558", "15464", "28145", "34824", "33779", "29556", "51958", "3570", "38230", "33", "24557", "7653", "1847", "15095", "15528", "49310", "46758", "33498", "15343", "10503", "50787", "50897", "50092", "44842", "11227", "46222", "29927", "39351", "32892", "6130", "25652", "22876", "45030", "13864", "6935", "59909", "24554", "18197", "57764", "7747", "43343", "9196", "50405", "33580", "9224", "44866", "611", "11919", "29587", "10796", "52979", "43793", "35016", "42747", "53393", "20555", "58826", "29368", "36770", "45931", "27839", "52057", "53546", "5061", "34329", "3790", "52978", "23611", "28095", "1535", "20335", "44726", "27522", "26715", "16920", "20303", "667", "491", "10197", "25864", "10845", "21362", "15285", "55788", "54275", "39033", "35319", "58915", "47687", "11034", "9958", "35040", "28542", "6587", "670", "7399", "37228", "12730", "13227", "29716", "15676", "3495", "21504", "59862", "54277", "33279", "43459", "1817", "16177", "56331", "40230", "24085", "51081", "33378", "5013", "42590", "19535", "24831", "16094", "2134", "26562", "28924", "21645", "34581", "220", "31008", "14171", "48737", "11396", "48163", "11486", "5152", "46346", "8105", "25251", "39388", "239", "18144", "1289", "18259", "38176", "43110", "10643", "25705", "24471", "20385", "7902", "9129", "42442", "47384", "43789", "58973", "35921", "44557", "43115", "7212", "9407", "39858", "3127", "40222", "33800", "48536", "58008", "42004", "54647", "47347", "20311", "53116", "32218", "55685", "17606", "645", "24879", "34917", "24275", "39356", "43390", "6726", "17324", "11235", "33629", "52107", "28472", "15696", "37004", "13159", "738", "6309", "31170", "4073", "27883", "54081", "38599", "47956", "50382", "30845", "47894", "49438", "25649", "4538", "53517", "11226", "3861", "35306", "27424", "27192", "26782", "33890", "32339", "9599", "52550", "2058", "30", "22438", "22669", "50684", "8656", "4419", "13982", "22600", "51746", "45769", "43685", "4982", "17573", "48474", "34073", "44528", "43391", "26739", "35837", "14367", "7638", "29308", "50616", "59597", "40063", "21451", "14816", "16796", "46099", "22559", "9099", "14163", "49472", "39078", "40803", "52271", "3001", "3263", "44068", "44797", "31212", "18853", "38029", "4781", "31061", "58808", "59997", "52270", "52860", "4342", "25176", "30115", "51268", "39818", "55036", "26292", "23379", "5211", "49269", "51846", "49911", "53986", "10031", "1228", "51393", "34378", "26121", "48523", "29370", "18079", "26969", "59994", "24004", "46381", "15599", "34815", "33614", "43013", "3416", "12616", "39853", "10597", "19987", "36129", "39116", "44672", "9796", "39007", "4672", "29695", "39036", "49307", "56666", "21741", "26467", "29224", "12542", "57222", "8098", "27487", "47140", "35946", "11099", "36034", "5027", "21597", "14244", "41029", "28356", "1646", "13887", "53350", "57737", "37308", "54352", "18711", "5602", "13637", "49808", "56647", "25730", "39275", "52223", "51616", "25434", "35340", "18597", "51169", "36216", "38888", "12828", "56809", "8518", "49722", "39891", "41319", "34706", "24599", "7842", "13358", "32933", "6602", "6441", "53732", "38633", "8588", "41401", "15728", "26038", "36869", "27234", "48261", "4375", "14986", "45913", "51161", "50621", "16458", "19743", "28809", "6997", "1941", "6099", "51347", "50162", "49002", "55719", "10198", "410", "52363", "48854", "57418", "12697", "48367", "9915", "58868", "36905", "463", "53073", "38210", "37344", "12069", "18013", "14174", "32213", "29621", "46729", "341", "2817", "38790", "5543", "490", "52325", "48100", "25764", "26883", "17448", "18590", "24159", "21560", "24688", "2963", "53518", "37772", "8919", "12175", "51991", "24065", "28501", "24807", "41305", "42502", "35289", "58529", "38635", "50789", "22815", "58131", "46523", "41800", "1486", "5502", "52993", "50116", "41201", "40146", "33883", "21490", "25636", "37080", "21134", "13585", "51814", "18044", "23575", "20179", "26199", "46553", "45227", "24397", "36994", "16573", "33983", "4483", "27227", "59214", "28034", "36237", "55421", "52441", "57818", "33880", "58186", "22287", "45794", "10470", "28752", "58512", "48818", "30713", "33009", "54437", "58810", "52646", "12823", "22298", "16248", "34838", "56947", "21825", "17486", "16700", "7207", "18663", "23354", "40099", "2061", "51742", "11514", "10862", "46487", "6019", "52881", "58697", "3259", "52785", "25996", "54772", "41328", "36357", "3190", "47408", "39612", "15234", "21530", "7994", "27375", "17526", "45518", "286", "45221", "19025", "23281", "44592", "24161", "3988", "33601", "52787", "111", "21497", "38313", "18324", "21698", "1677", "3571", "4888", "44123", "8143", "35188", "20145", "8327", "28096", "3066", "45957", "18929", "53131", "57671", "17213", "50758", "8334", "46283", "27380", "21730", "11846", "12083", "26954", "59509", "6576", "1874", "42743", "2489", "13756", "7182", "14101", "16351", "15811", "50350", "29032", "29106", "6913", "14466", "9178", "37011", "39740", "19435", "32734", "16336", "45283", "39666", "10393", "23773", "58711", "49444", "5628", "5555", "8526", "49877", "40609", "56518", "38086", "53863", "49525", "48087", "9001", "35352", "38759", "43212", "47438", "47970", "36933", "36311", "14428", "14538", "52894", "39637", "52623", "26059", "43893", "57736", "12825", "37514", "55018", "19894", "14891", "22257", "9466", "295", "17950", "4240", "39427", "13156", "58015", "47269", "46533", "35418", "13288", "14178", "31289", "7351", "19049", "36835", "23930", "16066", "53447", "26129", "25093", "18565", "40880", "30584", "52459", "18009", "1395", "25811", "52257", "11695", "46233", "44190", "55048", "27002", "41407", "52315", "52962", "30416", "51670", "33761", "45104", "30774", "55196", "3779", "19783", "22", "18794", "3213", "42394", "14162", "7331", "2502", "41384", "52330", "12181", "23170", "22354", "10083", "7504", "12036", "51853", "46240", "38216", "15486", "49369", "13651", "14261", "55122", "15722", "51079", "14980", "44811", "45139", "30832", "6021", "25770", "12335", "24505", "30565", "18759", "20100", "40743", "26925", "28660", "17539", "3739", "44900", "10893", "39955", "2162", "36141", "10562", "32295", "8744", "7710", "56760", "56973", "38998", "44430", "45605", "5694", "49121", "3312", "15254", "57168", "38155", "18124", "8708", "15102", "20263", "29579", "17339", "37863", "56402", "4866", "41194", "13842", "29807", "44273", "16454", "11325", "45912", "33360", "3732", "49693", "45057", "2804", "49941", "51849", "3990", "25907", "29739", "16162", "3020", "13294", "32127", "9247", "19566", "22667", "44141", "29718", "53630", "43640", "45333", "420", "17248", "6704", "43537", "22126", "23573", "58242", "15157", "48835", "4127", "9160", "24631", "3911", "59611", "58356", "11796", "31150", "27076", "23935", "23964", "24842", "40054", "18860", "53621", "54136", "56672", "46097", "3973", "23091", "18832", "21018", "27006", "9413", "15475", "35688", "58443", "51968", "10430", "22934", "23512", "6920", "34362", "25259", "4173", "16426", "44526", "45648", "45643", "849", "59936", "42146", "49632", "27944", "10140", "45936", "4385", "15224", "32389", "13753", "57916", "18390", "26516", "50589", "16341", "32936", "19007", "55570", "20259", "4019", "32317", "30887", "49333", "21935", "52453", "39992", "42624", "56789", "25105", "11295", "58883", "56096", "22417", "15961", "22366", "11693", "3967", "21793", "54435", "54290", "59193", "14507", "30196", "36458", "56188", "42022", "56080", "47657", "15585", "16763", "30517", "3636", "38227", "21447", "48619", "39914", "3885", "48110", "17174", "4769", "3045", "44647", "5939", "52137", "48586", "36785", "19585", "28661", "8332", "7752", "23629", "36245", "29691", "1917", "43313", "46008", "6367", "24452", "10846", "3724", "17273", "59956", "36631", "57389", "15593", "44381", "41666", "17569", "2563", "49521", "34181", "4968", "30916", "14433", "44342", "36734", "3406", "9395", "18942", "54264", "31222", "42370", "1563", "30770", "53532", "33702", "45394", "21617", "42424", "55535", "23319", "25835", "7659", "57515", "36464", "12673", "37949", "32899", "20013", "19846", "59246", "50644", "6083", "20506", "18756", "50451", "47542", "6645", "21023", "34968", "31658", "36324", "57098", "39435", "29599", "31454", "57141", "28424", "34004", "24201", "58979", "48777", "44163", "13313", "20620", "31172", "42460", "5138", "32347", "39828", "31725", "40932", "51187", "22054", "10667", "15738", "31334", "44183", "39885", "43625", "43782", "21471", "46765", "56888", "38099", "14053", "1222", "56171", "20118", "24319", "32143", "8347", "29824", "49860", "4462", "54048", "45395", "19485", "18752", "46149", "40310", "13037", "33509", "29673", "41469", "2971", "27157", "41902", "1601", "36058", "53871", "26657", "25373", "28532", "30687", "40380", "44735", "48109", "11252", "36272", "53972", "1465", "20203", "4878", "53132", "35281", "53781", "9626", "10095", "29304", "36777", "57630", "36552", "6415", "40679", "6830", "42735", "25711", "55199", "42577", "30342", "1918", "55288", "25032", "17127", "50641", "43585", "707", "38283", "30450", "29652", "49606", "10692", "33414", "9110", "16586", "43258", "7390", "13281", "21453", "39489", "33296", "54882", "1967", "49213", "44448", "27152", "25188", "38075", "33209", "28292", "13962", "11363", "17915", "4626", "46282", "49249", "52856", "56166", "42853", "27040", "54951", "5941", "4106", "29174", "16306", "27285", "29119", "2112", "59346", "18075", "53044", "6443", "37371", "42224", "47703", "2190", "46380", "41518", "46320", "629", "25140", "9367", "35772", "55073", "38984", "58609", "15249", "27732", "10162", "42966", "8063", "38033", "34662", "27262", "34035", "22907", "21185", "51978", "54994", "35020", "51510", "20494", "22000", "53074", "1724", "17747", "46970", "53197", "46460", "48161", "7582", "39280", "48245", "6605", "52083", "38982", "26029", "27342", "53589", "38129", "10254", "6076", "31359", "13758", "20188", "8500", "20069", "59260", "518", "57913", "55954", "38929", "50302", "58045", "5541", "17579", "23767", "26564", "31495", "6340", "3003", "29310", "23851", "19430", "52461", "15063", "24142", "15053", "4471", "46294", "5452", "42403", "19349", "53587", "34666", "22274", "6008", "16839", "3489", "20906", "9507", "59670", "58322", "8005", "50348", "32555", "58922", "42466", "38398", "23489", "7455", "9588", "37430", "39433", "49082", "5934", "14177", "30953", "34707", "20398", "10459", "11047", "22755", "45295", "5914", "17064", "9198", "3114", "50392", "57959", "22026", "24652", "41880", "11929", "40673", "7418", "30139", "56800", "14028", "29358", "3256", "56909", "22609", "39214", "52457", "12068", "10953", "36564", "55325", "34660", "22294", "16794", "54114", "33050", "47540", "40498", "14975", "63", "55531", "1816", "43475", "29207", "33733", "44763", "22240", "54445", "57", "5093", "27190", "39568", "49654", "31699", "14947", "46355", "25181", "29415", "37683", "30268", "31084", "26880", "27862", "35475", "18952", "39032", "21348", "21957", "42299", "47179", "38870", "57493", "41730", "34774", "57202", "18003", "53611", "39352", "32592", "27173", "38678", "34501", "7604", "23492", "27437", "34148", "52961", "54644", "5599", "4697", "44855", "47615", "28641", "1212", "44366", "32085", "13413", "16588", "31039", "47881", "9793", "40580", "55118", "32395", "4599", "9884", "34947", "26891", "45360", "56203", "2265", "43017", "17994", "54738", "6403", "54426", "35691", "37374", "37263", "39863", "4105", "9844", "35732", "13619", "54554", "20049", "32719", "29480", "24309", "40969", "6168", "36679", "14360", "48080", "44214", "42746", "52783", "18874", "47134", "10373", "59440", "54743", "25588", "33149", "49799", "4264", "34481", "22409", "56821", "14789", "27625", "28173", "54529", "21068", "33638", "42295", "4305", "42208", "610", "45455", "31637", "34040", "7303", "46117", "15153", "16934", "43316", "58089", "33370", "25921", "5365", "3707", "46280", "50062", "40449", "47718", "56770", "6553", "31926", "33700", "15087", "31600", "42009", "6791", "32044", "52758", "5839", "15867", "5060", "58360", "9300", "51657", "40736", "26126", "39673", "56142", "19836", "53627", "25299", "29206", "36608", "7987", "49580", "15027", "4382", "52285", "43032", "49846", "24596", "20358", "8234", "39411", "17208", "56039", "46524", "45718", "10049", "54920", "34170", "23035", "48103", "54660", "24839", "253", "57139", "25424", "16469", "23969", "5433", "47162", "23705", "45323", "15685", "28741", "43328", "8375", "20337", "40892", "51960", "951", "13444", "16745", "13597", "54102", "12020", "11884", "45949", "52203", "45948", "1824", "13519", "4432", "45128", "5569", "4527", "22194", "54029", "31962", "4193", "9817", "51259", "40145", "7855", "10383", "56857", "24754", "35143", "50851", "26381", "25580", "23201", "17819", "49759", "19239", "44946", "46973", "2505", "50359", "57179", "16768", "5173", "27179", "41119", "32439", "16431", "33431", "11020", "11611", "32532", "58658", "27580", "33753", "54536", "40470", "48664", "34020", "13312", "28724", "55250", "32959", "25131", "49956", "40247", "51208", "12152", "30658", "59881", "38777", "55689", "3599", "27932", "27000", "54885", "14572", "39721", "52479", "45410", "2210", "35272", "27975", "27874", "1380", "34663", "59064", "58318", "56731", "19899", "20708", "21487", "35024", "34645", "58850", "7753", "8544", "55800", "40870", "25130", "33272", "14095", "46829", "666", "45206", "17803", "17180", "59549", "42708", "45381", "9005", "21623", "51123", "51934", "44689", "45278", "47109", "18887", "28304", "18275", "26050", "55914", "17421", "37628", "26238", "48712", "56781", "41327", "31281", "6026", "45219", "20975", "25212", "15282", "33862", "24176", "38623", "32397", "53694", "33763", "41815", "43372", "44453", "19681", "50833", "56154", "42338", "25679", "36519", "50756", "56161", "27754", "34464", "33923", "25266", "32941", "38671", "4065", "1593", "5906", "13598", "6116", "29279", "55528", "23740", "6245", "52341", "48780", "22141", "7482", "12134", "32991", "27065", "5756", "33432", "1399", "37542", "18789", "21263", "47120", "32701", "48551", "9739", "29515", "13251", "29914", "36626", "9717", "48981", "45761", "58526", "19808", "17508", "50688", "44654", "43004", "55251", "45787", "53005", "7884", "1409", "40477", "643", "49091", "46227", "15223", "32238", "47282", "15471", "24248", "18543", "51013", "41184", "22451", "57214", "19507", "37112", "52509", "33215", "57699", "40282", "15733", "36263", "2287", "27506", "5816", "50431", "51206", "43165", "30976", "44715", "38267", "17106", "5451", "32752", "10387", "12785", "11544", "37594", "53985", "35605", "36530", "30317", "8768", "30521", "26437", "16056", "19596", "51813", "15061", "44668", "9100", "15579", "51139", "24450", "39177", "313", "26100", "19143", "35979", "31566", "41710", "14109", "24671", "13066", "18162", "34450", "8189", "5316", "16762", "51919", "41185", "47431", "44476", "20071", "46405", "57259", "20416", "18480", "15725", "32927", "58754", "33561", "54439", "13041", "29469", "7195", "34331", "40307", "7416", "44320", "58217", "25441", "46288", "2615", "51795", "18323", "30973", "12410", "19392", "32972", "35321", "12155", "4903", "43941", "36527", "4349", "3039", "54911", "15619", "58130", "5715", "29428", "34139", "49967", "35338", "39847", "11411", "57313", "41931", "44697", "53378", "58487", "1963", "42369", "11039", "55691", "50709", "12510", "605", "29783", "42388", "36765", "34002", "17538", "36355", "48515", "25409", "39512", "15749", "10045", "23650", "49961", "16129", "23885", "56424", "34016", "39616", "50822", "1403", "11609", "51798", "44446", "17599", "12041", "58262", "4695", "41168", "57673", "1774", "47635", "20831", "22861", "360", "29274", "9712", "22813", "52893", "27241", "13640", "30623", "57131", "40393", "42436", "36918", "7253", "22040", "24153", "54655", "33820", "36204", "7388", "59511", "19429", "38852", "36699", "54691", "15315", "39870", "24477", "16667", "52565", "4463", "19205", "45681", "15417", "46855", "6531", "56610", "57392", "29099", "10504", "8376", "2977", "29488", "30902", "19920", "37078", "55157", "59719", "27373", "58228", "25324", "37334", "39111", "56832", "53196", "12963", "17196", "36112", "17908", "1732", "24655", "53848", "38560", "19726", "36356", "35785", "40110", "32490", "38791", "36142", "59402", "90", "3899", "58257", "5207", "56173", "11586", "55110", "13375", "53049", "5990", "16076", "10177", "7598", "30593", "9502", "42131", "51002", "11749", "56603", "30929", "30990", "23335", "36667", "53543", "52564", "47412", "19021", "32983", "50167", "35440", "37524", "41152", "6102", "14457", "35810", "53910", "59445", "31078", "17502", "8972", "21396", "31239", "51179", "6719", "44241", "26151", "31858", "36456", "20574", "35053", "5021", "25595", "8140", "26955", "12383", "7651"]], "valid": ["int", ["24863", "34154", "36167", "29011", "42656", "55748", "50959", "34092", "48759", "19370", "18570", "53922", "23093", "32944", "46430", "26142", "10471", "32625", "50228", "8377", "35868", "50778", "959", "41341", "42259", "29291", "34546", "1842", "58053", "30441", "4205", "1050", "6427", "7028", "17695", "48509", "23223", "14315", "39623", "45339", "25552", "18493", "55755", "21336", "8451", "33288", "21769", "976", "29082", "44437", "48194", "12530", "50600", "9852", "54046", "7318", "8973", "43364", "34553", "5306", "46464", "18674", "9101", "35790", "19228", "1925", "55705", "11164", "47410", "20779", "2186", "56508", "18531", "36999", "31082", "47398", "49816", "58465", "59293", "9902", "17210", "29528", "51736", "1243", "18023", "53563", "31739", "31228", "1691", "13298", "59717", "7643", "16579", "39942", "47302", "18886", "44411", "13415", "7766", "46504", "46068", "12954", "2908", "5529", "56836", "29702", "8436", "28305", "36445", "1896", "38849", "6700", "29347", "26236", "59662", "5891", "40926", "42583", "21424", "52681", "41971", "2411", "58774", "19388", "10091", "6267", "31749", "2431", "36337", "53436", "1068", "53784", "53639", "46670", "921", "37953", "11467", "29292", "51887", "320", "33180", "58261", "3723", "50876", "1179", "58027", "570", "49381", "45824", "30048", "6043", "58292", "2051", "53258", "433", "2217", "17289", "31184", "37008", "42548", "47660", "51399", "41174", "18437", "4384", "49773", "26360", "8657", "15488", "18204", "3931", "31687", "2876", "21253", "12034", "9452", "59059", "54443", "23228", "8670", "41921", "12236", "22496", "28074", "44674", "12313", "52621", "42791", "22086", "43729", "13150", "5725", "47521", "29466", "22499", "23158", "53154", "24894", "12099", "52683", "49736", "22469", "30932", "34597", "47053", "7405", "33773", "28967", "45504", "33618", "39957", "11497", "50968", "31455", "28153", "57322", "45170", "15393", "23356", "22035", "4429", "10907", "344", "48390", "37672", "50106", "21141", "46868", "36732", "15057", "52587", "5586", "51448", "14331", "19865", "27563", "2279", "7711", "18594", "47860", "53964", "53645", "3188", "35345", "23198", "33687", "52349", "45654", "5662", "12168", "54547", "10490", "34019", "42230", "41218", "39235", "30603", "8067", "837", "43880", "35529", "9024", "16270", "2702", "44484", "58499", "36110", "36912", "48022", "21254", "32475", "31240", "36096", "23960", "58466", "54560", "6423", "43348", "14966", "43135", "12975", "43706", "33409", "50622", "2728", "41302", "35614", "34274", "4136", "32253", "37316", "35377", "10903", "10335", "9733", "23463", "19773", "15668", "46937", "48385", "28779", "31051", "49776", "5773", "31981", "24549", "25338", "46436", "56009", "18913", "1968", "30051", "19182", "16741", "39425", "39769", "19015", "34201", "53858", "43223", "27453", "31197", "2313", "14754", "23204", "28820", "37757", "54662", "59451", "42567", "39146", "53337", "22629", "37860", "58614", "22112", "1734", "13717", "42602", "35397", "17318", "17957", "38913", "41110", "41293", "32524", "29685", "43496", "38103", "22487", "36605", "44657", "31560", "28654", "9027", "46426", "36571", "54710", "54864", "47401", "56530", "7297", "7875", "14404", "59049", "40415", "12492", "3554", "21103", "22900", "38160", "9408", "56930", "13389", "41840", "43218", "27893", "31432", "57436", "10452", "3395", "43096", "7820", "30112", "33660", "35315", "35166", "4780", "55903", "54180", "52869", "48801", "22036", "31201", "45241", "11003", "9681", "25990", "26622", "8312", "24829", "4333", "11572", "3578", "35933", "2833", "39281", "55481", "43703", "36685", "2878", "58846", "56913", "4654", "29349", "30444", "13003", "22046", "20448", "33632", "44538", "15424", "8136", "49627", "12519", "24822", "29844", "16289", "45554", "38260", "8811", "59108", "47413", "25985", "28883", "19271", "55783", "57549", "31615", "50852", "28823", "13030", "3626", "8318", "35030", "1592", "4327", "2344", "56548", "26655", "17077", "41491", "4840", "52742", "11351", "5989", "14157", "27915", "45993", "4696", "23953", "58553", "33712", "27680", "58269", "5319", "42404", "43881", "55474", "59415", "16063", "24455", "20675", "34302", "57947", "3398", "44711", "37766", "41335", "44521", "23759", "46741", "43394", "39291", "35536", "29608", "35160", "22443", "58818", "40696", "39364", "41048", "32848", "30782", "26345", "36260", "3809", "24815", "17837", "40720", "56611", "5824", "20600", "15584", "38520", "19180", "16686", "58093", "46037", "58275", "26160", "36758", "27275", "5164", "1200", "9435", "30883", "57524", "7863", "22174", "39345", "52985", "37287", "31860", "51260", "59703", "16848", "43886", "49411", "50998", "48524", "10658", "9644", "30824", "43379", "59832", "31565", "11869", "16263", "54915", "57289", "5406", "44167", "35298", "53403", "36299", "33739", "19181", "23055", "26148", "36885", "56616", "28189", "54200", "22308", "14997", "26003", "14618", "54173", "40570", "46036", "27808", "59802", "45254", "26601", "28664", "38394", "37155", "8986", "44413", "16486", "7732", "9910", "38180", "59430", "55835", "46887", "52197", "49198", "12776", "15354", "29254", "407", "28303", "8807", "10077", "42194", "37329", "35881", "16602", "34169", "32009", "5285", "29013", "6493", "33583", "40283", "4815", "47420", "13406", "7961", "35796", "37647", "39219", "13510", "25202", "49499", "8698", "41058", "10942", "42190", "13634", "13047", "48682", "18550", "10644", "48330", "35358", "19144", "18979", "42678", "22296", "19876", "46485", "36666", "13544", "54388", "27918", "29633", "57455", "4979", "40842", "145", "31793", "14429", "40533", "14491", "29730", "8244", "26738", "15368", "59621", "39447", "32786", "7456", "59903", "54141", "28485", "31751", "50075", "15984", "15466", "54059", "24731", "53237", "54658", "6887", "10948", "45752", "56165", "2743", "50486", "45111", "1770", "50699", "37283", "56091", "32826", "46017", "26279", "10060", "44249", "24060", "18589", "16437", "52864", "47579", "57573", "49123", "28097", "47708", "13843", "3516", "34932", "42036", "24120", "10595", "42786", "10510", "47031", "40327", "7380", "8271", "58365", "19538", "10632", "6816", "2084", "44606", "48827", "17207", "6286", "24918", "17140", "10976", "57258", "9687", "29139", "50918", "58521", "1032", "41131", "932", "11259", "48247", "34992", "22028", "22865", "1362", "48480", "57307", "57974", "1517", "32750", "13586", "38234", "13401", "38400", "32237", "16581", "39980", "14267", "15051", "54513", "30240", "10813", "36423", "3594", "53750", "29125", "51905", "19723", "2618", "25022", "55995", "22922", "58298", "43905", "28884", "31442", "46636", "43908", "57742", "25482", "50049", "59943", "46826", "30380", "58385", "17036", "5753", "15388", "29969", "26414", "36883", "2076", "3133", "32113", "42546", "12146", "41889", "41584", "53492", "36440", "46077", "10461", "1720", "27361", "7613", "21052", "45262", "47208", "33549", "51177", "40205", "54613", "19753", "4459", "9597", "20473", "48615", "13684", "43304", "5878", "34579", "53280", "14930", "47425", "28753", "2381", "1499", "38969", "25898", "4895", "57859", "55021", "12202", "4340", "57639", "9953", "7304", "18227", "7694", "24232", "53372", "22226", "32176", "25222", "33447", "33747", "47081", "45403", "6769", "21364", "50829", "31327", "12392", "26532", "37117", "46688", "15015", "28471", "43911", "55597", "13856", "13985", "1932", "52992", "30005", "36955", "1645", "33486", "4417", "28540", "12601", "44157", "7509", "58753", "54193", "18529", "24619", "44544", "8287", "20524", "50118", "56780", "49304", "55430", "42737", "39768", "36563", "41870", "44367", "5703", "44467", "21281", "15743", "25851", "19923", "39430", "55076", "48000", "54316", "53942", "3777", "35048", "57074", "1088", "58129", "49483", "49488", "5757", "35679", "43752", "15656", "54699", "43530", "7373", "40344", "34039", "460", "30582", "29714", "46803", "13269", "53993", "20740", "41736", "48198", "49766", "52299", "3302", "36441", "38981", "55203", "59776", "8180", "7805", "8636", "7627", "51710", "48088", "35263", "35009", "54103", "31366", "52044", "37502", "43158", "27389", "681", "40151", "13026", "29753", "1341", "20430", "41262", "28900", "48235", "45749", "13955", "53380", "39693", "25084", "37387", "50026", "11097", "25456", "9459", "58331", "36067", "41031", "15083", "1719", "7691", "38885", "39419", "59493", "8374", "43698", "40734", "20310", "37174", "41520", "56880", "21363", "16897", "3768", "28758", "38990", "25286", "20211", "38284", "36081", "46744", "32464", "20948", "17615", "59136", "8462", "8091", "784", "21724", "22575", "11389", "10746", "37276", "20554", "50061", "14687", "44206", "18111", "53963", "52572", "593", "22700", "43172", "8796", "35565", "6241", "4023", "30952", "48915", "51119", "1608", "43879", "14640", "50324", "2650", "43310", "11924", "12925", "45973", "21218", "32288", "31426", "35245", "30760", "26837", "10915", "38165", "52419", "56859", "4879", "52586", "9688", "29104", "29687", "21140", "43762", "24891", "41929", "44760", "22947", "3664", "52897", "33604", "35254", "1139", "49441", "16124", "48039", "27804", "3124", "43007", "23239", "1203", "56694", "59739", "57664", "49578", "55357", "35276", "5322", "15689", "13687", "47155", "3309", "55141", "15216", "29162", "22818", "19092", "45346", "11187", "31906", "41948", "14342", "43178", "20907", "1057", "31346", "4210", "2394", "48725", "45722", "49215", "24474", "42558", "18250", "9074", "50768", "5561", "34475", "18795", "55756", "14104", "40536", "22277", "5148", "58509", "16198", "15702", "46324", "7813", "54017", "42157", "36170", "39597", "14143", "32249", "23082", "1168", "6963", "39098", "3447", "45173", "33437", "47170", "14656", "58054", "40446", "34588", "8598", "11087", "54952", "18173", "42773", "31213", "1346", "42163", "48546", "23173", "30414", "43724", "11391", "45628", "7702", "41530", "49174", "31343", "32415", "4303", "36432", "38664", "25419", "45169", "29475", "36644", "44428", "21299", "40727", "14059", "26325", "55333", "29128", "26400", "24146", "1116", "51712", "18834", "31016", "50341", "52200", "22921", "22068", "9256", "50850", "33329", "42393", "51897", "36124", "1366", "59601", "4197", "30518", "48101", "2671", "44688", "22563", "53941", "32481", "57528", "7791", "12526", "3883", "13734", "17211", "53133", "6626", "9151", "16425", "48899", "36277", "7956", "11836", "20221", "35835", "57574", "3734", "48142", "51478", "14481", "28449", "9355", "38301", "26815", "34495", "27952", "2330", "10231", "33250", "49492", "4594", "16462", "12522", "58805", "52871", "9877", "38274", "41356", "13016", "54328", "54396", "55594", "51938", "16773", "59807", "25281", "51829", "27191", "48946", "3479", "24019", "44893", "33568", "54397", "37569", "28541", "31278", "55372", "40219", "2413", "3102", "20415", "49871", "59891", "53966", "38266", "29588", "39551", "59447", "4930", "33080", "38889", "21202", "18612", "26455", "28052", "23651", "31834", "27670", "2816", "32141", "56179", "17942", "27704", "9995", "19502", "242", "13419", "9899", "52873", "47688", "1275", "28760", "31318", "32129", "12554", "57295", "17985", "43860", "14195", "36399", "9715", "4038", "18135", "49279", "14390", "6525", "2999", "3983", "20476", "55083", "48892", "36613", "20205", "25823", "54906", "14616", "24087", "8563", "39799", "51436", "6132", "18988", "43696", "27922", "1583", "18508", "49537", "51810", "24537", "53674", "388", "46896", "26599", "7524", "41463", "24160", "59874", "8871", "25180", "1334", "19802", "5985", "39621", "31664", "5636", "59072", "51293", "39024", "26195", "48507", "20460", "39801", "8418", "11425", "12443", "20271", "13483", "41650", "21013", "3761", "37185", "27140", "29550", "22709", "44418", "57275", "42872", "13652", "13789", "24131", "36638", "6739", "22018", "23547", "37713", "32262", "4928", "52497", "31674", "39422", "14262", "27072", "27368", "30248", "41537", "2469", "18559", "49501", "4978", "39687", "19412", "19017", "25900", "12912", "2496", "53323", "13412", "13830", "29003", "4596", "30537", "18153", "56174", "29542", "24878", "27319", "52517", "17360", "25334", "44934", "51617", "17533", "21302", "11855", "57233", "4154", "1546", "15921", "28669", "55197", "36051", "27202", "8399", "2003", "27757", "4517", "46184", "20725", "33844", "7216", "22077", "48971", "22450", "49297", "2238", "35548", "57422", "27690", "47492", "28028", "361", "50996", "49238", "39766", "39158", "44014", "12813", "46273", "19946", "45694", "44955", "3661", "49751", "5919", "28048", "23881", "28082", "9617", "41066", "20517", "35890", "36689", "10086", "12734", "19425", "16777", "19673", "53733", "29715", "58582", "4739", "48757", "21194", "48412", "53511", "6486", "2271", "14124", "20867", "50860", "46747", "31610", "43312", "24286", "27060", "37457", "56259", "9528", "32799", "50477", "59689", "21304", "3319", "59184", "4213", "43346", "55901", "27470", "12040", "56664", "42760", "15289", "1021", "7737", "10139", "42562", "28446", "5581", "47457", "55446", "34519", "48603", "58005", "33921", "1264", "29886", "52731", "36021", "32938", "28556", "14307", "12262", "42185", "46751", "10135", "37839", "8979", "5388", "31124", "38554", "18133", "4848", "41488", "57234", "50925", "48252", "44583", "8154", "35004", "30227", "26482", "35322", "57337", "50234", "12603", "19175", "11685", "27582", "9871", "4727", "23775", "21960", "6923", "41949", "18636", "35878", "2440", "42356", "20425", "46220", "2961", "7522", "33021", "27771", "58887", "26069", "7036", "17927", "20941", "5309", "24767", "32360", "22374", "1791", "14080", "45732", "27838", "35604", "20373", "28989", "29789", "51477", "23106", "34576", "38172", "51229", "20767", "5187", "17039", "25633", "29295", "5307", "6524", "20507", "14297", "57456", "33731", "5578", "25068", "52843", "35755", "36401", "30512", "4188", "36717", "47289", "8266", "58489", "41879", "21764", "44215", "49090", "51036", "24207", "803", "15177", "23552", "18960", "12131", "51163", "28634", "34472", "22913", "9006", "34983", "38610", "57247", "22103", "28217", "42092", "38940", "23742", "9957", "34142", "42042", "41190", "57962", "16803", "43652", "4488", "29346", "29455", "57927", "3423", "20550", "47663", "291", "30002", "37682", "24475", "57637", "37203", "21879", "45854", "3829", "42979", "52210", "2334", "13058", "48987", "35757", "19922", "47622", "45356", "47056", "21636", "16883", "19446", "9186", "21613", "8631", "33442", "49015", "18490", "17219", "53742", "29012", "27621", "35472", "17860", "54000", "56591", "49818", "9448", "35749", "26692", "55770", "1095", "22642", "47712", "5174", "42924", "40785", "11614", "25070", "10750", "23443", "54392", "45844", "9513", "46592", "3306", "45602", "39626", "5331", "211", "47927", "31671", "14273", "16067", "8671", "20151", "9484", "19915", "47742", "47814", "39117", "28647", "3517", "45160", "43755", "27921", "2952", "35955", "54466", "31652", "22023", "40643", "41420", "3158", "29944", "4634", "7679", "58384", "19825", "34430", "14617", "5909", "15340", "26157", "29565", "43904", "39571", "49074", "20477", "11528", "58604", "33859", "20183", "52780", "19785", "39926", "44901", "48172", "23701", "16005", "20539", "15148", "1554", "4183", "80", "54856", "33457", "17921", "54219", "58276", "28667", "50923", "12970", "34789", "57856", "36767", "42522", "43596", "32136", "19855", "46434", "12607", "30280", "11380", "2029", "48788", "58917", "42361", "50114", "53653", "3276", "2155", "19982", "58914", "50821", "18504", "55002", "49016", "9946", "46625", "12352", "46129", "16532", "43901", "42778", "39629", "36064", "16055", "10160", "32477", "34874", "31636", "6621", "43171", "29911", "39075", "32456", "23060", "59310", "40967", "7005", "3104", "12042", "24930", "18866", "56221", "40610", "6337", "26866", "24266", "36179", "37300", "57262", "12640", "42587", "49027", "110", "23559", "532", "43569", "25865", "2272", "15043", "30796", "25839", "47919", "56272", "12004", "54182", "8182", "42783", "24245", "29087", "44558", "59337", "9051", "25256", "55223", "46228", "8176", "21845", "24043", "36069", "28572", "44656", "32965", "39176", "29938", "27459", "13910", "49727", "45223", "49990", "16006", "49465", "45927", "24124", "25087", "43670", "14487", "30469", "40704", "49505", "10050", "12396", "41473", "37450", "48832", "19769", "50539", "34945", "20786", "37278", "20544", "46362", "27966", "5570", "59234", "44686", "44393", "8534", "4655", "20098", "14631", "6990", "55516", "6945", "32967", "18643", "29601", "30277", "32461", "23719", "7942", "26319", "9244", "46025", "19903", "40491", "8673", "56829", "9989", "38764", "25193", "22358", "48663", "33805", "6915", "13480", "23805", "5377", "23472", "8574", "55694", "5565", "31196", "39517", "54475", "51108", "4943", "48867", "26031", "1627", "28391", "30197", "41998", "26834", "5212", "11784", "16193", "3327", "56635", "8456", "22646", "25112", "9115", "33588", "18218", "17634", "42868", "48686", "19378", "45231", "54165", "28793", "56926", "49398", "54002", "40248", "34857", "29873", "15869", "5142", "4367", "32266", "50627", "55897", "33962", "38147", "58100", "381", "23770", "15934", "3167", "45803", "16210", "43148", "55885", "17841", "81", "1201", "52009", "49396", "1132", "11835", "41982", "33419", "42706", "26341", "22775", "3862", "14540", "45244", "1111", "52773", "28798", "23475", "25108", "3298", "45502", "40941", "10931", "18316", "9853", "58921", "3965", "52738", "55945", "51784", "56232", "5718", "998", "11925", "1415", "12965", "52274", "28811", "31952", "50912", "39157", "2074", "31682", "47601", "24703", "16651", "32596", "18806", "23980", "6339", "35226", "3011", "22661", "48691", "853", "24625", "26841", "59279", "41442", "8834", "56853", "10641", "45277", "8026", "47911", "48754", "12233", "26081", "33254", "30855", "11451", "38640", "32384", "56365", "22676", "19918", "33535", "31178", "41195", "26912", "31890", "20537", "56874", "32144", "16912", "48496", "36160", "21951", "26315", "27559", "797", "54267", "15553", "6751", "18432", "34848", "59077", "30608", "24478", "1679", "47278", "24868", "32693", "53663", "2441", "18373", "48962", "6014", "38867", "31237", "26752", "20821", "20923", "23646", "34669", "47900", "32209", "12908", "512", "19307", "44460", "52123", "55740", "44552", "45488", "38252", "7215", "45755", "37480", "28904", "35235", "53493", "38521", "20225", "27037", "15032", "3354", "39298", "22725", "23156", "54449", "56104", "47783", "49965", "43967", "25614", "787", "1288", "45133", "17909", "32115", "2712", "40612", "52395", "22537", "22171", "22908", "41767", "59232", "40840", "42302", "44290", "38495", "9500", "16657", "52343", "8020", "1319", "35415", "221", "5492", "1370", "32902", "1548", "57045", "45892", "6421", "55248", "43792", "53919", "5885", "8749", "39628", "22938", "46982", "32070", "26024", "55311", "412", "16303", "14179", "46609", "24098", "6780", "16703", "4570", "13526", "50667", "5673", "3638", "28676", "48749", "47551", "39640", "41275", "53174", "8111", "3021", "23410", "1475", "54220", "12815", "39136", "42984", "15298", "29947", "39569", "24889", "30309", "16543", "34316", "32158", "15126", "52345", "6827", "51570", "4298", "50468", "22556", "47366", "17709", "13624", "38128", "47926", "51522", "11720", "23234", "55918", "52425", "12534", "18741", "18811", "32949", "57852", "59119", "57542", "44604", "20760", "53500", "34296", "25957", "28246", "37685", "31009", "53034", "24545", "30562", "8085", "22823", "21014", "26790", "58578", "27273", "36050", "22718", "19630", "18145", "7450", "20988", "5364", "34239", "51619", "1314", "59876", "49618", "43839", "55429", "24091", "21817", "16693", "45007", "31496", "53459", "37014", "46818", "12190", "2181", "34048", "50658", "51609", "6080", "40424", "54521", "41796", "30149", "39886", "25923", "45420", "28408", "48457", "45580", "9571", "47430", "36690", "7745", "22631", "29524", "43454", "38820", "41070", "55320", "16684", "50529", "21217", "26681", "8597", "57136", "41350", "37277", "735", "21830", "51764", "737", "8797", "32219", "37761", "45364", "25082", "16025", "46022", "1503", "34783", "4922", "36611", "59847", "59644", "8926", "36415", "23554", "58970", "26013", "3221", "42498", "58801", "49966", "21509", "40246", "7006", "24053", "45166", "29866", "36781", "23594", "13591", "22051", "20202", "35900", "14971", "43837", "10988", "14126", "34426", "530", "12338", "21815", "6603", "7543", "32454", "20394", "40295", "11747", "45338", "17816", "47849", "30511", "40021", "2965", "57922", "52284", "8004", "17657", "3467", "9868", "34557", "35355", "33528", "39999", "3025", "9504", "26203", "10592", "26936", "14455", "36965", "18807", "29555", "31329", "9294", "46988", "4032", "32185", "20754", "27649", "53847", "12047", "58583", "346", "48968", "43001", "40834", "30538", "22135", "21099", "24974", "26803", "31283", "15745", "23888", "11812", "16984", "35370", "44166", "39703", "50151", "53782", "13825", "29394", "6247", "3151", "50540", "33132", "46960", "46597", "16238", "36773", "39683", "26847", "19765", "9195", "50962", "25305", "54462", "31715", "37578", "28553", "58599", "45603", "52582", "15706", "58975", "50182", "31555", "7794", "30350", "4262", "22870", "19926", "9538", "52166", "34671", "28590", "17512", "32442", "9956", "39310", "11070", "27293", "22748", "34195", "19456", "51714", "6348", "46658", "8590", "13433", "27903", "48787", "25055", "6477", "24443", "56920", "13293", "43567", "11253", "58113", "39301", "14937", "49026", "649", "16259", "56292", "17490", "3649", "20107", "8033", "7091", "20626", "2843", "29129", "17969", "34176", "39563", "47211", "11630", "3436", "18718", "20842", "14135", "20340", "32157", "53064", "59921", "36043", "29552", "33557", "31790", "18899", "51911", "8240", "32918", "52213", "15935", "1348", "48742", "41806", "15129", "13093", "42216", "1145", "33135", "21307", "48985", "1484", "42057", "46631", "26881", "53229", "8997", "12182", "995", "400", "18704", "22917", "55156", "11267", "47870", "18176", "35850", "19919", "27473", "9770", "49511", "23185", "37596", "33855", "31361", "28851", "50446", "26154", "7104", "47672", "57015", "29610", "39741", "20062", "36766", "15922", "36723", "57305", "9792", "14476", "20046", "4124", "43673", "52849", "13027", "19342", "57423", "29523", "43988", "19225", "11160", "50469", "18473", "30082", "3769", "34275", "47665", "51333", "50181", "6718", "8135", "10549", "23976", "32816", "18187", "1836", "606", "3799", "24910", "59371", "59418", "5449", "44602", "45205", "47175", "31147", "56201", "44926", "21778", "10982", "59242", "32825", "48545", "25313", "25808", "25514", "42089", "54024", "13615", "3238", "11269", "2774", "20608", "33235", "8619", "18448", "39285", "40382", "52327", "49188", "11583", "11459", "19126", "31386", "29545", "51235", "24411", "30960", "31497", "47070", "34429", "57924", "41202", "30626", "16389", "21196", "7174", "14846", "22225", "32715", "35756", "58227", "36335", "11647", "39011", "53688", "59745", "26721", "40102", "38231", "12218", "39555", "33311", "30938", "30753", "53144", "1933", "22102", "39296", "47679", "1034", "36169", "15996", "19068", "25896", "30375", "11181", "29460", "24726", "46420", "3308", "208", "40699", "42261", "58220", "20545", "30519", "49468", "37907", "35828", "22447", "58397", "15270", "16034", "34064", "41050", "42588", "27860", "41097", "22276", "24714", "47589", "18118", "40753", "53531", "31627", "30447", "58948", "52136", "4440", "45080", "11113", "22410", "12344", "12229", "57157", "36931", "39556", "23036", "57545", "2406", "9420", "11804", "13334", "35690", "49454", "47375", "12376", "2172", "28076", "44663", "57516", "26656", "30247", "39107", "21032", "34054", "47918", "17316", "55030", "58806", "28559", "51265", "19984", "25347", "33980", "35560", "25063", "5690", "32903", "53043", "6663", "43910", "34845", "59697", "5448", "36310", "38895", "46908", "16832", "29130", "3603", "41695", "13316", "40243", "18782", "45345", "10013", "12141", "3264", "39881", "2600", "11421", "8356", "39164", "15250", "49831", "4528", "9777", "36707", "25555", "32096", "56718", "45304", "39939", "32890", "17098", "55817", "25668", "55805", "16499", "12564", "3314", "22993", "30142", "6207", "55020", "12022", "28195", "25872", "23505", "37054", "28273", "4003", "24034", "51356", "28737", "28038", "49901", "11797", "27476", "23017", "48961", "4581", "48902", "17610", "42435", "1063", "36645", "23831", "29403", "8746", "34045", "53550", "55352", "35247", "51956", "36465", "41088", "43943", "41364", "21912", "5687", "53568", "53469", "53739", "27677", "7707", "26087", "24869", "1345", "45110", "55389", "12322", "23029", "37752", "35108", "6723", "23056", "26149", "36476", "48795", "11825", "40693", "44513", "30252", "39906", "17155", "10634", "22010", "30442", "52143", "28174", "55618", "24970", "19453", "21659", "41894", "8821", "58216", "39421", "9997", "48751", "54594", "41320", "28566", "51038", "18082", "34551", "7137", "24690", "36119", "49660", "28418", "15020", "21586", "14026", "12867", "10400", "58957", "24170", "15863", "23916", "7720", "44359", "47297", "28429", "12452", "7198", "9677", "3750", "21656", "53002", "21884", "38588", "52548", "30400", "41008", "55525", "11122", "51838", "37824", "13814", "50819", "47353", "43651", "16816", "52244", "22716", "41199", "23314", "29441", "52378", "15906", "38368", "24799", "2012", "21291", "7268", "17111", "19751", "18634", "57907", "6070", "11657", "6351", "6891", "30224", "31662", "310", "53217", "34697", "49065", "14733", "30187", "46633", "53121", "49585", "35293", "13951", "43020", "11862", "59243", "1474", "1530", "37006", "12722", "48056", "22577", "11970", "1072", "21773", "21899", "3916", "31472", "31469", "8747", "25803", "25464", "21377", "57370", "7931", "24604", "32430", "19748", "25454", "29203", "40691", "57041", "23177", "44782", "58804", "53693", "52929", "29398", "53076", "50482", "57792", "16328", "49738", "4482", "56390", "38196", "46821", "53575", "58622", "21856", "43043", "29926", "22357", "28854", "7177", "23950", "6092", "38360", "6549", "10589", "21086", "32076", "56689", "29171", "23352", "15209", "40368", "6459", "32656", "31165", "16927", "34374", "56348", "42413", "28834", "51912", "17363", "22926", "50581", "15798", "19247", "2569", "35037", "17603", "46433", "10875", "49291", "43324", "45564", "37239", "23673", "38073", "48020", "19461", "15829", "53743", "23901", "46322", "14379", "56243", "58952", "26635", "7105", "23057", "302", "17875", "10920", "11578", "3584", "46215", "50951", "3165", "23730", "25938", "40504", "7004", "27279", "3415", "53387", "9661", "37275", "11444", "56375", "14856", "51295", "20812", "22114", "35774", "7506", "47553", "42835", "979", "27499", "7140", "56193", "39618", "17149", "48329", "25953", "5056", "18203", "2687", "22904", "54753", "1962", "26971", "38499", "14834", "43145", "6632", "58630", "36383", "42047", "41768", "909", "19533", "28297", "20284", "24338", "21172", "38349", "45290", "20616", "16079", "12507", "22544", "50878", "24188", "971", "39901", "43593", "48468", "34085", "24740", "46232", "36977", "19022", "16591", "55277", "10353", "50369", "57546", "2974", "14527", "32636", "56563", "56308", "3665", "21953", "21988", "23275", "50807", "54750", "41907", "53951", "54308", "45852", "4962", "21944", "2788", "57138", "37842", "8051", "33571", "15929", "25478", "36082", "45631", "6866", "22281", "21412", "54605", "23396", "30514", "25710", "27823", "37484", "31351", "46909", "52018", "8887", "9132", "20612", "35751", "13381", "25122", "8378", "14742", "59705", "58065", "12791", "24998", "57242", "41094", "59833", "28784", "17046", "10668", "18001", "49779", "4246", "8130", "37452", "46329", "57291", "37689", "59350", "10438", "45594", "38457", "16440", "20883", "43622", "48764", "31823", "57306", "26167", "35766", "17935", "47822", "55967", "1114", "26150", "48897", "40515", "34514", "48146", "31736", "45109", "38233", "15746", "58943", "30096", "52464", "44053", "1494", "6112", "5125", "427", "9324", "59892", "41034", "12193", "3659", "37691", "8124", "29766", "24586", "31803", "43380", "26595", "15660", "7822", "57874", "40060", "40373", "39318", "41124", "45177", "50813", "32642", "21733", "53384", "18372", "50006", "4882", "10535", "2880", "26271", "33694", "50692", "35596", "21450", "54708", "55533", "55087", "33177", "58012", "50077", "11570", "46919", "54779", "10974", "33333", "6369", "16430", "13841", "37671", "29501", "30914", "33689", "17328", "35278", "55331", "19925", "45065", "15402", "52422", "17037", "59561", "20002", "16594", "18658", "38569", "462", "21842", "26197", "29100", "28098", "56109", "26161", "52314", "14294", "25816", "41347", "12695", "32980", "7941", "32241", "11585", "59338", "27731", "38533", "17781", "31605", "6363", "17568", "55096", "28341", "18737", "8787", "42670", "31494", "8467", "13123", "51850", "29679", "45215", "54535", "22301", "31087", "4939", "29185", "49303", "39167", "20525", "28367", "36239", "29670", "51149", "48541", "6073", "9166", "35578", "36654", "1505", "23816", "43548", "31019", "14426", "47550", "9637", "19123", "7533", "24058", "31901", "43503", "5330", "2708", "47156", "53330", "59631", "20312", "3077", "991", "17115", "17631", "4259", "56016", "44415", "41032", "27549", "18231", "30087", "13806", "34979", "42152", "16453", "53361", "12082", "38985", "12418", "25682", "26898", "2641", "32309", "3688", "29043", "49494", "59170", "58982", "12330", "54052", "547", "10687", "30493", "32154", "42863", "27199", "44257", "50008", "19655", "19014", "59325", "19295", "16603", "27539", "18606", "55853", "49823", "48339", "49184", "35674", "21515", "39880", "11078", "5784", "50634", "25592", "12843", "33797", "59831", "20956", "44994", "56105", "10622", "50196", "4245", "46777", "42728", "48611", "28399", "52249", "25298", "34235", "21371", "2672", "1416", "7375", "40593", "35574", "20736", "18347", "57740", "44685", "20132", "19716", "15127", "39464", "3106", "22820", "1524", "51599", "25321", "49196", "23535", "29612", "44956", "22753", "55461", "26343", "11673", "28282", "25727", "45049", "14815", "53737", "53662", "58415", "52280", "29261", "3368", "36720", "57081", "52256", "14602", "23988", "42179", "26077", "43691", "45584", "34584", "54062", "7151", "56818", "56021", "48136", "58612", "3223", "45321", "16735", "53731", "57799", "15371", "476", "34883", "24344", "19195", "9690", "12297", "49870", "27067", "8191", "14293", "45084", "13429", "58927", "13532", "43954", "58371", "53386", "38275", "43872", "51196", "40289", "13840", "59686", "49943", "42000", "3422", "15385", "19766", "26306", "25510", "47611", "49210", "50280", "12285", "25258", "47349", "46587", "7044", "5305", "26756", "11429", "50859", "39313", "28682", "26251", "22982", "46906", "54681", "25780", "38828", "40561", "55090", "15314", "14529", "35818", "13646", "31548", "52571", "1523", "36743", "34808", "24503", "10810", "41122", "36700", "20818", "50652", "48676", "5701", "21880", "42236", "44989", "20414", "23683", "27289", "15982", "16918", "17164", "47364", "40098", "24075", "12943", "23655", "19679", "26192", "33599", "10341", "53193", "21985", "51227", "39526", "31979", "49995", "31701", "25333", "22733", "1418", "56468", "58066", "48249", "9584", "57503", "49029", "16265", "28103", "33839", "10949", "5749", "58595", "17521", "21811", "34685", "27011", "30941", "13704", "1687", "13225", "39457", "19997", "10765", "25058", "50148", "37274", "1412", "52435", "10500", "30075", "35849", "49982", "47522", "23768", "11945", "9045", "33774", "56695", "50112", "12196", "41617", "1776", "54023", "17851", "12486", "40750", "35268", "34373", "45934", "24061", "40716", "35313", "299", "17782", "55841", "11555", "53335", "13180", "49594", "26702", "19163", "6538", "35286", "10367", "29219", "1533", "10608", "44469", "44322", "18396", "7112", "10191", "1704", "54423", "16589", "21865", "31681", "5500", "48315", "13253", "23085", "4318", "25651", "16375", "24305", "35912", "899", "19629", "1868", "25369", "9838", "10660", "2055", "24969", "58040", "49080", "52275", "52415", "27277", "56975", "43508", "23037", "20588", "3869", "24937", "11013", "36624", "45064", "13024", "32332", "22675", "36866", "59015", "10282", "48125", "35134", "20685", "39436", "57440", "34057", "31336", "45210", "49750", "4719", "5826", "59314", "8342", "28826", "36627", "6994", "44370", "10025", "48204", "3155", "2682", "12732", "33186", "26235", "46384", "27970", "58909", "25216", "46823", "48355", "16433", "4436", "53327", "37332", "20943", "35627", "50191", "10838", "33581", "34698", "37760", "44724", "21927", "21770", "30664", "25778", "14085", "59829", "33109", "54221", "39432", "4567", "6696", "55071", "9208", "54805", "41170", "36752", "1055", "57069", "41254", "42477", "18156", "9854", "11199", "2909", "39742", "44764", "58646", "16528", "38012", "25924", "55004", "42730", "42523", "31734", "51548", "25091", "22056", "23642", "29885", "27570", "22902", "52642", "12727", "6104", "31294", "11983", "30071", "24934", "56073", "27413", "21633", "7246", "41551", "35610", "25594", "24018", "13155", "51132", "16030", "41863", "1287", "48651", "42698", "26280", "20068", "56814", "26982", "12904", "19793", "43337", "22689", "51324", "53727", "58140", "38948", "38660", "23972", "53127", "22682", "24677", "56631", "24922", "45611", "23706", "25841", "42862", "23600", "24626", "53488", "19970", "47723", "55970", "46992", "21357", "40109", "29749", "30050", "19081", "37614", "51058", "34957", "35300", "34956", "34668", "9799", "21229", "17251", "24813", "11952", "57185", "6844", "119", "3174", "44282", "59753", "13394", "31826", "45025", "47270", "34977", "42635", "23503", "52516", "54983", "49720", "46807", "11309", "16311", "32746", "6996", "51272", "58150", "25587", "47796", "23871", "17736", "41599", "36002", "20724", "59987", "57846", "22275", "57053", "27420", "30840", "776", "32361", "14869", "40220", "43764", "35539", "8849", "621", "50025", "53168", "19458", "31546", "41975", "12835", "40857", "43428", "58163", "49235", "27208", "18607", "58255", "19682", "41244", "15983", "16765", "58144", "18641", "24031", "9549", "31374", "5779", "45229", "53192", "2542", "16331", "17913", "55035", "5117", "48409", "58233", "20097", "38310", "24195", "59043", "27312", "55496", "14674", "26615", "46425", "53891", "26133", "5833", "3454", "39", "2855", "55826", "59347", "43807", "32645", "6581", "21939", "11002", "34711", "31798", "38070", "1501", "41556", "2044", "41263", "3097", "43671", "52696", "14456", "32851", "38345", "49677", "34924", "1760", "14012", "27458", "40715", "28689", "53166", "46998", "14585", "25060", "27475", "18788", "3969", "56420", "50943", "19962", "36663", "47244", "57271", "19264", "25660", "55094", "130", "51233", "15097", "56454", "57660", "33152", "10058", "51008", "51382", "16132", "13828", "10996", "43617", "7458", "30307", "41904", "18808", "13535", "18127", "27879", "38444", "20215", "59395", "35252", "25073", "18894", "20052", "30452", "26202", "21552", "16228", "52279", "23864", "54118", "6619", "57876", "23741", "6461", "53417", "48592", "20809", "42885", "34442", "12652", "49833", "2766", "38845", "43836", "19803", "21595", "34575", "55043", "45560", "16018", "49125", "24208", "43657", "14557", "46616", "36671", "24147", "8627", "25325", "34161", "32271", "51889", "21961", "53877", "16991", "1468", "43946", "27946", "26633", "4086", "52494", "40394", "55296", "5335", "17960", "57891", "38158", "8649", "49841", "52627", "24836", "56688", "53861", "17926", "51977", "28756", "51785", "8058", "29696", "34804", "47069", "43374", "31788", "44227", "8714", "28591", "29979", "52438", "13446", "52377", "33269", "15975", "16743", "26001", "24076", "49110", "20131", "12778", "56843", "19222", "11052", "19851", "42559", "39479", "46755", "36897", "22379", "52911", "19738", "16721", "29483", "59636", "28626", "19544", "47935", "6834", "4281", "53544", "28202", "28483", "46279", "41093", "52000", "37717", "1604", "8028", "5084", "40225", "56639", "25884", "49768", "53869", "30677", "16717", "20503", "1751", "54672", "22927", "143", "5769", "58481", "5170", "8925", "18760", "46620", "52984", "25041", "24438", "6399", "1350", "52759", "23090", "29611", "42060", "38387", "41772", "7834", "37013", "20237", "3741", "40657", "29980", "8712", "31848", "58719", "16798", "46096", "11166", "19607", "6848", "15947", "39480", "9031", "44294", "52650", "3295", "2056", "59327", "55729", "5369", "44629", "20079", "6072", "19064", "48363", "50222", "48779", "7124", "17761", "40409", "2176", "25048", "45698", "23702", "41628", "11247", "41957", "26556", "30114", "21661", "52135", "24742", "36074", "19822", "8978", "35299", "4", "43438", "20854", "45678", "7165", "38126", "30872", "46589", "23867", "30164", "43897", "40847", "43995", "43208", "11934", "27403", "56785", "7879", "31037", "27352", "44031", "14280", "43888", "45218", "29659", "22633", "479", "50757", "16262", "4288", "23227", "48027", "9765", "41422", "27546", "19156", "49283", "21758", "49730", "13782", "15413", "59577", "43898", "5053", "25542", "55055", "50596", "53646", "48842", "12701", "35663", "10773", "21393", "20285", "21085", "9017", "754", "4313", "45078", "25374", "55258", "32333", "18844", "59309", "31485", "23979", "14548", "12052", "54251", "17542", "31832", "39646", "44845", "56807", "2302", "1565", "19359", "25394", "44244", "7789", "576", "56483", "57865", "39614", "16258", "3650", "14918", "17524", "47986", "14081", "42099", "56020", "17216", "44487", "19779", "41315", "13743", "42687", "9057", "27666", "53975", "27233", "56238", "20317", "6946", "32165", "31888", "45222", "30425", "55968", "19004", "21031", "22997", "201", "50390", "22635", "28172", "34764", "11920", "17491", "21074", "51538", "32133", "53597", "1276", "10989", "42505", "23010", "14385", "56720", "30508", "35658", "46091", "16359", "12171", "50314", "31740", "35087", "17871", "40793", "29454", "5510", "24772", "26988", "49844", "50583", "20852", "36650", "3059", "32675", "3515", "21711", "28109", "16255", "5589", "11670", "49680", "20910", "45450", "6362", "11496", "894", "59376", "33954", "57019", "45835", "10574", "23500", "40903", "23680", "19888", "17467", "42335", "12282", "29796", "8956", "25824", "31186", "5963", "38391", "32791", "58393", "32608", "4293", "15764", "26731", "55394", "14761", "38874", "34177", "17907", "16343", "20817", "46866", "59291", "2946", "3424", "16088", "44887", "42198", "47388", "15716", "35466", "20319", "2113", "3919", "18635", "48057", "51532", "11743", "54156", "4243", "55037", "11939", "31258", "2938", "1749", "44456", "55741", "26744", "16834", "58375", "1814", "1102", "31285", "9396", "36318", "49809", "42411", "13291", "12642", "34107", "3078", "41139", "34523", "31461", "51071", "40881", "49671", "50782", "41494", "13722", "44373", "6979", "49798", "29828", "30709", "23264", "57726", "36989", "8081", "52916", "11575", "12689", "30382", "41610", "53943", "9418", "38963", "20247", "27155", "16318", "13335", "13832", "49709", "31512", "46375", "31489", "43309", "44617", "18814", "32382", "37608", "33228", "19610", "28975", "29901", "50533", "30972", "43857", "3192", "20443", "13211", "39524", "33391", "3148", "38728", "16117", "13103", "22027", "47024", "16519", "9377", "47130", "3229", "45660", "26034", "26538", "38956", "52418", "6221", "53796", "42140", "19395", "28986", "32177", "44127", "40198", "24993", "5430", "42573", "47827", "19475", "38661", "24367", "32224", "34794", "31968", "4850", "11775", "58388", "23467", "19418", "46213", "45439", "6641", "31421", "23004", "36405", "14693", "20769", "27702", "10809", "47393", "42226", "4590", "54527", "58978", "4937", "2423", "59026", "29598", "39234", "38059", "25566", "2871", "19628", "19443", "45508", "7230", "22838", "22939", "31223", "50548", "26882", "9544", "43383", "11518", "21251", "53881", "31224", "28794", "32134", "48794", "44338", "4953", "2668", "20419", "27414", "36139", "36574", "52812", "3026", "52490", "12909", "41377", "57231", "17786", "50610", "58222", "45220", "17591", "1929", "55242", "5960", "42699", "32780", "26064", "8724", "16204", "1007", "11212", "20728", "1236", "28765", "44541", "15021", "41256", "2584", "22286", "12378", "56124", "2282", "13509", "13503", "12385", "9887", "17739", "59405", "1876", "20917", "5381", "15701", "20329", "4814", "18603", "51844", "17238", "14393", "53813", "37789", "47157", "18539", "3971", "42997", "43572", "55839", "27523", "44929", "23712", "40974", "28721", "15312", "34397", "18374", "16332", "49045", "10175", "53370", "20981", "45212", "10092", "32214", "15626", "48911", "10896", "2993", "39943", "54698", "15847", "26741", "5233", "7474", "23305", "30047", "11461", "12466", "51552", "17514", "7802", "45186", "21858", "24791", "53265", "38598", "36217", "37381", "16907", "41856", "35085", "15805", "59584", "39903", "33116", "54363", "47084", "14919", "30083", "23193", "57901", "19335", "2997", "28920", "1202", "8048", "22002", "15732", "12915", "9891", "35171", "23933", "11468", "33043", "17602", "33291", "3876", "55509", "55207", "22745", "4165", "28067", "54419", "29876", "34477", "15034", "28137", "2621", "14306", "45932", "24025", "10728", "40271", "50223", "4713", "10273", "24882", "32705", "8053", "59883", "51440", "45937", "35886", "28014", "23195", "55819", "28985", "16392", "15918", "3621", "15628", "7337", "36535", "7500", "16656", "40068", "9437", "7132", "17832", "21347", "12192", "51391", "26295", "30876", "4252", "5446", "46961", "390", "52704", "29672", "7353", "58858", "30351", "17054", "38672", "33310", "7696", "9020", "10308", "23669", "19256", "46766", "43707", "8914", "58170", "49467", "15172", "38250", "39668", "42455", "3752", "35581", "34985", "42594", "58392", "43011", "22915", "19029", "49879", "50351", "8829", "16046", "35496", "10645", "40069", "30259", "13397", "26508", "53656", "31002", "42995", "49571", "9744", "33710", "55763", "40128", "56181", "47835", "55557", "12548", "55934", "37482", "51463", "43093", "44203", "28596", "7675", "43903", "10979", "18443", "23237", "2760", "46535", "27491", "6193", "32610", "59713", "52068", "40598", "24351", "24139", "3283", "19223", "43985", "44510", "51528", "13858", "11536", "29908", "22318", "42869", "13791", "51285", "40468", "625", "10028", "59516", "34994", "24370", "36283", "41561", "22312", "47798", "40284", "3815", "54618", "36812", "4312", "55964", "8711", "48242", "17082", "18155", "7984", "34183", "38505", "33231", "46164", "862", "4450", "16520", "2579", "34389", "25295", "11866", "46389", "24451", "26018", "28401", "52999", "26333", "15164", "33429", "5401", "20351", "13575", "13609", "43875", "45627", "33526", "37928", "13367", "49272", "38307", "12085", "59186", "23351", "12672", "19167", "8785", "58866", "6164", "49005", "3487", "21763", "21176", "39047", "52344", "14084", "9266", "31005", "23792", "53542", "58669", "33443", "32822", "74", "25357", "21133", "17466", "27562", "14325", "51747", "47071", "54930", "24002", "15897", "28368", "50781", "56892", "54876", "34748", "53923", "30951", "48609", "23556", "17964", "53247", "898", "54787", "58265", "45633", "11307", "24887", "41517", "43891", "45935", "44353", "12222", "41737", "40004", "36059", "22668", "53819", "37613", "58785", "53103", "24883", "35980", "11968", "13867", "41864", "22952", "9686", "19074", "48229", "12362", "49962", "42712", "34860", "17413", "38662", "21378", "34914", "59143", "8394", "25255", "17861", "36091", "19016", "25616", "19208", "53189", "22872", "9388", "4269", "27265", "10530", "43843", "26390", "24711", "6774", "12636", "7909", "25708", "167", "12402", "10708", "37507", "59240", "46927", "55955", "30649", "47514", "42685", "48938", "38787", "24347", "58417", "30407", "6103", "54402", "13796", "56944", "52010", "36858", "26255", "47561", "54115", "10138", "48798", "21871", "27929", "45138", "25741", "59754", "31102", "39667", "50175", "28", "19120", "57675", "487", "55746", "36768", "53559", "1285", "20989", "48963", "350", "29178", "29562", "31660", "17389", "8995", "47469", "44278", "6382", "51776", "20019", "53211", "3402", "32337", "5575", "49696", "12448", "1284", "24140", "49099", "11392", "12935", "19797", "5834", "12907", "38098", "49670", "21057", "19722", "48904", "32931", "16829", "19625", "50411", "37068", "59114", "188", "14747", "43586", "21249", "53503", "27836", "57193", "12037", "31253", "7038", "35546", "58617", "50527", "41307", "41011", "36199", "21242", "49998", "2887", "5597", "1707", "1138", "14880", "8166", "29717", "30654", "22470", "16223", "10872", "15616", "42650", "4157", "50906", "24237", "51365", "7025", "55039", "36668", "14348", "22150", "45672", "22393", "51297", "27740", "31371", "54999", "43823", "27068", "26112", "8985", "33343", "38148", "49328", "50471", "6625", "21546", "1873", "45591", "10137", "37852", "5269", "57518", "6599", "34474", "55894", "39132", "15294", "11158", "44471", "23958", "3960", "50869", "24884", "20251", "20011", "22091", "27890", "38813", "1031", "56600", "17469", "23927", "21153", "33729", "3660", "5713", "56599", "40175", "1133", "39418", "36790", "25701", "41138", "5968", "16475", "49105", "19917", "27653", "34984", "34950", "39917", "51239", "55408", "12602", "54564", "8179", "35781", "19160", "17416", "40788", "48658", "20228", "9802", "43240", "10178", "45117", "46440", "3199", "35750", "41125", "53875", "59587", "55651", "7077", "19122", "45094", "54608", "29170", "40472", "53864", "52522", "36878", "7724", "14192", "51351", "46013", "10066", "35991", "29988", "56962", "25817", "44485", "12221", "10766", "56113", "34083", "47580", "45056", "20735", "54830", "44256", "54464", "8009", "27269", "19463", "18368", "6933", "13422", "19651", "34730", "57169", "3317", "29163", "18953", "45158", "22191", "8893", "48331", "16442", "17456", "59633", "53692", "21239", "42796", "27074", "30305", "31897", "41466", "45350", "14237", "39010", "6290", "53618", "35484", "33851", "29496", "43828", "19609", "12091", "8410", "46541", "14625", "55269", "50993", "4062", "14567", "35613", "10074", "16372", "20099", "43375", "8086", "28091", "30066", "58734", "54585", "22082", "8159", "25219", "8602", "35416", "9610", "57754", "38074", "32446", "5279", "16312", "44176", "23806", "29709", "29803", "41165", "26629", "50734", "36757", "47049", "35005", "53591", "42977", "4821", "533", "57340", "27478", "4059", "14896", "26136", "46506", "41092", "43150", "18535", "45620", "24921", "21408", "47630", "23074", "51074", "39357", "54296", "16621", "39394", "3703", "5726", "27795", "53617", "17016", "9468", "44111", "43682", "18171", "6331", "13820", "7585", "18008", "45593", "5573", "38730", "26108", "57517", "25121", "32205", "49035", "40299", "39339", "31581", "46946", "16523", "7117", "46154", "57721", "52128", "51818", "46708", "24014", "22152", "55889", "11519", "12947", "32075", "25519", "54494", "34516", "28574", "9843", "22598", "12006", "19442", "8453", "7849", "41415", "38498", "52839", "39338", "39708", "16811", "36532", "384", "25171", "27379", "16613", "59317", "53521", "55879", "23734", "34425", "7314", "2594", "37092", "2613", "28783", "22971", "35915", "37994", "32155", "29450", "38123", "55398", "40191", "35809", "36505", "27531", "17652", "38162", "58635", "31924", "43040", "47009", "14006", "33919", "24326", "15326", "15375", "56961", "44329", "35568", "56937", "9996", "21275", "54128", "19125", "25231", "33592", "23962", "29198", "56824", "35680", "34875", "59207", "11667", "49288", "51906", "5122", "12128", "4017", "53102", "57863", "33856", "36692", "50378", "23613", "15369", "16023", "59696", "8935", "59474", "11238", "52810", "18677", "49994", "54532", "1286", "56816", "3909", "45303", "16979", "54304", "13752", "42234", "23737", "55569", "47763", "10", "33675", "59828", "6380", "15928", "22309", "40229", "45782", "1160", "15115", "2278", "19574", "12686", "15461", "49753", "14752", "3345", "10042", "21296", "13862", "2868", "45041", "12183", "32351", "47961", "8731", "51288", "24801", "46188", "7767", "8069", "19462", "15048", "37788", "33674", "53499", "46250", "14920", "50762", "7715", "12215", "30869", "15524", "10676", "40733", "2567", "51976", "50407", "7837", "26555", "14810", "41288", "29199", "17441", "28888", "57680", "59191", "30146", "15381", "26662", "1644", "34819", "52361", "7116", "47914", "46060", "43130", "24050", "1849", "11364", "41484", "51162", "36850", "3474", "2028", "447", "31612", "54233", "51686", "30398", "22380", "45789", "20871", "25645", "19385", "51122", "26480", "40273", "37050", "38058", "48930", "14239", "37419", "48924", "7999", "4562", "39813", "55975", "21283", "10244", "54091", "59236", "22731", "39898", "15231", "46690", "179", "53017", "20231", "19314", "20056", "52947", "58637", "12066", "16487", "58525", "55195", "2700", "48752", "34607", "10395", "54962", "37178", "47428", "14521", "6754", "327", "46266", "3902", "11662", "57413", "318", "87", "7254", "48711", "16826", "11151", "25296", "8616", "25336", "25777", "56458", "27578", "21628", "23007", "24744", "56487", "41913", "2571", "5348", "10334", "40315", "19869", "42866", "57495", "24306", "2127", "31865", "59198", "23246", "5928", "18293", "29785", "15189", "7426", "32893", "16485", "58707", "54395", "25054", "23934", "16072", "39068", "55173", "48031", "43694", "58433", "415", "22265", "21506", "24489", "20364", "28503", "18278", "55829", "6970", "19278", "12749", "57724", "29141", "9412", "9638", "15884", "3134", "18228", "27329", "5440", "55998", "19139", "17335", "49455", "28974", "45137", "28607", "6747", "48220", "38866", "46118", "32162", "54329", "2688", "28725", "37605", "11808", "23030", "26436", "42173", "54650", "20771", "25995", "55984", "54459", "671", "59110", "58364", "11847", "6368", "59897", "51595", "53169", "4753", "25769", "12844", "29137", "15851", "28505", "11413", "3714", "31007", "23021", "43288", "19989", "59085", "43912", "9734", "21860", "54661", "50048", "28199", "54053", "22878", "12998", "27322", "50561", "12067", "43486", "38912", "8304", "33098", "14556", "33556", "3538", "7106", "48144", "34763", "26291", "20566", "2617", "7329", "20639", "41535", "2379", "46793", "1736", "43507", "6162", "9220", "14270", "48882", "33736", "46624", "51319", "39367", "1813", "52930", "18295", "15787", "32939", "28810", "57300", "34422", "18355", "12320", "26816", "7997", "19784", "39796", "37687", "31649", "18967", "16856", "34760", "52076", "11622", "1446", "24165", "49148", "39210", "33039", "50076", "31648", "16510", "55228", "57230", "59125", "40223", "30338", "22159", "53686", "34396", "53059", "41771", "51236", "12515", "34370", "56209", "15501", "48446", "44947", "50509", "44767", "47784", "41306", "42291", "48108", "23551", "5350", "46106", "56425", "25012", "6874", "50679", "40333", "13383", "53947", "29318", "7061", "49832", "58379", "17552", "19705", "48817", "14683", "22623", "19499", "57571", "9111", "25197", "35519", "42981", "21154", "6803", "57794", "34155", "25440", "29015", "32684", "8992", "50399", "3819", "56778", "9281", "16209", "2858", "17530", "50276", "40066", "7769", "33322", "26473", "1188", "10350", "56451", "43344", "12941", "36494", "54192", "17688", "13657", "12625", "48391", "6537", "59868", "19302", "40448", "46756", "56593", "58548", "59365", "49792", "30744", "55437", "24763", "30332", "2448", "13775", "35182", "16962", "35743", "8108", "34423", "55640", "4512", "23549", "51411", "22557", "25009", "38972", "8514", "54949", "36329", "46530", "29051", "22848", "31450", "44573", "21853", "4389", "35903", "17129", "11564", "57541", "24135", "4321", "35460", "37157", "51290", "58410", "34941", "46484", "5315", "35325", "23672", "24871", "5036", "56841", "31275", "2510", "59623", "18371", "27998", "40972", "17406", "25104", "45097", "31948", "30978", "41344", "1208", "15707", "15473", "36980", "24106", "32364", "12852", "49495", "57789", "25610", "59820", "48134", "7510", "38436", "2550", "3822", "18116", "32473", "41713", "6856", "24340", "14566", "50682", "31654", "9425", "22096", "11339", "42920", "35390", "20965", "44423", "37305", "12716", "1845", "3478", "50180", "18412", "32663", "18692", "40623", "5089", "35815", "25179", "31743", "34847", "23939", "17998", "11123", "18717", "41296", "10789", "37279", "45792", "42083", "24262", "12744", "7988", "47902", "21525", "44114", "3243", "59550", "21442", "41903", "18924", "50938", "56517", "42066", "22182", "14711", "24683", "21349", "44091", "35111", "13573", "4568", "21977", "5787", "7588", "12546", "25596", "18002", "26848", "25174", "30320", "36630", "51555", "44835", "11563", "2189", "3177", "46239", "33997", "3346", "54587", "34010", "2358", "35056", "31686", "59722", "15506", "4525", "21220", "40809", "10626", "38453", "37487", "26998", "13246", "34119", "9753", "1738", "26445", "43631", "57752", "22510", "38595", "33444", "38294", "3202", "30856", "15339", "23865", "40332", "24456", "43167", "52144", "52845", "52335", "51107", "18112", "27376", "15762", "56618", "18591", "36936", "2477", "12731", "17565", "42815", "36233", "48372", "27166", "54082", "37867", "38104", "15119", "35861", "48416", "2174", "1602", "52198", "31120", "28755", "1357", "49245", "48828", "51579", "50614", "21475", "28156", "36676", "53687", "11283", "48572", "4645", "43765", "11432", "41541", "7976", "25363", "46994", "26550", "35934", "57529", "44718", "36507", "36175", "44534", "55644", "29841", "37472", "59536", "34184", "10442", "51494", "40202", "47152", "33876", "7189", "38253", "8788", "28200", "50187", "29742", "27842", "47576", "29305", "49083", "34404", "21612", "28306", "32928", "51091", "36046", "39198", "43427", "32121", "36947", "29680", "14002", "53297", "41989", "51360", "10306", "56114", "16159", "21908", "5594", "48602", "50625", "9794", "41252", "34071", "46873", "44840", "13430", "39633", "17931", "29050", "38324", "7188", "44619", "43283", "39531", "49997", "59890", "23369", "48042", "6901", "32804", "14022", "49012", "32470", "17330", "2685", "38716", "4335", "8044", "43709", "43242", "39522", "55696", "7430", "2620", "1648", "56753", "19934", "18624", "45858", "26973", "22749", "40211", "42585", "34570", "48631", "40430", "42266", "8696", "5115", "10124", "49948", "10865", "51615", "41890", "50842", "8351", "46287", "36436", "13420", "18113", "48260", "23172", "27092", "29806", "11422", "55890", "5681", "59993", "12247", "13299", "40217", "29397", "23708", "13102", "23286", "49538", "7556", "31047", "33951", "24676", "52399", "27853", "6766", "40496", "24949", "22163", "10227", "41134", "37942", "43570", "49449", "43191", "39936", "15766", "36978", "40632", "16840", "9179", "55881", "19037", "25684", "49577", "54804", "11081", "26147", "25099", "11737", "2407", "46327", "9518", "22336", "16849", "57744", "12739", "1198", "19570", "34829", "47332", "29589", "22516", "12502", "13956", "30179", "42584", "54540", "46718", "46236", "6419", "16970", "23557", "9219", "7284", "59292", "46545", "56176", "25092", "5622", "11234", "34558", "44825", "44522", "28670", "47018", "15802", "49413", "29243", "49075", "38145", "1451", "16206", "31072", "35817", "55962", "39261", "10883", "50738", "28936", "46204", "42959", "7128", "35486", "6852", "36289", "10408", "59639", "49386", "52039", "36319", "5359", "5915", "32001", "4045", "54377", "12792", "27923", "21687", "56304", "21735", "20792", "7729", "25150", "23729", "28011", "20668", "27223", "2823", "28284", "29269", "33142", "37972", "31182", "35846", "54559", "56198", "52675", "51193", "27947", "39776", "18653", "36127", "12573", "22219", "19594", "20573", "34970", "33906", "56752", "31744", "26442", "424", "4800", "4615", "46913", "46685", "15697", "6875", "9304", "11273", "55613", "1568", "21697", "15322", "46618", "6644", "40831", "7043", "21768", "18564", "54011", "43264", "34087", "14950", "59805", "30003", "53336", "22955", "58096", "47206", "36213", "7658", "51015", "31620", "26917", "17126", "26261", "11821", "41412", "32406", "15871", "11612", "44948", "9684", "52803", "25640", "37073", "27682", "36029", "3559", "34751", "18885", "10808", "7231", "35084", "4911", "57690", "10618", "322", "35975", "41111", "50604", "32303", "46654", "11911", "52670", "5522", "631", "6692", "744", "2648", "54287", "18959", "43407", "26413", "12150", "12779", "24945", "51604", "14633", "26469", "29350", "4018", "6608", "10068", "43351", "20896", "26084", "38214", "36939", "50129", "56347", "38839", "22136", "33705", "38144", "49775", "44962", "41901", "14224", "59190", "23760", "20124", "481", "19780", "33905", "46848", "6269", "33089", "17173", "39497", "6200", "16806", "14077", "50761", "38218", "19416", "22471", "18317", "48319", "31076", "4287", "54490", "40081", "13860", "41151", "3233", "17095", "48715", "57272", "59625", "57161", "25278", "42432", "53989", "23403", "37624", "8483", "11209", "50985", "21589", "53647", "59149", "38088", "50370", "27515", "56336", "29115", "1439", "38493", "35712", "807", "38317", "21792", "37849", "25966", "43768", "20040", "31462", "51668", "17451", "20095", "41033", "32396", "44391", "5304", "18106", "3771", "36126", "32611", "545", "55332", "41060", "59080", "5962", "40605", "237", "47858", "33727", "12766", "54143", "12160", "43211", "22401", "9260", "49644", "20791", "19166", "27415", "53062", "31473", "37638", "40573", "33826", "59453", "40131", "617", "555", "33722", "13958", "19098", "21021", "31553", "5907", "19477", "21672", "40137", "4775", "50674", "16309", "45669", "56980", "30700", "43803", "38178", "35764", "7323", "30291", "9539", "34248", "4009", "18676", "51828", "44010", "13060", "23674", "42986", "20764", "46502", "20893", "58352", "31384", "1630", "34960", "54124", "46845", "47899", "24017", "36225", "23353", "18094", "15625", "24958", "7417", "7046", "41719", "22074", "20765", "29374", "14336", "1405", "12206", "33017", "28703", "58650", "50565", "38772", "51080", "10546", "13343", "30310", "36944", "36945", "38570", "36568", "17497", "13493", "23893", "20700", "38109", "32714", "19619", "5719", "16995", "44160", "16809", "4132", "54934", "23357", "25739", "55265", "5143", "28750", "38976", "5414", "32704", "26180", "25264", "54740", "17043", "810", "52559", "5514", "49452", "42535", "33651", "19003", "19424", "11032", "40210", "12752", "801", "20592", "10355", "43884", "54216", "30337", "22579", "17962", "24564", "17793", "39644", "1162", "1110", "26290", "38829", "18684", "43780", "36763", "59141", "47541", "8419", "21369", "50033", "18705", "51090", "12114", "24638", "21303", "5868", "31066", "19240", "32118", "20092", "48825", "12024", "41811", "49603", "8265", "14672", "22215", "19936", "40492", "11703", "36270", "52747", "46559", "9808", "22624", "34028", "6179", "6272", "20584", "6694", "33657", "15711", "28695", "46128", "36840", "33240", "11941", "56779", "50720", "28268", "49937", "56282", "21550", "25647", "50719", "17883", "18656", "20559", "4949", "42385", "40745", "40639", "5916", "40078", "36749", "9664", "5057", "31079", "49177", "40998", "33022", "30844", "30955", "12045", "57464", "43832", "51854", "27039", "48264", "22644", "39063", "36023", "58445", "58777", "43300", "4827", "53140", "41207", "16972", "49151", "50198", "23819", "9619", "9818", "33693", "39897", "45476", "43198", "8077", "21193", "5631", "45016", "11045", "58305", "51361", "7305", "27461", "36349", "12112", "11448", "45763", "48182", "32393", "10210", "20367", "6864", "55642", "8927", "41569", "51113", "54980", "27545", "6055", "29502", "35775", "8230", "52448", "18900", "32388", "885", "50271", "28412", "27408", "30787", "16640", "28688", "41331", "40253", "6188", "17683", "58589", "16846", "13913", "58237", "21751", "7131", "32108", "40813", "31870", "40869", "9755", "13350", "52760", "28745", "37248", "34245", "18224", "54761", "9647", "42487", "16746", "13192", "59783", "29751", "6306", "27401", "47358", "46727", "10760", "3964", "5574", "21395", "30789", "44017", "42444", "1109", "25322", "19627", "39203", "11665", "49040", "13698", "24066", "25956", "58316", "10107", "43964", "44064", "39258", "29235", "56368", "56432", "2233", "52093", "2928", "36987", "20795", "43162", "23001", "16011", "15405", "38452", "19102", "48773", "37237", "11456", "46378", "11345", "53413", "39022", "7619", "36709", "18878", "27951", "58108", "29215", "21614", "38508", "59247", "23808", "40846", "25480", "24244", "32990", "48135", "48852", "57685", "54074", "57331", "37010", "1654", "11061", "17483", "58626", "50914", "7111", "41287", "41089", "47220", "22859", "2299", "14112", "15093", "30651", "58006", "23940", "48858", "56423", "56551", "35055", "48043", "37743", "27502", "19621", "1175", "28201", "26417", "31012", "7070", "40045", "51377", "22057", "52206", "20080", "45408", "32641", "39757", "49704", "23033", "15336", "51266", "14601", "999", "58343", "24535", "57701", "41932", "17461", "21588", "31629", "56234", "25719", "2280", "33271", "52277", "40732", "5488", "34935", "12694", "48989", "28437", "21179", "12612", "4636", "12382", "4576", "11985", "14606", "47952", "35798", "18883", "55706", "50875", "35693", "12758", "9457", "24334", "23807", "29533", "53432", "32883", "46158", "13971", "43215", "12000", "6873", "8591", "6682", "10694", "27540", "36649", "17930", "50193", "26294", "4331", "39064", "12092", "10432", "38380", "10000", "12341", "29030", "32558", "36419", "6665", "45862", "24047", "49060", "3336", "13557", "9522", "22430", "16506", "8246", "53592", "44817", "51250", "20339", "43090", "47806", "56529", "36971", "38054", "14753", "28032", "2487", "51483", "22610", "8034", "27098", "12227", "51408", "7990", "6623", "45184", "27661", "11666", "32016", "56395", "13005", "8572", "49316", "4869", "27235", "45964", "24081", "10481", "55981", "15297", "19229", "11180", "2081", "36941", "27161", "21066", "9447", "23296", "2209", "1619", "14552", "47216", "42565", "12053", "58328", "58309", "44756", "48181", "23217", "51308", "47603", "2724", "36916", "8737", "27716", "43077", "1971", "45069", "4265", "51511", "8891", "3668", "20652", "11277", "56747", "20800", "12605", "52469", "30676", "47669", "18630", "16963", "17146", "32522", "37087", "32869", "40731", "6373", "11096", "47252", "18042", "5933", "43909", "56172", "7276", "2860", "21376", "29365", "21710", "5447", "29994", "31959", "38602", "18969", "9666", "9736", "54628", "16978", "58607", "20551", "36565", "1187", "59168", "34809", "20976", "56486", "28350", "22692", "14974", "4958", "3624", "50213", "41040", "31315", "11904", "53667", "17584", "57839", "40475", "38480", "47743", "53016", "48887", "14795", "26719", "43430", "45444", "23830", "27551", "59398", "36554", "1561", "59575", "40713", "53741", "24078", "53218", "1515", "41024", "17350", "16550", "11271", "47606", "41575", "24858", "19091", "760", "54087", "5082", "4856", "30258", "54089", "40391", "48998", "26070", "59928", "23129", "2043", "53099", "51102", "26737", "39088", "12407", "10550", "59934", "27351", "35401", "23725", "3814", "32298", "23785", "47025", "43114", "16087", "36172", "28512", "12863", "41056", "342", "2414", "8866", "50976", "31561", "15230", "7471", "17209", "56692", "55555", "5484", "53308", "21301", "26042", "17721", "443", "35523", "49423", "24395", "7290", "32806", "28123", "23956", "22491", "564", "30262", "49152", "24541", "20761", "37456", "49951", "10672", "50694", "59362", "40362", "193", "8809", "50435", "40770", "50591", "45288", "40616", "39591", "50605", "55864", "1708", "28441", "30730", "40815", "26989", "25909", "51097", "48803", "13318", "51167", "10263", "28653", "16322", "52191", "43653", "9255", "36443", "38943", "41460", "17856", "41962", "57417", "9283", "20334", "321", "34263", "21120", "36868", "22032", "37330", "47246", "7979", "8699", "9364", "4901", "27020", "40853", "26270", "54243", "21208", "25355", "13109", "17509", "41619", "37346", "43128", "57715", "37730", "58640", "417", "53976", "2371", "24303", "29836", "51231", "5347", "39775", "10245", "7548", "36986", "9573", "39807", "41378", "13811", "21642", "57284", "38269", "12185", "56627", "49569", "19658", "32726", "49496", "39854", "24722", "26999", "44570", "26997", "15846", "14714", "39831", "35463", "14609", "38440", "6651", "49096", "48770", "36782", "16267", "47139", "56266", "40647", "56163", "36760", "59622", "31180", "53679", "58686", "12742", "42006", "25457", "8302", "52970", "32876", "25599", "56267", "25558", "7018", "22122", "28935", "21092", "27084", "734", "4677", "31410", "3557", "15107", "54487", "636", "45702", "3632", "29512", "40930", "19986", "3480", "39437", "57817", "9249", "29008", "49497", "24854", "23698", "22760", "44211", "52026", "29266", "25136", "20487", "11054", "11315", "38617", "1305", "43368", "25977", "26460", "20877", "11823", "58347", "19012", "58259", "58554", "42342", "26386", "52333", "6615", "39746", "29425", "18921", "38007", "41483", "47872", "58670", "49195", "27438", "59363", "26784", "13233", "33219", "25766", "56798", "5921", "21317", "18366", "26350", "52041", "58662", "31453", "48430", "19752", "26573", "21380", "54874", "3760", "48849", "4620", "40089", "46590", "1543", "14777", "6546", "53677", "59179", "34648", "52814", "34910", "12955", "55627", "23974", "52637", "53392", "6238", "48678", "2562", "19829", "25237", "11557", "22681", "37589", "34402", "11837", "13989", "25659", "35661", "43396", "55649", "26855", "34471", "49057", "11341", "39810", "30395", "41470", "4060", "31712", "22555", "7179", "44803", "53888", "35021", "29155", "4660", "47062", "46859", "54626", "9118", "24846", "15303", "6638", "25232", "56815", "6123", "41112", "16998", "28155", "32147", "30354", "57184", "1993", "39817", "20438", "1926", "2821", "22620", "27388", "12958", "27972", "47242", "41627", "31198", "57842", "51386", "30254", "22100", "32099", "42287", "34258", "42345", "2652", "12846", "14072", "12475", "2426", "53799", "45690", "52600", "34237", "4186", "48834", "45389", "51460", "27525", "9745", "17846", "24832", "47826", "5740", "59174", "20085", "42222", "33680", "41878", "58235", "41769", "7890", "55271", "16668", "34587", "55164", "52085", "15262", "1177", "55778", "31516", "43113", "7481", "41540", "32041", "39106", "35579", "3245", "25733", "14027", "20244", "5058", "12682", "2305", "53574", "6280", "1779", "42054", "32231", "56102", "34802", "34346", "8723", "8519", "50256", "28294", "54866", "37615", "35211", "56392", "44308", "12639", "11316", "1622", "4272", "41789", "25656", "16133", "55279", "45581", "16293", "34813", "44128", "6562", "50928", "41582", "45236", "16357", "15096", "46398", "39981", "40816", "45509", "3332", "27318", "19927", "6048", "49932", "28170", "22316", "58608", "41502", "46519", "34091", "43261", "1689", "11299", "32849", "48978", "36515", "13903", "27619", "32952", "14676", "42882", "28696", "37141", "48743", "46869", "55551", "1012", "49558", "28611", "40169", "36447", "32760", "22107", "17822", "51915", "42228", "28629", "56397", "35072", "1154", "5303", "25800", "47912", "53460", "41605", "8151", "8721", "11463", "56153", "36210", "17609", "13541", "3407", "43770", "49711", "16739", "38157", "28182", "16671", "59916", "9124", "56561", "52532", "36244", "49613", "15253", "461", "51410", "31415", "24575", "19260", "21338", "30557", "444", "11135", "54703", "53009", "38531", "21911", "10800", "48913", "21054", "3577", "33035", "18862", "32571", "56200", "31132", "27153", "39377", "44058", "8900", "23197", "58717", "58895", "43157", "25473", "42074", "27445", "36398", "55847", "25346", "17554", "6119", "7443", "40276", "46493", "9689", "34658", "251", "5728", "10484", "33606", "43376", "47293", "51730", "3520", "16397", "14094", "39958", "43485", "35449", "8231", "40452", "52934", "50772", "27629", "12497", "20519", "32171", "58021", "25683", "39277", "2752", "3932", "10710", "56074", "51637", "31342", "43455", "7966", "53069", "10776", "55852", "41242", "37773", "12487", "57277", "29184", "12789", "25439", "11182", "35153", "1137", "9554", "31349", "22936", "46810", "9422", "42107", "29357", "44150", "29153", "10261", "56384", "42116", "57395", "2405", "6542", "11397", "35583", "13028", "26169", "25468", "19576", "13414", "2800", "11263", "10386", "768", "48761", "31611", "15007", "54938", "46527", "35883", "7775", "52512", "44590", "33663", "12911", "30885", "48398", "895", "34755", "20625", "54224", "48011", "368", "6967", "48053", "29546", "39348", "56413", "47143", "7877", "50893", "11210", "48137", "6213", "37724", "34025", "4113", "6216", "41663", "32427", "59132", "30912", "12553", "53854", "34138", "49126", "38695", "14740", "44462", "58460", "10950", "31050", "8208", "28763", "7110", "18481", "41144", "32622", "19293", "56997", "10844", "21933", "3032", "31063", "39391", "44623", "11372", "46243", "1542", "50802", "38615", "44565", "8648", "43037", "11050", "49648", "7243", "57432", "56929", "57605", "42238", "55930", "37085", "54227", "4174", "32754", "4352", "24007", "3365", "59592", "17069", "56582", "53584", "54357", "18824", "14907", "6758", "39495", "59911", "28817", "59482", "42771", "2017", "54088", "12979", "55722", "3107", "59357", "28751", "34146", "17672", "55407", "39232", "35684", "49143", "1176", "55338", "19897", "4836", "57102", "52721", "33160", "20933", "47122", "19337", "19767", "24113", "59385", "54757", "1008", "4611", "17900", "49004", "51819", "3693", "41376", "30411", "6588", "44995", "54020", "29518", "13421", "39700", "26058", "59822", "22529", "34555", "19132", "6294", "50366", "21591", "4481", "20459", "2483", "57082", "6938", "20495", "29984", "44072", "13498", "18181", "59751", "34524", "20718", "14153", "27752", "9066", "40339", "6936", "59152", "18099", "19000", "26530", "45299", "38379", "40677", "20840", "39865", "33313", "13118", "54005", "26648", "44237", "48588", "18954", "48649", "22437", "26911", "5248", "17285", "46623", "15081", "29654", "50903", "23609", "57897", "56465", "30133", "6109", "15447", "43737", "18964", "861", "10864", "51972", "10343", "7126", "48675", "37809", "58767", "10621", "54274", "50030", "46185", "11525", "52546", "14441", "28521", "34976", "45549", "11152", "49003", "13658", "18511", "24721", "36242", "4004", "23748", "49769", "40859", "23428", "31232", "13927", "51567", "52859", "9616", "18738", "32634", "1221", "47039", "41054", "48275", "27700", "50919", "26316", "44364", "2539", "56743", "38676", "16919", "3037", "7717", "24241", "10036", "51904", "18211", "37989", "40907", "25947", "1826", "37719", "355", "57388", "53744", "29301", "4817", "44701", "52312", "11242", "29142", "47231", "51636", "2781", "15426", "34079", "4551", "59598", "39682", "8662", "8280", "5865", "59454", "4658", "46529", "36930", "48123", "41183", "56839", "21498", "46768", "22625", "2225", "23590", "57558", "51525", "12199", "25183", "9086", "26273", "14137", "7391", "53758", "45830", "5892", "51824", "11595", "48722", "44646", "37599", "8628", "34574", "51426", "48040", "52106", "25475", "30725", "44179", "58561", "54991", "13307", "38644", "53707", "50221", "5521", "19924", "44599", "1254", "8664", "49772", "26408", "48377", "27088", "40501", "42784", "31854", "31675", "59922", "10791", "25944", "12561", "40125", "47509", "30897", "1332", "15583", "29201", "38037", "46594", "1527", "18966", "4714", "28280", "57012", "6418", "38331", "58742", "13325", "11895", "23824", "53113", "20930", "6710", "53333", "34788", "16616", "33721", "52431", "50916", "48885", "49066", "43958", "10258", "48035", "44992", "15693", "51899", "4108", "59157", "7265", "6917", "49111", "18011", "55455", "46824", "20697", "31161", "51809", "30106", "28976", "5149", "43630", "23189", "20624", "6129", "12280", "4885", "34029", "49218", "38932", "34569", "27902", "55799", "10242", "17073", "45072", "31465", "52878", "33742", "28133", "7449", "52896", "10048", "45621", "17262", "45652", "28247", "46790", "6379", "21213", "8156", "53604", "32831", "54211", "32772", "39663", "45647", "13554", "11431", "47770", "42197", "32201", "58899", "55027", "25163", "18177", "28754", "30962", "47612", "25398", "14758", "25098", "38027", "5850", "49308", "58118", "59730", "52412", "16711", "30306", "7169", "40966", "37281", "37473", "56311", "54541", "27811", "30579", "39415", "35882", "23339", "27055", "10870", "37486", "31835", "40663", "43480", "10345", "42212", "51947", "51591", "11088", "26488", "28573", "35395", "21670", "8339", "52506", "17702", "17571", "59204", "53552", "54302", "688", "45588", "10975", "20006", "17495", "29884", "48927", "40301", "8652", "5656", "22868", "31021", "550", "33239", "54018", "19740", "24254", "14119", "36181", "31935", "9292", "48958", "13746", "14134", "19691", "9489", "32862", "24307", "15537", "47506", "23834", "41585", "44851", "54158", "24603", "20240", "41679", "683", "52977", "31131", "17218", "53438", "22930", "40910", "47218", "46379", "22603", "43716", "37616", "25806", "33931", "13537", "6971", "41091", "15325", "53232", "52485", "23667", "44529", "43416", "58083", "5516", "15037", "13402", "3561", "2549", "36525", "18868", "3143", "26964", "56027", "13137", "17284", "54683", "58530", "42310", "6356", "42156", "56999", "4323", "59200", "45492", "20248", "12524", "47974", "55091", "3511", "718", "5503", "15327", "56788", "6395", "24752", "37593", "22674", "35162", "20384", "44779", "20265", "43355", "13260", "5661", "51948", "17277", "28159", "49591", "4846", "743", "45484", "19320", "34868", "23094", "17159", "14233", "1435", "1723", "57743", "34467", "4674", "28488", "423", "45861", "9173", "46759", "28107", "33430", "8178", "36822", "3828", "33282", "27448", "44702", "53390", "55424", "57415", "54501", "2995", "55529", "26412", "29474", "35729", "53048", "56298", "53295", "46366", "8226", "53038", "38501", "29997", "55256", "47055", "23005", "43233", "21467", "49910", "5472", "51195", "56310", "42494", "20848", "52520", "2288", "59972", "24924", "13991", "50168", "36113", "20344", "17633", "42114", "35131", "41255", "44500", "34147", "22306", "44100", "47171", "16338", "5031", "58323", "36578", "21197", "30253", "42474", "50372", "8088", "2927", "18206", "21739", "48922", "44968", "43920", "50230", "4456", "20698", "26759", "35838", "42628", "22415", "21472", "49756", "31685", "6498", "37957", "38461", "22707", "4191", "11116", "57023", "49509", "18190", "34266", "42541", "53582", "39592", "11950", "10456", "50083", "10311", "28050", "5154", "33796", "41616", "1004", "22656", "35361", "28161", "48278", "58381", "54095", "14955", "46767", "49530", "22472", "39675", "54696", "39843", "51513", "42306", "20912", "17", "52405", "31089", "34159", "10952", "36384", "59841", "27188", "54948", "46499", "58007", "12027", "16040", "44874", "12671", "1897", "23157", "9721", "39598", "34478", "2025", "19488", "37115", "38462", "18747", "47488", "8733", "13738", "9657", "46968", "47117", "55687", "42857", "35742", "26691", "30958", "25220", "17923", "5692", "39240", "2926", "54871", "6648", "19901", "30697", "13100", "31999", "28238", "33178", "11619", "33854", "36194", "36614", "48308", "24172", "22076", "26919", "17870", "30399", "16131", "15251", "55824", "7677", "1615", "27034", "32227", "27789", "24820", "15633", "17660", "33766", "45315", "18870", "6787", "44407", "12188", "18262", "31921", "58446", "37081", "1361", "21897", "51186", "59522", "31220", "1074", "37336", "41813", "57672", "9181", "21432", "45294", "356", "29815", "44748", "18908", "47272", "21655", "14739", "3627", "1900", "50282", "33533", "52741", "30937", "29091", "12996", "4706", "54457", "28628", "25987", "28160", "5680", "29500", "32377", "33681", "39325", "19153", "34781", "51531", "28618", "38361", "1119", "5738", "48374", "8270", "24178", "50948", "27534", "6218", "23576", "27465", "42364", "31373", "27377", "10892", "50597", "42246", "24723", "11157", "19002", "43524", "53968", "13521", "27624", "33531", "7928", "5293", "43799", "59012", "20727", "35369", "13994", "17674", "57480", "24728", "13392", "13017", "30158", "56979", "56594", "14944", "25303", "36778", "9676", "6618", "49746", "19468", "10511", "45029", "37857", "20369", "31229", "51084", "40956", "7757", "28933", "23439", "34462", "24430", "38277", "190", "22918", "49268", "9519", "7172", "6962", "17666", "20387", "31067", "15396", "44650", "12814", "50320", "17597", "48583", "34855", "56968", "59183", "43123", "8041", "29061", "26730", "33647", "13463", "36838", "18335", "17794", "11888", "7551", "19297", "29059", "45379", "17817", "4675", "6814", "49206", "56677", "9722", "19053", "3824", "20938", "43940", "33294", "6956", "18841", "54643", "26537", "37561", "189", "50338", "50289", "14213", "5426", "36344", "45411", "19742", "49737", "34104", "12992", "32994", "31152", "29259", "25372", "40950", "32635", "19587", "55324", "50461", "21776", "4566", "42665", "17707", "18101", "34778", "7923", "3182", "57314", "5734", "48674", "45978", "58680", "9252", "38798", "18263", "13332", "52053", "43981", "9649", "56751", "11511", "16456", "49555", "57182", "43117", "8089", "2390", "772", "31308", "38814", "36509", "58315", "48874", "52782", "31896", "43184", "41456", "21669", "58142", "27121", "10149", "32323", "47440", "58642", "27524", "46439", "20308", "35209", "4121", "41702", "30459", "12812", "15350", "45446", "13167", "45528", "15703", "31737", "37698", "9067", "28921", "22007", "34693", "32054", "34105", "42938", "59392", "46305", "6728", "45897", "37579", "15526", "32327", "53266", "19505", "46483", "4801", "33175", "57997", "9174", "50502", "26361", "16705", "13443", "540", "45940", "14928", "1855", "37125", "58733", "51381", "24907", "6699", "21298", "39208", "25312", "23598", "22890", "13330", "31256", "14532", "14890", "16707", "8824", "7302", "9849", "23917", "54036", "50260", "39222", "42733", "49365", "26750", "54950", "10664", "49059", "21183", "46854", "53426", "13416", "58747", "38800", "58522", "37606", "14199", "44410", "54642", "15611", "44291", "55282", "53250", "48090", "36418", "18815", "22329", "8061", "56990", "41807", "46387", "58260", "50264", "25173", "40898", "53884", "36267", "9773", "9669", "4947", "6568", "37229", "55940", "39475", "32717", "20983", "18563", "25880", "56544", "911", "5538", "44959", "59988", "57498", "40897", "6402", "24983", "50805", "46317", "41722", "31607", "8498", "11", "42290", "31093", "16584", "25477", "53419", "32828", "43751", "25201", "9964", "15577", "59419", "30009", "37451", "13935", "33709", "53303", "12862", "47869", "48469", "30645", "45090", "11701", "916", "59688", "28348", "42926", "3860", "8826", "44396", "58616", "22747", "59093", "49922", "7925", "24936", "54964", "31119", "13963", "59630", "11185", "14615", "45536", "1717", "52540", "22735", "24421", "22479", "45512", "39182", "53334", "14686", "50611", "937", "54454", "30116", "10155", "54756", "52519", "18305", "818", "3855", "50736", "19405", "51661", "24994", "6248", "16715", "14976", "30281", "1227", "43136", "36304", "20601", "16472", "47667", "13719", "22140", "15568", "35078", "6387", "51792", "55838", "38564", "36208", "47539", "902", "50165", "33673", "37498", "49623", "56307", "48520", "33384", "27064", "40177", "28090", "38620", "6170", "37468", "51181", "46598", "59197", "58756", "5637", "40147", "3525", "9339", "37823", "58610", "3300", "49500", "5190", "21626", "59458", "28258", "18402", "54737", "25277", "16940", "57976", "50911", "816", "17903", "31504", "33316", "19568", "15295", "40119", "58394", "42970", "22590", "53640", "29809", "38648", "27287", "59046", "7945", "55084", "46795", "23977", "59567", "9994", "56118", "46040", "17130", "8000", "1653", "1825", "1580", "48366", "8965", "38690", "47705", "29444", "59559", "49397", "55180", "26454", "22458", "57756", "26806", "28262", "56220", "7779", "46371", "16554", "40767", "56150", "52664", "39613", "56701", "56472", "28958", "20942", "8392", "13244", "45682", "7171", "26632", "16464", "49162", "5357", "17162", "30739", "33226", "17308", "22324", "37256", "11600", "45167", "54878", "56540", "13966", "33757", "15497", "26559", "58565", "30585", "36641", "27023", "1939", "17163", "31127", "8786", "45496", "58256", "3212", "41757", "33036", "27736", "40464", "9492", "31864", "17370", "28068", "23991", "50570", "22132", "42429", "49", "3833", "34651", "47432", "26802", "1384", "12629", "42682", "47516", "5113", "49641", "12159", "5866", "51462", "2976", "16480", "33292", "52557", "19811", "50248", "51926", "38056", "287", "25531", "12389", "58354", "15013", "2196", "15518", "17843", "28882", "15293", "37436", "52957", "59968", "8106", "33401", "56284", "16855", "19486", "48576", "27347", "45977", "19670", "982", "2864", "40490", "45675", "13453", "58407", "49476", "38314", "51957", "8489", "30954", "25421", "30032", "35453", "12660", "56505", "21982", "49474", "22481", "51908", "49933", "43182", "54342", "32817", "45839", "37349", "29019", "13054", "4237", "28594", "6662", "20043", "39173", "7683", "57459", "9347", "33374", "20530", "559", "34343", "38980", "49572", "13548", "9441", "35323", "45888", "40235", "45273", "56087", "6176", "40763", "56341", "26644", "54656", "22552", "12509", "22420", "36262", "11951", "52484", "27021", "37356", "37107", "20144", "3204", "26265", "55138", "23692", "45456", "3612", "54517", "17679", "6490", "38418", "50505", "21019", "19028", "52190", "42769", "51925", "15104", "24182", "38423", "35187", "30560", "14528", "47878", "48707", "53270", "23520", "22081", "52545", "27260", "42400", "58913", "39635", "16944", "21834", "11450", "24223", "28309", "34891", "59760", "15275", "27398", "7478", "45429", "5461", "17703", "32202", "40526", "4351", "42759", "5028", "38272", "16896", "58821", "14142", "18848", "41896", "57731", "40854", "57902", "58440", "9161", "23360", "16817", "2856", "33630", "41164", "19001", "35812", "49155", "26318", "40019", "25910", "1790", "21069", "3444", "4607", "49158", "29921", "41012", "55041", "24952", "22486", "37986", "51805", "10856", "35902", "27425", "16180", "15136", "26014", "28971", "3324", "58736", "38327", "55335", "35740", "49797", "49276", "35349", "19005", "59563", "2409", "22283", "8538", "1035", "38882", "25208", "53453", "28445", "45503", "30896", "32992", "55029", "58455", "6560", "11147", "22338", "32909", "17409", "39928", "10377", "41245", "53928", "20471", "46437", "3998", "30052", "39371", "26669", "8725", "20391", "52017", "1246", "36643", "48369", "4972", "2408", "28965", "47573", "3892", "56801", "49517", "11142", "33267", "1509", "59079", "28787", "48527", "31621", "37827", "9473", "31889", "19437", "41413", "47460", "37712", "43549", "24212", "4399", "23814", "10326", "21374", "29540", "58714", "38578", "52599", "39648", "11684", "44773", "47965", "21524", "22562", "8814", "56452", "19496", "30099", "40298", "30228", "36688", "28169", "11409", "59382", "39185", "36339", "27775", "11828", "51110", "18522", "33887", "36874", "20882", "6173", "52951", "32110", "4994", "40565", "42625", "34528", "14168", "1681", "19634", "55053", "52234", "13568", "22104", "15450", "31542", "10938", "41426", "24746", "5767", "29907", "41572", "41385", "32637", "23912", "57093", "1097", "19374", "3235", "2857", "55902", "33546", "55362", "51046", "37209", "27407", "7274", "55483", "46726", "51349", "27468", "20267", "6018", "42030", "42158", "21868", "11311", "35952", "12377", "59818", "34830", "22854", "10215", "18105", "56690", "13327", "45995", "56988", "14412", "21922", "8319", "31375", "58817", "47903", "21203", "28331", "43462", "33065", "52562", "18329", "16521", "42447", "58974", "59664", "26685", "53789", "23068", "15945", "3410", "51514", "22405", "33895", "37903", "35259", "40895", "43085", "29937", "17169", "21680", "24940", "49220", "14817", "24036", "11387", "6507", "47354", "31230", "5482", "34515", "23258", "3456", "34392", "55698", "52728", "3770", "33350", "34603", "21510", "19613", "19832", "5480", "53241", "51923", "37363", "50239", "26554", "59021", "39204", "11718", "51750", "32538", "25822", "47937", "26647", "35246", "877", "35643", "25103", "52645", "12296", "37131", "13996", "23542", "68", "16308", "45988", "29048", "15663", "28257", "9909", "31277", "7146", "29645", "11186", "10689", "11479", "53539", "56714", "7143", "39245", "11560", "7032", "37404", "45211", "50763", "44152", "16445", "30108", "17125", "40653", "10752", "14248", "29875", "3242", "52348", "22331", "10636", "41277", "22698", "59728", "12949", "47629", "3730", "25698", "48248", "53124", "35932", "9190", "48511", "8905", "16168", "59205", "11914", "8434", "8740", "45194", "53837", "10246", "18912", "19084", "15141", "27941", "16214", "49252", "6468", "23397", "50815", "53720", "16407", "17769", "22948", "42082", "49442", "45798", "55629", "16626", "5548", "35967", "34044", "44993", "11899", "30863", "16600", "36371", "1220", "18799", "22881", "34332", "6424", "7488", "34325", "36661", "26666", "23044", "25273", "20885", "18896", "40303", "8257", "38590", "23034", "51678", "47836", "2373", "43667", "26557", "51011", "54271", "8207", "22164", "6604", "2258", "32296", "20073", "13359", "57758", "57668", "39369", "9795", "54958", "21166", "39841", "19529", "42754", "35461", "49063", "30194", "49192", "20547", "27744", "15200", "18534", "49187", "33913", "14604", "852", "8095", "11125", "17060", "22421", "50345", "29088", "15821", "51251", "52541", "44297", "48420", "29626", "20410", "27481", "38044", "25124", "4905", "42215", "2439", "35165", "20651", "52952", "40791", "45457", "37925", "35458", "9152", "30443", "49470", "44287", "52589", "24808", "21386", "42538", "4140", "30393", "36276", "25248", "5775", "54977", "29892", "50323", "25541", "55842", "35141", "30498", "40088", "16964", "28388", "56940", "19584", "50579", "16673", "55511", "26821", "26569", "14459", "38741", "8139", "25799", "50246", "19725", "21011", "20269", "20238", "3628", "4963", "37061", "26143", "10960", "8703", "5979", "37787", "26274", "49988", "21201", "39319", "44447", "9040", "37242", "22686", "1744", "22824", "15629", "55478", "39589", "43955", "34456", "354", "31211", "27599", "24312", "357", "15410", "44173", "27735", "3091", "54292", "8684", "1331", "41234", "45748", "42518", "13301", "11604", "51997", "26086", "32591", "24393", "39988", "37915", "15572", "25643", "44350", "47916", "22177", "25005", "45915", "6151", "51935", "45302", "5384", "58020", "25654", "41428", "34928", "19850", "30239", "48806", "10073", "31183", "7232", "8720", "25589", "10978", "45250", "657", "40978", "26434", "12740", "3062", "2251", "6922", "28671", "4355", "43919", "24578", "35017", "37665", "39273", "18917", "49821", "28296", "20360", "31821", "5069", "15563", "22694", "32391", "44652", "54215", "20266", "29741", "49782", "47643", "4876", "10174", "46252", "3532", "28135", "28375", "5362", "10772", "38946", "47609", "14783", "57351", "10332", "32222", "44117", "4556", "52997", "46394", "38427", "27779", "41660", "53089", "44878", "41021", "46900", "43614", "938", "36921", "58627", "31024", "9176", "58097", "33686", "10522", "47776", "16457", "14227", "33114", "58061", "6454", "4410", "59965", "56213", "34721", "22554", "54747", "47749", "33967", "41408", "4762", "52052", "11552", "3924", "43708", "32493", "35556", "11658", "47529", "17444", "39657", "11829", "53958", "52229", "57654", "37303", "37170", "1149", "29622", "37427", "35628", "25819", "9205", "12552", "7206", "58387", "12303", "4448", "14870", "57649", "30675", "43690", "35547", "41830", "52394", "2776", "36769", "13838", "4147", "53821", "25575", "10519", "33038", "22349", "21831", "25718", "7448", "37093", "40256", "47497", "2329", "17768", "13513", "3279", "7591", "57040", "1347", "29083", "37413", "16720", "6142", "43332", "22110", "13013", "16822", "23092", "12015", "10886", "17000", "53244", "34643", "6892", "55754", "6702", "37158", "22565", "9021", "30653", "2554", "47247", "45348", "36625", "46102", "26522", "13073", "25714", "48871", "21795", "45015", "52766", "3662", "53513", "39041", "35703", "43256", "43606", "535", "36740", "18727", "54718", "30030", "41399", "43149", "26243", "47886", "3614", "46033", "54675", "36923", "9472", "7319", "48321", "17267", "27561", "6927", "34561", "14934", "46673", "1081", "25088", "41316", "8185", "39515", "49300", "42600", "54238", "24349", "25574", "22424", "44616", "38356", "30042", "13874", "23802", "42426", "43291", "40218", "51209", "50374", "44871", "37100", "18772", "7647", "31092", "11036", "15075", "56995", "29578", "48023", "56197", "34765", "4955", "47851", "34097", "52149", "33413", "8060", "5698", "47517", "27104", "38322", "38470", "2204", "5112", "30295", "23466", "23644", "3856", "49686", "58327", "52663", "32414", "5577", "47463", "46408", "49020", "26664", "46061", "57983", "39288", "41229", "6767", "3149", "2185", "58947", "37388", "42727", "49978", "11457", "22784", "27875", "28022", "39311", "55470", "43822", "33091", "48890", "36455", "11848", "33814", "58162", "49058", "34252", "31827", "191", "32037", "50801", "17206", "6931", "2442", "12230", "47230", "51542", "18090", "713", "50045", "6250", "16587", "3438", "50490", "54511", "38949", "6071", "25169", "23548", "24127", "58267", "6296", "16989", "32654", "28330", "2771", "47652", "19979", "36875", "24574", "14497", "9274", "53901", "24109", "1301", "50747", "44789", "35137", "35368", "2632", "6681", "34219", "28719", "14973", "29802", "22706", "34943", "30795", "52296", "12784", "10006", "24914", "7366", "3031", "43209", "9289", "14479", "40457", "12310", "25360", "12431", "34216", "21271", "16888", "429", "42221", "16394", "29447", "20468", "20691", "9293", "37020", "8884", "39686", "35158", "15875", "7807", "13990", "31838", "16436", "9237", "35197", "29071", "19450", "34702", "40901", "7313", "33617", "52404", "10211", "15269", "54319", "33214", "22389", "9914", "1956", "54428", "7444", "22809", "24251", "29075", "50544", "23199", "22271", "20990", "40953", "7824", "14952", "43861", "54895", "32836", "23287", "4289", "54058", "48984", "43437", "18671", "42820", "18076", "6485", "9960", "31839", "28017", "55486", "7661", "11106", "16185", "15150", "4515", "44004", "15852", "51076", "8478", "22170", "42273", "13183", "4752", "1164", "25961", "7527", "28204", "51445", "47973", "54151", "58999", "54495", "38035", "12055", "56769", "53075", "52207", "10600", "3683", "58699", "35692", "22268", "6113", "26446", "23097", "6226", "149", "56319", "17507", "39620", "56393", "22779", "11756", "48346", "3569", "8198", "46692", "10599", "27663", "23754", "19510", "10513", "22038", "56065", "43119", "13928", "38988", "29959", "31362", "26836", "12481", "142", "38187", "59192", "42072", "448", "3358", "52585", "25767", "26056", "56467", "18551", "31003", "46042", "13925", "9268", "26651", "7494", "53779", "3816", "23415", "34899", "37027", "33764", "6732", "43460", "36274", "22772", "36295", "57210", "54065", "41728", "34701", "56077", "13052", "4407", "50147", "43138", "21505", "9432", "29443", "4671", "17429", "38687", "51693", "45783", "14535", "25842", "33001", "841", "44848", "58408", "34357", "35130", "32921", "19688", "10122", "23063", "24051", "55400", "4996", "57855", "53173", "23662", "46734", "33426", "46010", "35332", "24702", "44260", "19965", "29093", "46088", "32061", "42227", "43112", "41363", "48621", "79", "58043", "879", "15594", "58994", "41611", "14864", "19605", "25224", "13908", "57614", "51613", "1606", "41492", "54314", "19967", "17563", "49356", "31468", "52301", "33220", "48152", "33155", "59768", "42532", "55615", "59455", "45623", "7751", "4770", "2228", "46302", "33748", "57249", "20766", "49917", "10368", "52833", "42841", "35003", "39724", "11754", "11417", "24258", "32411", "26426", "17885", "48230", "41755", "43740", "36268", "42232", "4582", "12700", "40540", "54341", "43094", "24809", "10358", "23345", "17059", "12234", "13460", "19031", "20596", "35630", "34530", "45308", "37708", "29193", "55059", "31026", "17901", "46787", "58786", "29580", "37541", "48071", "44944", "52098", "2205", "52776", "36576", "28525", "50855", "6988", "50391", "32404", "27061", "10392", "21892", "128", "35677", "51690", "38097", "18681", "6032", "41909", "27622", "8432", "39134", "16026", "4418", "29508", "53933", "36041", "26113", "21523", "49927", "26026", "33730", "29724", "47668", "25712", "34806", "23322", "9276", "6680", "21694", "34946", "49551", "9123", "116", "36252", "23238", "47005", "20599", "36504", "10348", "3600", "724", "2451", "42852", "16748", "36836", "31976", "18463", "7673", "47391", "31572", "31236", "3574", "35253", "9595", "6300", "34076", "7446", "23595", "51534", "59991", "43855", "44949", "16388", "53505", "41534", "28198", "2005", "23973", "20498", "29297", "48047", "27770", "48800", "11085", "34459", "46458", "42211", "23304", "13434", "40698", "12578", "44246", "6709", "46153", "31420", "34112", "614", "53680", "33916", "48525", "5556", "8066", "40356", "8100", "11082", "17408", "21334", "46168", "29211", "1741", "53085", "28552", "2931", "6688", "29537", "25742", "19398", "40887", "5263", "59057", "26576", "51937", "43107", "30704", "45491", "35233", "49658", "42571", "41391", "13114", "48311", "34627", "10849", "20665", "58090", "24530", "36190", "23837", "42956", "23880", "9095", "10312", "56917", "29790", "23845", "24712", "57338", "55130", "18258", "19890", "51307", "5453", "13363", "6097", "37049", "31040", "6338", "34134", "45185", "50072", "32744", "21421", "51864", "26048", "20799", "21403", "46990", "40296", "57532", "4682", "44049", "12684", "19828", "7149", "51598", "39143", "59124", "10795", "50924", "32727", "12581", "43686", "38510", "11650", "1784", "44967", "21857", "40294", "54489", "3533", "31043", "22440", "254", "28581", "33249", "38228", "859", "12113", "12375", "40730", "34661", "55291", "13514", "56648", "36538", "28332", "838", "25612", "45341", "5424", "39650", "58416", "37919", "19586", "32549", "5250", "2846", "52121", "52387", "23039", "59366", "43852", "19364", "58696", "7982", "52644", "30434", "16234", "37826", "58809", "46221", "48636", "38357", "19311", "23161", "45665", "44344", "44236", "20853", "19149", "51508", "58606", "52808", "55521", "103", "7735", "39566", "57805", "3773", "4988", "20194", "58459", "7347", "52067", "25754", "46193", "11473", "50791", "11366", "52570", "52025", "2892", "6321", "14274", "1714", "23016", "4643", "3867", "26712", "8769", "57926", "18268", "18720", "21750", "3377", "25030", "20954", "36326", "59940", "47112", "38907", "55397", "43046", "4746", "41704", "45774", "4774", "10823", "46074", "6789", "27371", "9896", "13095", "33128", "2547", "39719", "51861", "25925", "38567", "40329", "59884", "349", "25089", "23125", "59406", "43761", "10946", "40996", "29412", "31760", "47429", "36832", "6191", "6504", "38370", "39890", "44158", "36404", "51852", "10701", "35052", "9551", "17109", "5506", "6465", "10212", "42203", "58201", "1528", "40061", "31145", "38467", "42707", "34521", "45441", "47060", "37483", "52717", "42690", "58244", "2667", "31297", "3939", "25315", "28939", "50864", "47766", "19231", "41600", "51327", "14866", "57525", "46637", "3617", "2070", "42497", "49055", "23527", "17113", "27129", "52088", "59374", "10229", "50746", "2144", "453", "24187", "43604", "14268", "49859", "31849", "58724", "56653", "39550", "1018", "20502", "20277", "35641", "58787", "36908", "19471", "35018", "33603", "19680", "13311", "18016", "54282", "51965", "32942", "14710", "20165", "7559", "26256", "21260", "54850", "43195", "57965", "21212", "50413", "37194", "19579", "8529", "50599", "16724", "42767", "54218", "27729", "18618", "42644", "12311", "52684", "5499", "25986", "10514", "13020", "4980", "49380", "15741", "32485", "26728", "25775", "32175", "10255", "57935", "24826", "35482", "20226", "50796", "16937", "55428", "35603", "38190", "26008", "8074", "42878", "397", "7883", "31527", "34901", "49194", "6585", "15684", "21", "17472", "34954", "7040", "12805", "30681", "11659", "32723", "48439", "5159", "2062", "24257", "58678", "46058", "13036", "40133", "16775", "18489", "23083", "35683", "31235", "56409", "3615", "1927", "34768", "57004", "41042", "57572", "18209", "16371", "19059", "43015", "14149", "7333", "20281", "49602", "44591", "51278", "16556", "40212", "9127", "4216", "48074", "12528", "57491", "1295", "43378", "48977", "37625", "50449", "28100", "1130", "59019", "51603", "32482", "2360", "14445", "23790", "33159", "47738", "33578", "28371", "57209", "50474", "59599", "46681", "24439", "26237", "4468", "31986", "46513", "29270", "32244", "31914", "18977", "35994", "6108", "43447", "4363", "56742", "34964", "26330", "58597", "30878", "18904", "5368", "39056", "37598", "26672", "6288", "17229", "54359", "52530", "43185", "52033", "52706", "14806", "56449", "29085", "50927", "23687", "47507", "7411", "6035", "21483", "8381", "27200", "5277", "25657", "23779", "48037", "31524", "18146", "13212", "11643", "44535", "16552", "22988", "50965", "12830", "1768", "54984", "39973", "32090", "48223", "16568", "53491", "30283", "12876", "54285", "48975", "2126", "4126", "24579", "21469", "5607", "5048", "55794", "13268", "39094", "18387", "25887", "39905", "58377", "57035", "30221", "16780", "56673", "30304", "25715", "7655", "42141", "5695", "54822", "22433", "20401", "28403", "39405", "32647", "47614", "57683", "21848", "16938", "53660", "4168", "34230", "54892", "12163", "25187", "4271", "4039", "6140", "5302", "38911", "32390", "30406", "12050", "19999", "6332", "41398", "12164", "49547", "1745", "59461", "14283", "24528", "16666", "11264", "49793", "5487", "24289", "32626", "16697", "19487", "6078", "16199", "59895", "53149", "45532", "41594", "50564", "13425", "27436", "21576", "23326", "3897", "42632", "45552", "32922", "30182", "17767", "551", "38751", "44478", "29405", "55257", "49976", "9622", "15279", "49103", "14593", "125", "49745", "6574", "36219", "44347", "2522", "37246", "4219", "9601", "17042", "15422", "8907", "32294", "25149", "58555", "49117", "47779", "46656", "32184", "17748", "52071", "3598", "43676", "32448", "11419", "23932", "20010", "30550", "32463", "52156", "25933", "4602", "9131", "13938", "2653", "35534", "10873", "42657", "19077", "19326", "18230", "1355", "54543", "20987", "9138", "351", "15218", "53828", "52817", "43588", "38691", "35831", "30894", "28726", "57594", "6429", "51140", "39837", "32623", "15509", "6734", "55589", "33476", "57929", "19057", "44081", "28469", "17074", "32280", "48424", "29609", "24954", "7495", "23429", "10170", "12355", "1249", "27443", "53878", "11752", "10573", "41482", "59609", "9308", "1711", "39671", "29056", "40416", "30127", "27646", "40016", "54012", "46742", "53798", "48378", "43305", "29982", "4160", "59547", "40564", "48299", "25282", "8363", "37398", "57593", "37918", "36967", "59129", "43228", "32607", "47202", "27017", "792", "56649", "38864", "58859", "33222", "43749", "37582", "44852", "45674", "25540", "4662", "49651", "50306", "35404", "30070", "11705", "34927", "33948", "36088", "25544", "37431", "26240", "53383", "45376", "11057", "35257", "39005", "263", "33803", "56959", "12029", "41232", "30345", "12327", "34354", "33051", "43424", "29236", "31661", "45841", "21489", "55318", "2158", "9547", "32479", "16104", "32546", "33756", "48918", "21644", "37904", "45820", "23457", "33106", "33041", "35636", "22851", "20168", "16893", "13629", "32528", "1338", "29017", "33824", "25015", "45908", "57677", "9109", "54821", "41079", "48705", "14343", "26653", "27845", "27662", "17312", "16823", "29729", "30594", "17928", "50179", "24980", "30455", "27174", "29383", "49350", "49169", "16333", "23497", "49676", "41105", "43511", "32212", "59593", "48837", "18214", "36704", "25433", "42879", "51032", "11789", "10352", "15160", "12305", "34225", "47785", "37250", "52089", "41443", "31490", "4068", "39732", "35382", "27810", "46155", "27957", "11550", "34082", "33379", "17474", "39964", "30200", "36830", "15571", "1440", "20770", "18701", "12488", "1466", "47020", "13275", "1964", "48292", "6065", "25010", "43423", "36396", "14285", "2243", "43273", "44426", "30233", "42320", "48258", "13369", "10857", "14169", "21695", "35107", "48005", "19116", "14478", "19413", "43230", "49862", "25117", "25401", "44140", "51473", "25411", "42409", "50721", "314", "56086", "16138", "16593", "19859", "35054", "31700", "54922", "26810", "32424", "39054", "44634", "5679", "57119", "26175", "55749", "32597", "49318", "35366", "43854", "27137", "12896", "45081", "49366", "602", "29074", "25003", "1003", "5743", "57759", "39216", "55966", "260", "25635", "37033", "40411", "13605", "15578", "1859", "56722", "22190", "23857", "48410", "44459", "35145", "13869", "52935", "9517", "52556", "24103", "45547", "45801", "55100", "31564", "46501", "6707", "50192", "38446", "31966", "25600", "41529", "11520", "33896", "15589", "56719", "24855", "17297", "54943", "16804", "32878", "16169", "13848", "32386", "37009", "16294", "20193", "39956", "53202", "25405", "48236", "10559", "6263", "55661", "4803", "5268", "59880", "35255", "32467", "44444", "30683", "9041", "57782", "33334", "21118", "36954", "40619", "28651", "20634", "51706", "4344", "28143", "29073", "4102", "41193", "58176", "56309", "46319", "44501", "24150", "48808", "42319", "43419", "26827", "19657", "22951", "42304", "1207", "48352", "42968", "26335", "55464", "48490", "15074", "20536", "30969", "5988", "42651", "4263", "52159", "21278", "36148", "9746", "16496", "56441", "24569", "22754", "39731", "29573", "16390", "8477", "30666", "12130", "47038", "7922", "16015", "20841", "47546", "18787", "25432", "37858", "42646", "35102", "38704", "7150", "37651", "27132", "45352", "3583", "21631", "58862", "118", "17283", "39161", "28856", "10274", "10346", "2770", "57918", "35002", "14927", "34821", "10167", "21575", "35023", "13536", "30035", "42876", "37791", "26196", "30903", "6208", "22030", "44589", "16123", "38201", "13612", "35310", "15641", "9984", "31141", "59894", "25922", "25537", "37255", "50141", "32370", "45028", "51607", "17770", "32259", "14951", "27103", "33484", "42525", "57746", "3002", "39101", "51342", "29487", "10943", "17141", "41109", "44296", "55739", "6005", "40746", "32887", "15575", "36966", "29369", "13602", "36389", "18762", "39538", "32148", "46392", "38853", "13515", "21743", "58659", "59548", "15508", "538", "56573", "56006", "20091", "13530", "27554", "56749", "24747", "19267", "15397", "1084", "5486", "36979", "24645", "13171", "20828", "3918", "40989", "31115", "3694", "34811", "39747", "720", "16895", "10016", "4257", "2859", "37282", "53152", "3367", "55916", "8772", "50909", "22480", "38036", "12070", "3791", "11010", "38585", "52603", "2325", "31792", "33849", "36211", "55489", "5367", "50843", "46661", "37257", "16036", "3428", "59612", "33078", "16737", "8604", "15980", "30775", "28913", "16116", "669", "26005", "572", "7629", "32986", "27284", "34144", "8617", "58133", "35668", "45900", "45136", "59139", "552", "22879", "44787", "5612", "44265", "50563", "12174", "19974", "45216", "21354", "58251", "31476", "49740", "28800", "6458", "2429", "30409", "30821", "40482", "53448", "45188", "23495", "18273", "29618", "58202", "7401", "42890", "24944", "57401", "29776", "32871", "22898", "11601", "39617", "37406", "21789", "42019", "20629", "38302", "20191", "39162", "24213", "10926", "45891", "59738", "27168", "17048", "46731", "7406", "17142", "16761", "29332", "27709", "53305", "22461", "2148", "15180", "22145", "6528", "756", "8845", "20117", "56978", "7566", "14494", "39402", "54246", "38212", "2638", "19457", "49905", "47967", "10705", "21578", "56426", "9444", "49426", "46904", "44243", "16201", "52971", "5766", "23661", "8501", "3804", "1103", "38127", "23376", "952", "12033", "22492", "12877", "25096", "40341", "35146", "44527", "38782", "40194", "48033", "11546", "44681", "44238", "19914", "20395", "31155", "42373", "16108", "41545", "9141", "28389", "20022", "47368", "3206", "36799", "44234", "31780", "10096", "12927", "40268", "41249", "45461", "37736", "56537", "16858", "14504", "57843", "48846", "35909", "48471", "25720", "50401", "28564", "6302", "52662", "35804", "21171", "22523", "42957", "39100", "14332", "41013", "54800", "12531", "11005", "11781", "54469", "55388", "17293", "27417", "11352", "6794", "2557", "33720", "17099", "14884", "42766", "49533", "58814", "33514", "55896", "27650", "38494", "24873", "50753", "5349", "8359", "41574", "6690", "44170", "42895", "10990", "29643", "17400", "18238", "22016", "59660", "4842", "13482", "15644", "6101", "54993", "35176", "47752", "25689", "37886", "31355", "155", "1910", "10239", "13116", "12680", "28453", "37905", "9972", "14322", "18660", "54456", "18301", "45856", "19083", "47570", "29200", "47699", "19087", "35593", "16058", "44458", "30793", "49062", "30532", "18642", "52994", "42151", "8424", "49484", "58419", "48982", "56070", "7242", "53746", "42418", "11400", "34556", "20664", "39488", "54174", "36469", "26491", "56669", "56609", "12704", "43087", "53790", "23715", "37423", "55452", "24492", "28404", "56385", "38422", "8211", "42493", "50546", "49345", "30379", "53347", "17564", "57779", "32616", "503", "29395", "28938", "3075", "11937", "52155", "51772", "11435", "13919", "59907", "29411", "15544", "11923", "45005", "2666", "24547", "39461", "100", "40866", "272", "20569", "43581", "56416", "20087", "8054", "25788", "8888", "20234", "26745", "778", "15502", "40554", "51213", "47286", "6314", "39784", "32368", "42955", "53396", "54375", "44226", "41787", "42031", "36828", "52515", "38385", "1610", "7552", "45896", "15358", "42354", "54332", "24984", "10744", "47459", "12798", "40465", "45343", "28524", "48910", "39823", "11367", "50518", "46194", "6235", "32547", "207", "22221", "15165", "36426", "35517", "49097", "53619", "20223", "49872", "22785", "2943", "57203", "26992", "31756", "47166", "29946", "14744", "17706", "49552", "16144", "13778", "14255", "722", "48178", "15806", "10496", "25182", "9277", "38486", "9789", "15995", "25485", "33653", "12051", "1848", "9706", "9540", "4043", "56108", "12274", "36331", "44651", "48936", "44714", "24689", "33658", "1263", "33621", "40626", "20412", "5610", "29375", "6775", "35870", "14821", "16845", "691", "56230", "9082", "11454", "15391", "29192", "16400", "7486", "20366", "40835", "39172", "43699", "46526", "38891", "40855", "30063", "32401", "55813", "14717", "28827", "18820", "57990", "11675", "53926", "48662", "19580", "16807", "23796", "988", "28778", "43915", "30402", "23445", "32458", "48188", "29360", "12706", "11138", "8637", "58700", "38492", "35734", "24989", "46979", "15617", "53200", "51212", "17953", "5617", "30129", "28104", "708", "26812", "13739", "9375", "59102", "30661", "17639", "34898", "5976", "30510", "32628", "9134", "30596", "17436", "19717", "36620", "8795", "17009", "52625", "745", "1150", "43987", "53816", "59222", "27819", "46972", "20375", "41115", "2608", "19972", "27395", "5184", "55990", "28619", "52443", "29591", "17246", "53377", "40130", "39192", "41656", "38367", "29830", "33822", "30759", "58624", "10040", "38471", "33255", "5469", "5647", "14191", "1193", "44610", "1686", "55360", "59525", "2484", "26585", "47774", "59523", "52196", "1298", "20998", "22146", "24724", "57066", "34987", "12217", "38122", "5706", "49544", "48130", "18970", "45971", "25763", "54191", "3065", "10992", "41601", "17426", "40865", "2914", "56773", "58588", "24093", "8707", "26871", "19960", "54887", "4114", "18180", "56942", "17924", "49239", "48508", "7397", "44461", "4178", "32797", "3658", "897", "35304", "43285", "6965", "12788", "11038", "50065", "47386", "7957", "13176", "11577", "59142", "59865", "2541", "16045", "18696", "14265", "50349", "18089", "38096", "3743", "22300", "40585", "51964", "2655", "11495", "8062", "50071", "40531", "1830", "45096", "31571", "19210", "5938", "58813", "4365", "19858", "33297", "19857", "41039", "51920", "26494", "35491", "22834", "20400", "57466", "41261", "43124", "17458", "26006", "55658", "47985", "52358", "22768", "9758", "13760", "49707", "6526", "33349", "7442", "38118", "50706", "57399", "4830", "7191", "40487", "14475", "55292", "56852", "6025", "16256", "53400", "22761", "6664", "44314", "4838", "45024", "46179", "50932", "57984", "48484", "28983", "30326", "3913", "12316", "9797", "40917", "31501", "20031", "7889", "55734", "6122", "48864", "21572", "42551", "26439", "55690", "3811", "57485", "38908", "54409", "13191", "12422", "12116", "46132", "3338", "14317", "41324", "15220", "12173", "18195", "20233", "15419", "50022", "26528", "2553", "31713", "57613", "28479", "15105", "13404", "20751", "21874", "10899", "9168", "10780", "25115", "41882", "57945", "56164", "56463", "14296", "46711", "61", "17372", "49853", "37960", "12586", "17337", "56705", "7365", "24283", "22034", "17947", "18592", "51806", "1666", "34088", "21545", "54354", "44899", "58337", "40620", "19150", "16772", "41188", "39983", "8223", "2955", "52238", "45330", "46396", "54499", "14774", "45363", "39734", "34844", "31985", "7760", "40097", "52887", "9126", "57958", "38374", "42053", "44047", "23861", "50450", "45246", "4685", "32570", "42483", "24766", "24635", "1258", "8283", "22161", "32950", "31750", "57208", "53112", "27979", "47888", "18191", "48076", "43818", "20711", "22334", "52503", "44395", "3490", "26088", "7521", "53441", "1870", "32167", "17576", "11458", "42358", "14704", "55905", "24424", "25153", "20075", "50388", "49264", "4266", "42619", "11636", "35177", "48226", "55652", "10082", "12824", "55168", "40114", "38680", "23755", "52173", "39346", "48574", "23067", "38616", "31551", "18655", "10468", "16493", "5671", "21216", "48708", "3115", "24497", "36166", "40928", "46632", "18514", "40025", "51606", "45855", "28588", "41577", "46612", "17876", "46083", "984", "55519", "7202", "5700", "7995", "3195", "37557", "15071", "46103", "29676", "18261", "37411", "20546", "27999", "24164", "11071", "15441", "14467", "48992", "2741", "44916", "4347", "51576", "52649", "27101", "10642", "26596", "35981", "23675", "55344", "53666", "32036", "21330", "18599", "57347", "5945", "3290", "16498", "13105", "5239", "8582", "35515", "13721", "4052", "615", "13147", "14353", "33452", "12165", "26626", "39037", "59084", "54037", "47744", "31919", "25863", "1972", "1006", "31776", "20609", "51283", "9114", "12161", "32672", "56938", "6150", "55192", "12988", "8840", "43336", "9893", "16951", "2437", "15796", "54186", "34350", "5040", "34742", "52832", "19694", "53289", "19045", "43827", "58326", "34604", "24431", "20178", "18236", "10757", "28390", "723", "25965", "18755", "25609", "50043", "37366", "21492", "48290", "1092", "21907", "13366", "45688", "39952", "20959", "96", "12917", "49546", "51681", "26628", "48340", "35039", "2270", "2100", "4933", "25885", "9314", "48786", "46991", "43360", "27541", "23107", "23447", "41729", "42462", "5643", "18157", "27572", "13266", "34270", "49373", "22705", "7296", "21355", "4651", "8874", "18093", "35120", "1735", "5127", "21335", "56019", "9011", "43849", "35552", "46939", "18513", "50978", "49664", "3466", "46872", "45291", "20037", "23311", "27521", "33466", "42980", "22435", "48192", "57463", "9449", "30905", "48726", "40082", "59795", "11369", "30580", "49717", "20835", "18691", "16538", "36343", "32562", "8552", "18352", "11281", "22273", "29771", "10136", "13380", "6260", "44279", "39851", "57467", "53528", "55313", "14303", "58842", "58638", "29390", "15111", "49705", "24272", "32730", "22786", "51741", "22526", "45709", "26229", "17830", "23401", "38487", "50639", "59489", "20316", "53900", "39315", "10677", "14218", "51339", "4927", "27110", "40156", "38015", "19949", "16851", "10125", "52533", "22227", "59212", "6826", "48878", "52500", "12555", "3038", "11664", "52714", "16373", "28493", "33095", "26929", "43932", "38355", "10391", "50023", "45311", "47411", "24090", "9009", "11076", "6894", "15359", "30430", "46585", "33286", "42268", "19913", "48996", "37041", "49273", "52876", "38468", "33949", "24810", "42367", "54910", "9253", "41409", "30922", "57492", "47174", "36150", "46981", "41101", "19966", "10616", "21882", "13129", "11423", "15827", "12438", "45404", "59558", "10582", "26191", "54320", "50070", "12910", "45910", "36595", "36230", "24928", "26605", "51880", "25352", "57263", "48018", "25166", "56382", "42711", "20747", "25407", "56941", "41690", "52958", "451", "36159", "42964", "36736", "29719", "42419", "2559", "52254", "1919", "15125", "14078", "5086", "11216", "54262", "42043", "24823", "8149", "34261", "37341", "37245", "29671", "14705", "59875", "58876", "26975", "7208", "50059", "57294", "23424", "17659", "56146", "13454", "8197", "19361", "5068", "2304", "53510", "56691", "54574", "48107", "19095", "13237", "46964", "47844", "47989", "30039", "49460", "27576", "45592", "26661", "15999", "1322", "50204", "4493", "11909", "53637", "56970", "42782", "14993", "13197", "19272", "19269", "25675", "32529", "24453", "22978", "10555", "58042", "29486", "58401", "28194", "4829", "3328", "1551", "6237", "17147", "58703", "25759", "23417", "40712", "50293", "19839", "33494", "34993", "30602", "23315", "11278", "24932", "41369", "39408", "58980", "11656", "21152", "57645", "18771", "15123", "38408", "38516", "48894", "13104", "35880", "57932", "804", "37931", "57937", "10732", "22972", "13074", "46424", "7410", "50636", "21010", "33169", "23483", "28421", "42931", "8094", "492", "14935", "2686", "4117", "452", "47480", "49330", "367", "41576", "15449", "22931", "2295", "22059", "31940", "58185", "37533", "12281", "13496", "15828", "32286", "49781", "7122", "1789", "55001", "3731", "34531", "26773", "45689", "7771", "47099", "19814", "15943", "30509", "55958", "45799", "25773", "39080", "29992", "57349", "16564", "41227", "46012", "27004", "21230", "27325", "38003", "948", "10064", "42003", "45204", "23809", "21050", "11043", "28949", "39627", "19449", "41743", "56127", "38340", "33491", "33208", "50402", "7281", "26263", "14442", "32762", "41258", "42093", "51573", "26393", "14989", "5375", "38031", "33217", "33670", "6475", "49274", "52357", "58171", "37819", "15966", "28420", "14662", "55892", "16178", "1009", "21238", "44738", "10602", "40419", "35186", "47724", "12839", "42865", "3825", "26331", "13736", "50774", "24445", "6185", "54524", "12309", "34896", "33817", "46684", "58041", "35770", "30110", "47078", "5334", "34294", "44283", "16208", "40871", "9319", "35091", "29762", "2881", "22230", "56905", "23544", "40633", "28513", "6143", "5185", "2978", "35007", "58156", "36132", "17192", "3453", "53890", "26288", "49212", "52242", "25672", "4961", "1948", "6654", "49958", "19944", "7453", "20242", "14730", "49838", "40330", "9454", "24183", "46671", "59683", "11838", "4733", "29435", "55416", "52964", "57152", "16601", "45034", "13022", "27203", "16175", "55663", "37921", "53712", "49678", "39006", "5660", "15712", "53087", "13691", "4159", "29176", "56912", "24401", "48059", "59513", "24125", "46332", "27600", "39849", "45745", "10446", "52339", "7858", "55375", "47467", "10614", "455", "32973", "5454", "6435", "55054", "37386", "54936", "5471", "42192", "22503", "6139", "48640", "3241", "5352", "53160", "54309", "13289", "873", "41846", "11534", "18376", "3979", "45877", "5473", "19062", "12148", "50524", "34494", "37652", "52043", "58533", "47454", "38090", "30694", "23133", "38989", "30331", "35876", "25350", "18482", "57191", "4689", "6320", "32523", "51914", "30057", "11129", "51144", "52116", "23395", "47161", "33664", "45568", "32245", "16733", "7665", "38906", "42647", "31320", "40189", "15782", "58034", "36433", "36038", "7640", "49014", "18516", "17245", "6541", "39557", "43384", "53995", "8107", "42529", "29996", "59756", "53484", "27087", "49338", "24032", "15905", "9354", "59789", "29531", "59927", "49225", "46890", "57342", "13718", "16645", "56251", "58286", "34017", "9135", "1071", "55873", "22804", "28492", "14485", "47571", "11616", "46830", "58855", "47158", "10054", "37916", "41983", "45772", "50837", "53340", "33619", "13254", "30591", "22987", "46289", "46249", "53275", "39120", "24880", "58750", "15026", "7688", "27806", "13898", "6240", "4843", "13889", "25505", "4844", "17001", "29272", "43551", "9655", "14872", "2340", "44706", "12884", "8760", "25151", "15631", "1101", "38638", "40775", "26396", "6384", "16988", "23798", "2133", "16513", "18130", "25902", "41340", "235", "45269", "59368", "42903", "42780", "32086", "13809", "12304", "26927", "13574", "45501", "7359", "22535", "26944", "14536", "12831", "23420", "23187", "31443", "10206", "58324", "34005", "57632", "32474", "50908", "54355", "30010", "26698", "42366", "39260", "9630", "35666", "24163", "36364", "51230", "59078", "5076", "21521", "10188", "44099", "17315", "21981", "41121", "9587", "29810", "41927", "14735", "48830", "29522", "51506", "21775", "30124", "41217", "33213", "44554", "904", "58517", "10318", "53433", "33609", "32664", "1059", "1400", "48999", "11660", "29095", "6548", "53007", "9116", "19396", "5940", "47443", "31796", "2157", "31217", "5611", "24117", "7825", "41394", "27668", "8212", "16337", "1578", "16304", "21022", "24219", "2042", "17806", "21653", "16064", "26040", "50856", "56370", "44145", "25593", "42018", "48004", "49309", "45996", "53981", "27579", "55772", "21234", "1603", "14139", "54933", "3675", "49421", "15727", "16624", "17694", "41640", "12665", "10115", "18294", "8473", "34466", "40408", "49650", "15860", "57078", "43869", "5745", "30988", "43842", "39548", "59663", "4743", "58474", "43979", "14990", "7721", "31216", "29027", "35336", "24184", "57363", "45199", "15304", "57588", "10741", "54780", "48964", "22584", "35351", "45890", "11574", "19745", "8164", "10758", "19854", "23254", "52397", "22154", "23180", "37338", "15435", "20701", "13792", "41817", "11231", "38974", "51825", "56588", "4541", "26747", "1564", "49731", "17979", "1994", "39694", "31154", "46177", "48458", "56229", "19105", "46834", "4156", "26448", "37783", "10855", "13", "33981", "32018", "35414", "19968", "23301", "9572", "46813", "54909", "22280", "16965", "21757", "58084", "8959", "38401", "53912", "59607", "58749", "41510", "51331", "42563", "57124", "22169", "45051", "4904", "18847", "13481", "29954", "32015", "17044", "29467", "40306", "14581", "59977", "39171", "3653", "19772", "25687", "9150", "56317", "18051", "9870", "24388", "31054", "44383", "44988", "53696", "944", "49847", "27268", "25836", "22540", "40588", "36696", "15785", "25974", "5416", "19608", "59682", "6652", "33024", "19995", "12060", "13777", "47670", "16675", "33644", "45098", "54599", "39186", "28843", "41009", "25348", "31689", "45426", "20054", "11992", "44512", "6500", "4479", "25906", "21268", "4635", "46291", "47113", "43386", "18766", "12412", "56536", "35632", "31851", "52024", "16508", "2699", "26713", "33113", "24899", "7972", "12339", "23882", "9184", "59871", "3854", "56484", "43074", "37378", "23383", "9338", "36831", "34416", "47857", "8299", "2369", "45668", "11018", "16687", "17017", "6215", "25268", "52611", "28444", "29067", "5659", "47604", "26224", "38393", "17139", "48421", "31055", "51594", "29657", "20333", "58731", "40648", "4138", "20239", "1547", "21854", "15808", "13194", "14619", "17738", "36387", "4484", "55786", "47837", "51316", "40722", "5063", "54105", "51225", "37763", "21513", "52118", "16120", "30913", "26144", "24403", "8621", "47538", "41116", "42087", "2951", "5925", "2798", "26017", "23812", "37959", "55820", "22313", "37817", "22003", "54072", "29600", "22508", "37728", "1122", "44662", "52680", "18833", "41313", "4075", "33970", "41010", "28102", "7099", "23523", "1682", "56448", "16664", "15106", "31357", "54092", "4275", "19777", "33781", "26046", "2416", "18549", "4487", "51755", "14724", "40351", "8736", "30278", "39717", "40985", "14621", "42479", "8371", "43554", "46113", "59667", "21259", "6580", "18233", "10951", "7812", "24155", "27128", "35598", "45677", "56156", "31481", "53249", "236", "59225", "5950", "27906", "27146", "55012", "27665", "10192", "38170", "10017", "28635", "31460", "15780", "43758", "21881", "47498", "58685", "6633", "14982", "17648", "66", "1493", "17559", "36422", "1299", "46902", "16226", "25111", "19774", "64", "25807", "2945", "6037", "26457", "20807", "8402", "46435", "3954", "4180", "37368", "59422", "15695", "15482", "11773", "44055", "56261", "53560", "23993", "31883", "3460", "53215", "9403", "37686", "18255", "8", "17344", "24157", "23758", "20667", "30734", "41620", "4413", "29635", "24624", "35273", "35200", "17439", "10985", "7705", "54954", "19076", "44043", "53755", "57765", "27482", "17654", "32436", "55796", "46814", "40686", "8810", "37152", "1567", "2136", "40513", "9863", "13078", "5811", "27828", "13905", "40455", "39123", "50444", "5884", "56140", "9642", "39755", "14807", "10220", "49895", "45039", "20944", "7187", "13940", "42912", "47887", "58893", "6753", "49289", "3274", "4000", "31430", "51652", "52462", "56007", "35424", "21629", "6426", "21749", "56735", "48811", "20855", "33641", "25488", "18143", "1268", "18586", "22841", "28979", "17281", "31056", "10225", "39012", "16489", "14260", "20876", "46831", "47263", "16501", "14820", "30482", "41366", "45727", "51257", "42489", "11567", "21719", "58155", "33540", "30410", "17662", "54380", "46642", "44222", "7950", "57841", "55883", "32987", "25928", "48451", "38004", "36783", "51835", "3390", "31310", "43298", "15777", "8551", "34171", "7712", "15989", "53838", "59016", "16329", "4513", "30427", "58613", "31300", "14516", "17937", "48307", "21027", "57833", "15114", "53024", "45431", "30163", "24552", "32920", "56332", "34841", "28409", "10450", "47671", "52945", "7090", "21813", "51325", "18614", "25244", "7546", "20858", "37240", "5180", "16500", "25107", "18714", "31525", "38707", "9728", "29117", "1469", "49855", "97", "17543", "21794", "11740", "37028", "50426", "35372", "58990", "32372", "59669", "56887", "59628", "22167", "52195", "54716", "22783", "59954", "7641", "15939", "31484", "56833", "16285", "45483", "24264", "25139", "21132", "19653", "45383", "3462", "41367", "48298", "41085", "28108", "15323", "52268", "53512", "14320", "7951", "30490", "20682", "14961", "14493", "52671", "16153", "57058", "12119", "57255", "26679", "31137", "8335", "55804", "16660", "36617", "43710", "2080", "22137", "12132", "29861", "51859", "17072", "29330", "51943", "19663", "4684", "3064", "48126", "57133", "27149", "13347", "34055", "14692", "46804", "2530", "3431", "21704", "52099", "4886", "40476", "35899", "48623", "21191", "24496", "30457", "30156", "49784", "23292", "36099", "9361", "15222", "49865", "41196", "58114", "450", "22585", "46345", "42458", "27882", "31917", "56204", "29728", "10556", "20531", "11633", "47944", "45151", "41330", "54212", "7704", "47183", "41480", "25158", "19203", "33020", "55051", "45760", "59565", "3644", "8452", "18088", "39393", "37993", "2432", "10984", "19770", "31209", "4171", "7974", "12973", "50929", "8454", "48547", "11083", "6706", "24342", "38184", "1902", "24418", "6863", "42248", "17821", "27250", "46828", "32385", "30561", "51646", "14678", "56045", "8556", "704", "48342", "18587", "25703", "3611", "1490", "25971", "10283", "17231", "14996", "10480", "31591", "5937", "50138", "43798", "56872", "57919", "50268", "10655", "50677", "1040", "58444", "54415", "27317", "37297", "25137", "30625", "54674", "4580", "6950", "9052", "17239", "16393", "18519", "30207", "35637", "30266", "54974", "25458", "31879", "57122", "41659", "9363", "4571", "43963", "46428", "43029", "35357", "45919", "22547", "39159", "26083", "53052", "30806", "16690", "57565", "11638", "45257", "1226", "27827", "5383", "34218", "29968", "21979", "871", "59915", "20830", "6796", "57227", "22258", "922", "17371", "53312", "24028", "2936", "22771", "15576", "56971", "439", "1879", "23793", "52446", "53234", "37422", "33265", "32582", "59733", "31370", "34803", "59581", "24991", "16200", "12875", "34141", "10381", "41448", "55564", "45099", "11012", "54988", "8118", "28310", "57986", "57162", "41499", "9581", "31522", "2140", "8152", "13398", "8977", "20189", "48084", "30656", "23567", "22289", "5829", "24538", "53974", "57587", "50515", "59128", "36849", "48949", "58344", "34387", "4158", "10409", "9801", "6999", "3674", "29136", "21844", "27914", "24458", "7810", "29468", "7213", "52020", "14626", "19815", "39664", "52049", "38014", "275", "28225", "2260", "12479", "13690", "30939", "13946", "44452", "32152", "8939", "23230", "5199", "1545", "38459", "22148", "30137", "18628", "24886", "22591", "7609", "38986", "23368", "7750", "7223", "47781", "30811", "1419", "58585", "55327", "15310", "54199", "19366", "2948", "41614", "9711", "42561", "40201", "38701", "54789", "19786", "32166", "49515", "56998", "52536", "48542", "24852", "22291", "23709", "2377", "52109", "14047", "22193", "27995", "12307", "30001", "10098", "43463", "52019", "58712", "15688", "14847", "8661", "11466", "50441", "32453", "24511", "6312", "13610", "43302", "54588", "49130", "31586", "58135", "49375", "33123", "43143", "1694", "26799", "48179", "2745", "50620", "52855", "56640", "36018", "46073", "8289", "13384", "24758", "9788", "17259", "55915", "24561", "857", "29581", "16758", "31589", "59408", "18141", "58792", "46918", "28604", "46048", "56492", "26346", "33067", "59465", "31479", "16073", "55545", "58853", "59229", "26099", "7708", "16071", "11894", "47563", "38783", "26949", "39230", "40936", "26283", "29277", "47217", "23150", "55142", "22885", "7119", "4022", "59359", "31594", "40270", "36787", "47501", "38092", "37172", "50783", "23278", "33810", "36284", "18723", "31158", "39265", "40833", "1427", "16562", "2863", "47641", "16776", "55825", "24992", "11150", "20060", "27708", "43561", "5704", "8691", "43763", "53281", "39184", "58099", "44269", "10736", "50519", "43357", "9879", "308", "10879", "22958", "2339", "23686", "45053", "21709", "32242", "48197", "34265", "38954", "26440", "10790", "591", "5313", "39067", "17989", "14065", "42422", "21449", "44077", "53762", "15680", "14031", "4936", "11589", "48351", "38428", "53959", "43649", "51182", "52259", "317", "395", "48702", "40521", "13231", "20565", "58052", "32912", "25825", "18557", "40421", "46490", "12565", "12268", "34539", "22883", "44330", "29929", "39027", "21262", "14675", "47928", "27483", "51249", "6669", "8040", "57215", "52940", "59355", "48411", "23955", "31567", "47750", "48099", "2110", "24380", "17338", "54893", "11484", "27243", "5789", "36066", "41758", "12191", "43552", "9029", "26354", "28186", "26211", "25242", "42437", "51423", "14373", "49173", "29936", "14286", "27620", "18823", "56028", "18872", "716", "19216", "46847", "2050", "15854", "29189", "7528", "582", "37762", "25375", "1041", "32560", "26688", "59586", "16041", "32495", "55807", "48371", "45405", "53083", "10792", "32052", "3131", "33755", "15938", "50263", "53402", "40144", "1005", "12455", "45112", "8315", "10832", "33562", "19038", "17895", "7969", "41825", "7335", "48816", "7423", "37331", "31878", "3526", "22653", "27446", "42788", "54852", "32308", "38812", "50205", "26025", "39441", "11029", "19109", "8290", "38136", "29853", "715", "57403", "53046", "32005", "39654", "8153", "9462", "44335", "58248", "23829", "56993", "44334", "8170", "4383", "20241", "690", "10159", "37086", "27788", "24291", "22009", "41827", "39544", "54549", "42610", "41118", "53879", "56219", "40077", "38511", "33956", "22474", "42134", "47885", "2002", "28815", "45812", "2045", "7162", "36077", "2729", "58494", "11515", "39609", "10464", "14133", "35057", "28231", "17751", "27257", "49382", "34290", "29333", "13490", "31583", "3010", "22134", "30678", "46697", "59476", "27802", "54897", "2211", "24419", "39866", "35148", "31185", "41014", "14580", "11548", "13352", "52851", "10323", "11120", "40113", "12781", "5708", "33811", "18393", "45388", "33741", "15885", "37678", "51794", "6428", "11093", "40890", "16101", "25002", "12997", "51535", "19355", "53374", "489", "53725", "21838", "15377", "30069", "58014", "28923", "1404", "41598", "34736", "38841", "37962", "35625", "4449", "44286", "19215", "12818", "37328", "38609", "52326", "43650", "29816", "4320", "22539", "34980", "14388", "14833", "4701", "58936", "46669", "12769", "43715", "43829", "20999", "46878", "40654", "45309", "3007", "11011", "32644", "18461", "31726", "39484", "6258", "28648", "33874", "1802", "21402", "24020", "34001", "9837", "51562", "44164", "37590", "24583", "57822", "41858", "48052", "10675", "58210", "58471", "14123", "12023", "38801", "12080", "6735", "42260", "53502", "7773", "51877", "30944", "34123", "16276", "15047", "37725", "2356", "18139", "3502", "51471", "21165", "38115", "58600", "6310", "38893", "43675", "25225", "21904", "55802", "32559", "55234", "26726", "2464", "34185", "8952", "53715", "11105", "1235", "39073", "40467", "23571", "10011", "23040", "44884", "20262", "17014", "41434", "40749", "24197", "18941", "17997", "29339", "16217", "57512", "39450", "32865", "48444", "34562", "5793", "53538", "35564", "3803", "16091", "3448", "4691", "32341", "30040", "40399", "8869", "54334", "53342", "52809", "33173", "18937", "14953", "6631", "3963", "197", "49262", "56210", "25509", "39448", "33063", "23331", "15717", "33719", "10314", "29226", "1638", "35615", "37998", "54400", "48680", "29925", "50194", "12124", "55383", "40725", "13441", "54376", "3103", "15836", "23903", "15445", "25559", "49643", "40594", "21353", "15822", "46986", "16902", "40239", "21452", "59793", "56602", "26357", "46195", "22945", "26548", "46146", "9713", "40902", "16702", "47933", "29607", "31568", "33711", "21605", "44524", "54496", "40499", "15080", "8881", "16795", "4290", "37930", "44839", "37395", "3766", "45054", "22636", "42827", "44638", "3231", "31419", "12736", "34936", "51622", "11394", "55172", "54827", "6343", "30907", "35081", "35275", "47325", "50844", "48532", "58366", "11803", "7270", "54621", "12847", "47666", "28955", "50741", "12771", "37347", "5019", "3208", "52954", "31508", "20857", "34594", "38264", "36534", "36876", "5495", "12347", "10610", "20379", "48954", "54731", "31070", "46373", "19008", "26076", "10033", "58025", "37620", "15960", "10609", "50123", "22684", "42709", "21156", "16755", "30727", "53924", "49539", "37825", "58085", "44042", "52976", "12661", "15467", "20372", "37684", "48843", "36185", "15205", "33008", "53053", "10380", "43439", "2780", "53853", "36157", "2024", "40952", "51112", "33141", "16414", "13710", "3905", "43204", "10134", "51082", "50650", "30422", "51608", "36526", "52034", "25598", "37273", "25427", "30264", "22593", "48095", "47300", "11262", "5730", "11491", "10900", "40403", "27454", "28729", "52707", "54786", "27045", "49763", "30524", "2294", "26210", "6478", "633", "50080", "55293", "45550", "18344", "47013", "14416", "11274", "7637", "20570", "26060", "45075", "40029", "40567", "47943", "35284", "17912", "13855", "35073", "27224", "30028", "15438", "45811", "54326", "30094", "30991", "13094", "58363", "36988", "3521", "9037", "51200", "9245", "1807", "15155", "24892", "3315", "49843", "2899", "55198", "16365", "50558", "4260", "1367", "59288", "6127", "13007", "9218", "49589", "18056", "49368", "52820", "58772", "59803", "4135", "48215", "51498", "56052", "56349", "21681", "57977", "13132", "23597", "26762", "37016", "48459", "51521", "17555", "16926", "51192", "56522", "55816", "1242", "58886", "41309", "45878", "39148", "13214", "38447", "3537", "47125", "38120", "43972", "7483", "21461", "2309", "23240", "8227", "842", "6773", "47016", "48210", "9003", "22750", "13812", "58881", "3841", "50105", "49566", "10995", "12550", "9569", "35966", "31284", "15643", "47061", "6466", "31588", "6167", "23046", "40153", "57046", "21511", "49739", "57625", "30513", "11066", "46634", "5760", "28526", "12674", "56078", "8415", "22407", "44120", "47964", "57330", "16681", "12725", "48728", "40358", "17801", "28209", "15437", "25247", "28057", "36511", "40383", "14644", "40751", "14298", "30330", "47372", "45505", "21163", "51127", "12257", "35378", "30368", "49193", "16474", "55504", "13884", "39748", "30801", "21991", "34260", "21840", "42761", "55988", "32648", "35841", "17607", "8222", "31388", "52495", "20673", "26500", "14176", "50004", "9813", "9740", "3180", "57943", "7532", "56345", "46665", "40478", "21321", "48400", "23677", "55933", "16292", "30576", "57127", "58157", "43948", "59116", "14201", "24359", "42717", "51005", "10918", "32915", "22739", "57130", "48119", "42996", "24966", "40451", "39723", "53865", "17301", "59496", "44806", "40970", "14130", "13491", "31872", "4294", "32953", "40510", "16853", "47046", "13070", "42264", "35983", "16497", "13459", "44333", "39373", "1709", "15833", "56607", "33302", "22679", "26521", "24802", "37626", "32505", "51453", "44676", "25479", "12096", "45785", "41361", "14511", "20909", "43601", "4388", "51641", "32682", "15444", "25110", "46778", "37571", "27396", "50595", "38248", "47008", "34585", "19523", "54617", "7574", "24394", "20236", "18359", "33365", "39312", "48240", "20409", "34304", "47947", "58470", "56834", "112", "3033", "49780", "6557", "57068", "58250", "50710", "47173", "19327", "41970", "20027", "55637", "37396", "28632", "10721", "56894", "55016", "8288", "6279", "12600", "10916", "19824", "44865", "54851", "25737", "28241", "34106", "33929", "58605", "712", "20793", "9526", "51740", "37774", "7958", "39887", "34695", "17795", "14859", "38312", "37266", "34859", "37214", "39725", "21718", "27385", "10275", "12275", "52080", "23120", "8580", "42184", "35006", "3362", "26951", "1447", "11994", "49392", "16148", "10236", "59348", "15900", "24986", "3619", "38603", "34236", "39921", "11606", "48665", "14401", "3851", "24526", "1136", "39954", "51874", "57232", "36928", "41209", "57791", "29590", "42307", "27609", "55336", "11022", "38030", "957", "41872", "34053", "33600", "24761", "25052", "13571", "5507", "10472", "25791", "24129", "44175", "55382", "46928", "7076", "36762", "21168", "25499", "14942", "22984", "22364", "28475", "1413", "27961", "32506", "26406", "12570", "38497", "46689", "41567", "26470", "51744", "34335", "14520", "62", "6722", "12149", "53526", "7681", "35279", "49762", "15226", "23321", "13845", "22200", "58456", "43639", "52806", "25483", "45200", "3873", "2033", "16994", "53082", "48562", "31925", "31886", "43036", "58602", "18680", "21502", "38024", "50245", "41665", "41025", "43659", "25757", "1392", "40584", "39300", "15750", "4343", "58728", "9890", "3249", "5759", "19174", "51287", "13819", "21593", "963", "41455", "56187", "978", "39915", "40324", "22751", "56864", "23123", "4749", "8274", "57365", "29294", "31459", "43413", "6047", "43154", "22282", "55270", "38077", "13999", "56489", "54451", "34842", "47855", "10384", "24817", "28699", "46488", "45975", "13829", "58656", "14226", "1143", "51507", "19646", "32251", "53561", "18018", "6115", "53097", "10826", "10532", "39044", "45203", "1757", "36341", "14874", "14510", "53487", "53682", "6799", "5314", "59032", "50013", "50750", "54299", "55143", "12914", "124", "46167", "12848", "21878", "13470", "6643", "54687", "20504", "46921", "9030", "32957", "49473", "24189", "28271", "33202", "20737", "57970", "11995", "45342", "55633", "21801", "15372", "44684", "28044", "43325", "30572", "19545", "6882", "54637", "18515", "8533", "25658", "27282", "52581", "13450", "45401", "17692", "50703", "27590", "57334", "13488", "42121", "13666", "16679", "31840", "1224", "953", "8144", "58196", "59788", "40433", "23894", "25086", "57800", "32039", "19236", "45174", "31096", "43517", "38706", "13671", "4236", "32873", "53702", "33861", "50707", "14895", "7505", "29505", "1061", "16747", "12079", "21686", "12095", "6533", "12804", "46722", "15166", "41213", "12416", "830", "25745", "12180", "38366", "53434", "13376", "33093", "122", "18430", "25382", "42609", "3895", "34518", "12093", "27490", "46779", "55259", "41038", "11362", "48804", "23716", "43739", "4802", "35070", "36101", "44443", "1481", "41680", "42560", "22359", "56003", "30540", "14000", "7612", "48850", "48164", "11258", "4754", "40979", "39811", "11921", "38113", "46675", "21466", "9182", "31625", "49981", "45495", "5226", "29001", "9889", "12451", "6464", "49681", "26353", "20806", "10787", "28692", "5782", "11795", "28080", "29827", "26860", "2393", "32300", "27018", "27626", "49726", "41355", "40922", "51435", "55159", "53058", "33799", "37322", "40640", "40095", "1473", "15040", "21221", "4625", "8129", "55830", "55097", "28802", "30282", "44132", "33837", "28808", "25572", "14106", "43791", "5201", "21065", "7201", "993", "45809", "21115", "52677", "31939", "25065", "16646", "29750", "40814", "32325", "55112", "36862", "8883", "10378", "9703", "27169", "25307", "37528", "34364", "40769", "35265", "58971", "14217", "9331", "264", "26375", "39438", "20904", "29490", "40613", "40918", "31033", "55938", "34743", "11961", "31782", "13720", "55537", "14187", "59379", "7907", "1166", "21206", "4884", "35399", "41629", "52502", "43491", "11175", "56617", "36290", "47751", "41786", "26230", "20895", "1589", "59423", "47526", "4896", "19428", "4253", "30970", "21106", "3374", "31763", "56410", "18038", "47010", "25353", "28980", "48216", "7660", "1884", "10200", "14292", "14419", "29813", "9206", "19226", "41799", "14916", "19666", "14154", "15362", "3042", "23194", "32299", "57434", "36889", "40385", "32784", "25624", "4692", "19676", "4056", "22403", "35291", "15952", "22337", "12170", "27938", "35518", "16900", "20656", "48196", "17949", "37784", "58239", "51300", "47988", "58976", "24812", "21507", "9032", "18672", "20154", "13593", "5078", "37293", "26594", "54873", "12248", "36682", "7358", "15281", "3340", "17131", "35575", "42196", "54474", "48231", "40574", "41451", "37369", "55886", "46978", "13088", "51560", "58134", "49028", "32146", "49888", "4791", "2991", "5300", "41279", "16059", "13202", "48475", "58491", "51611", "20294", "52865", "6818", "59845", "57547", "17980", "46704", "32872", "1340", "3093", "6145", "26758", "13135", "26696", "43730", "43497", "13200", "2121", "55433", "37668", "35098", "55000", "17593", "36011", "565", "52487", "52746", "554", "21722", "25455", "29313", "35445", "20878", "54140", "3736", "307", "55158", "57087", "13295", "710", "46713", "16152", "48772", "57716", "18427", "13873", "52657", "15092", "53423", "22158", "26840", "9026", "32304", "54824", "31877", "23750", "48175", "36176", "31951", "38649", "6850", "14561", "27069", "50360", "9718", "49657", "52736", "51963", "14051", "56445", "53157", "24354", "56657", "48714", "58493", "12184", "30272", "46643", "21464", "10925", "2536", "29780", "22484", "30465", "45781", "14827", "14790", "20630", "16583", "11384", "7874", "42637", "47124", "34312", "31250", "57939", "10044", "35885", "15558", "13344", "45307", "25408", "5402", "37983", "8873", "693", "19883", "36243", "49370", "42105", "24851", "29315", "43", "17081", "3426", "56794", "18338", "17558", "39358", "57995", "55743", "43883", "53355", "1898", "32328", "42065", "46899", "5050", "54310", "12138", "57999", "22427", "32194", "27963", "8449", "8373", "21632", "59765", "57409", "17732", "9075", "16422", "34221", "639", "8216", "37779", "45268", "33157", "36161", "48929", "17244", "47735", "34476", "20716", "18561", "21790", "54562", "58907", "52346", "14225", "18361", "45833", "32256", "20230", "18530", "2983", "53223", "15036", "5911", "56579", "36655", "39102", "58730", "37622", "20433", "16119", "35582", "1147", "48790", "35294", "22459", "57080", "53763", "54531", "44419", "2813", "1204", "23072", "59159", "53586", "54537", "7965", "34339", "49404", "27633", "50042", "29280", "44960", "57686", "20270", "30847", "27339", "34902", "37570", "44889", "58744", "24653", "47434", "52827", "16418", "51823", "57217", "28713", "33027", "59637", "52531", "46558", "26801", "57171", "45882", "12743", "34254", "39902", "36118", "59265", "46571", "31492", "32318", "48019", "7857", "5896", "6552", "11292", "40348", "4854", "51951", "14067", "37777", "13936", "48720", "55410", "52021", "57072", "15601", "38727", "55530", "20785", "24484", "57429", "48473", "16882", "28694", "27781", "676", "34890", "8932", "56099", "8305", "40587", "51400", "31563", "58372", "25783", "24239", "27949", "43688", "49165", "41286", "42554", "20913", "10444", "33816", "58514", "19274", "39765", "41562", "29414", "3144", "12293", "9685", "52110", "42407", "28868", "12890", "57866", "50555", "48199", "7581", "8492", "41055", "57356", "19444", "28898", "54826", "44692", "729", "44502", "41829", "35013", "16904", "38107", "50790", "57581", "1796", "24171", "32775", "21184", "6932", "52164", "1905", "12834", "34897", "57239", "22201", "58141", "1308", "30854", "5846", "51030", "59801", "33453", "34322", "25241", "16774", "1511", "50523", "2093", "57160", "56499", "29329", "8329", "54107", "28340", "12314", "4441", "43889", "33342", "26593", "59144", "28164", "18583", "17449", "11487", "21161", "55680", "7992", "32188", "17381", "45655", "31255", "16490", "5157", "25288", "46728", "23531", "46372", "13082", "2630", "33508", "46104", "47127", "17663", "30243", "33972", "24079", "49172", "17271", "51104", "42807", "34320", "18650", "46075", "17877", "42914", "14037", "49889", "54279", "30889", "6360", "10554", "17070", "21673", "13644", "50227", "3953", "55662", "57218", "38824", "44024", "14843", "26693", "2166", "16708", "14587", "30496", "18184", "24779", "23191", "15529", "14909", "51961", "24204", "26183", "53110", "19300", "10648", "18433", "9516", "55295", "19169", "29461", "34700", "36235", "54407", "16438", "35777", "57763", "57386", "22744", "44832", "32540", "39268", "31909", "58011", "40379", "13182", "49100", "4406", "25078", "30543", "52209", "53391", "57205", "53728", "24587", "50068", "33903", "53825", "6659", "219", "59495", "41723", "29597", "21415", "26658", "12500", "30503", "17038", "30981", "30812", "7678", "22192", "25340", "56491", "56041", "55014", "6502", "34000", "25814", "234", "21928", "26792", "19467", "16325", "21337", "35230", "25503", "21554", "29842", "42922", "14596", "54232", "46446", "52863", "45319", "49683", "1916", "57248", "26428", "56121", "25997", "3812", "834", "3524", "6136", "26543", "19266", "15042", "13178", "15893", "1028", "51648", "46142", "46679", "52265", "411", "24236", "11908", "12235", "35066", "9873", "33573", "37636", "5087", "33127", "8761", "55621", "37040", "22195", "2528", "47726", "47825", "9761", "44124", "224", "29647", "49072", "222", "6711", "51418", "36927", "7440", "37607", "13653", "52725", "26787", "3378", "55025", "59000", "50458", "20723", "51439", "59335", "43550", "29123", "19795", "32150", "37765", "58625", "12332", "7555", "55337", "57369", "37790", "52437", "24637", "2783", "22580", "31309", "7605", "11059", "59744", "36192", "45416", "2412", "46754", "3871", "19840", "43607", "30663", "25436", "35906", "15609", "9482", "47154", "21252", "717", "17153", "19280", "8142", "42507", "32195", "19165", "14458", "7518", "20865", "14831", "32140", "4816", "29526", "37680", "39670", "47957", "15545", "11892", "21276", "14763", "20804", "55858", "28454", "3686", "50251", "52913", "47004", "14070", "17051", "48565", "37184", "57056", "43977", "25994", "1640", "32029", "1852", "5161", "2178", "47813", "18399", "54692", "27093", "33477", "12133", "36498", "52451", "32074", "18805", "34746", "11462", "42941", "49093", "42017", "20564", "30176", "38137", "30592", "55979", "32792", "7420", "34654", "10793", "49049", "56288", "21292", "41967", "30463", "8924", "33532", "47999", "15547", "31365", "32769", "21678", "35391", "15089", "36250", "5298", "56939", "509", "8220", "28791", "55133", "22079", "40312", "57569", "4465", "47932", "34056", "17191", "12318", "1389", "37920", "8307", "33813", "55473", "44881", "37306", "55845", "10734", "51480", "58480", "17365", "477", "56312", "52313", "5854", "3304", "7261", "10693", "11817", "31784", "52460", "4632", "22699", "58988", "17116", "5151", "43924", "17094", "25838", "29525", "37639", "12521", "12158", "37449", "1886", "22269", "27899", "45997", "46050", "27596", "26301", "13583", "38997", "30842", "36155", "54057", "24440", "10375", "7433", "41003", "28767", "47809", "7308", "31010", "46333", "16945", "3729", "35326", "22129", "8192", "21821", "18367", "8601", "10604", "18208", "33013", "47176", "35780", "4412", "26057", "30613", "50334", "46247", "9239", "41449", "54362", "31068", "50603", "55239", "370", "43217", "7197", "13365", "32565", "16233", "43599", "37267", "15842", "27689", "34124", "36121", "29147", "20253", "3958", "2900", "37201", "54281", "8672", "11788", "43329", "19224", "15587", "54682", "39494", "6982", "34356", "57659", "45118", "11598", "4369", "29811", "12167", "44306", "8732", "30710", "48419", "4454", "27090", "58951", "32824", "52421", "40124", "45189", "38403", "696", "9580", "53130", "16326", "56521", "57555", "16182", "44323", "21952", "38552", "28948", "41136", "26794", "56206", "8994", "25878", "48143", "8101", "20123", "12337", "44449", "38372", "5391", "46861", "27308", "812", "7317", "43619", "30859", "57951", "10296", "42353", "19253", "30383", "9194", "57451", "39611", "9790", "9301", "5557", "45003", "48860", "50760", "23756", "40596", "18919", "51", "49560", "3111", "27278", "15429", "31347", "37478", "56531", "30720", "4987", "42311", "11246", "17520", "26138", "59588", "48466", "12491", "7435", "26753", "1637", "33287", "48506", "471", "21622", "52208", "41566", "7626", "54066", "17427", "12484", "57638", "36841", "14670", "55125", "28287", "55558", "42988", "50695", "55227", "26606", "27667", "59147", "59081", "5145", "8885", "36196", "53454", "5566", "53708", "37618", "40000", "27049", "10593", "9543", "26735", "33749", "55601", "19090", "52577", "59011", "49620", "25310", "5253", "7748", "28387", "58329", "36524", "5956", "24500", "31868", "48170", "46098", "17341", "20678", "18071", "15724", "13209", "22693", "2693", "45061", "15550", "9290", "15694", "4585", "23732", "9278", "14382", "77", "43293", "5600", "43996", "14189", "30559", "55632", "56058", "36973", "44063", "22235", "843", "32827", "39909", "38735", "21843", "19350", "5408", "41988", "15255", "50056", "30476", "30428", "5182", "1485", "27307", "27773", "57397", "26825", "21059", "29477", "43541", "27505", "33268", "8455", "33939", "7459", "9982", "11371", "38857", "55061", "48744", "35217", "17222", "28959", "50500", "51502", "26844", "13945", "36975", "5651", "51465", "25395", "16612", "52668", "17333", "33077", "15401", "48614", "22722", "23028", "24939", "18605", "11608", "11711", "16525", "26610", "44161", "8268", "41231", "1077", "39286", "24553", "35333", "44078", "11287", "3119", "57835", "38081", "19249", "55190", "11406", "48877", "6476", "26394", "30335", "37576", "49216", "23117", "8346", "45901", "58389", "38961", "2747", "53675", "13504", "36856", "3392", "35194", "52943", "52287", "58072", "21872", "59566", "12733", "39309", "36604", "8654", "53021", "54096", "17287", "44392", "12611", "37083", "39454", "25812", "48642", "58137", "41624", "24071", "11678", "52768", "4242", "52840", "33864", "10366", "35008", "30749", "7602", "44639", "42719", "54925", "32316", "9743", "18200", "45768", "14983", "1895", "13569", "37187", "31304", "40705", "1869", "5172", "43661", "47333", "37872", "19316", "18608", "44254", "18802", "12329", "5477", "1782", "57531", "1438", "3605", "49349", "26250", "13822", "54807", "896", "338", "34759", "37845", "25203", "261", "8758", "54109", "9356", "26563", "26650", "40782", "15757", "3806", "3720", "23300", "7259", "33434", "48285", "10084", "45701", "53349", "4300", "37212", "31930", "35599", "32236", "41621", "3946", "43498", "34673", "42471", "33889", "57923", "54071", "45920", "59938", "600", "50879", "11300", "46000", "4549", "36557", "26489", "33331", "53822", "21886", "27488", "12808", "48445", "51886", "32035", "56444", "32613", "2711", "29775", "57073", "18459", "21949", "31953", "11390", "6201", "24750", "41997", "29777", "32988", "11991", "3877", "44000", "7537", "505", "50438", "35731", "43562", "11549", "25854", "13621", "2570", "2001", "45050", "39868", "18109", "53794", "55171", "56867", "8759", "29221", "57079", "49882", "45609", "39061", "28715", "8413", "18796", "30851", "59299", "29656", "44168", "39585", "55941", "38502", "36819", "56915", "22241", "20582", "25142", "4341", "28329", "46205", "11133", "47772", "57400", "11471", "12892", "2713", "19928", "48584", "21256", "43418", "25361", "11628", "23324", "45390", "12857", "3027", "13854", "17053", "29489", "52396", "10656", "57733", "8581", "27133", "52612", "7845", "4137", "34352", "28243", "37796", "36637", "28931", "54979", "27160", "47281", "17792", "29865", "23603", "51561", "1955", "13960", "18446", "4079", "29276", "27659", "39977", "3318", "17724", "39082", "13472", "53357", "57745", "40338", "48768", "4446", "57261", "24179", "1496", "25127", "48862", "12819", "13173", "38821", "31167", "50580", "28380", "38884", "25602", "46515", "9951", "16570", "15266", "8297", "780", "27908", "56110", "41667", "38809", "31452", "44972", "12398", "14539", "17992", "7084", "56589", "37360", "12850", "4118", "10912", "19177", "41365", "7692", "42680", "26914", "11977", "8433", "36072", "18905", "54130", "18715", "377", "755", "50853", "20950", "23926", "17468", "27130", "54572", "27683", "13669", "59842", "40817", "57000", "38822", "26867", "35825", "942", "387", "8947", "8298", "38903", "45651", "21826", "21286", "47407", "31261", "30529", "38579", "1785", "35537", "40575", "6876", "13053", "59316", "20581", "6442", "21708", "22803", "22168", "15286", "19365", "42130", "57117", "28168", "165", "15465", "2249", "37426", "58476", "20889", "57481", "49996", "41250", "1265", "29387", "20116", "40771", "37105", "53406", "56783", "1498", "7159", "57393", "52347", "32979", "28645", "58672", "26102", "2041", "49118", "4940", "46965", "42734", "58944", "49536", "27916", "46510", "41381", "19118", "45135", "40431", "22875", "496", "9348", "58427", "56506", "14357", "2525", "30530", "36186", "5276", "46746", "8468", "26214", "29373", "13525", "39864", "41631", "59354", "18249", "38544", "58294", "40166", "23818", "59126", "31427", "33416", "41336", "12902", "543", "58246", "10785", "47214", "37999", "30792", "21706", "42225", "21033", "52549", "24328", "48802", "24476", "15171", "6077", "52248", "39114", "41423", "12207", "18898", "33420", "56026", "53439", "519", "19407", "18940", "34301", "44687", "9172", "34491", "47054", "22385", "39481", "12483", "6381", "9776", "34801", "45023", "3391", "58687", "12219", "54110", "29804", "38720", "56215", "29764", "1872", "24173", "49874", "24499", "47701", "14206", "38485", "36403", "29732", "29630", "44097", "22977", "15483", "26171", "15698", "22125", "31967", "19611", "33162", "8943", "10848", "3604", "513", "44304", "59175", "17302", "27320", "29923", "57622", "30007", "48131", "46511", "58022", "18968", "9983", "17884", "14222", "36363", "51109", "44998", "54534", "59777", "14546", "13247", "11712", "50418", "3147", "43795", "40591", "7597", "32810", "10365", "41113", "28323", "9609", "37380", "16367", "50642", "28578", "36203", "6940", "38901", "39603", "22792", "34224", "36824", "55456", "2705", "8679", "52781", "30948", "34870", "43367", "55047", "44051", "57438", "22974", "9248", "53368", "12888", "43354", "50883", "17004", "22062", "47983", "27047", "14236", "37244", "8530", "7620", "2850", "22649", "12469", "58127", "5635", "53997", "57470", "7224", "51298", "31735", "15158", "54202", "38142", "26242", "53013", "5498", "14278", "50232", "3725", "17589", "31381", "25784", "52147", "56094", "48942", "6202", "45082", "5655", "18407", "41578", "23284", "36009", "16836", "56565", "57398", "54078", "4161", "11669", "16402", "31727", "54194", "18418", "24613", "51427", "56279", "9189", "11590", "41440", "1010", "58166", "8209", "44680", "638", "43711", "40913", "56822", "40604", "10420", "38858", "6980", "13379", "31973", "57025", "45471", "14782", "35095", "5666", "41899", "5788", "22375", "5794", "2177", "6666", "24200", "38112", "1516", "9764", "54624", "2969", "46638", "32943", "11220", "9885", "37756", "3382", "5008", "59472", "36489", "59460", "4841", "57100", "32379", "16245", "51350", "33362", "24290", "46092", "53395", "28381", "40589", "44094", "44285", "20571", "53867", "44991", "22236", "23517", "59735", "20593", "40277", "23317", "38597", "51827", "22249", "546", "25723", "59685", "43642", "55267", "5731", "40355", "4036", "36514", "9546", "36873", "52805", "30308", "56885", "21962", "24485", "38199", "55326", "21187", "23421", "34030", "31778", "35490", "28211", "53778", "48244", "4578", "653", "44488", "43781", "42653", "32956", "5020", "41533", "25959", "35359", "57867", "366", "2269", "42415", "3817", "10313", "59248", "30888", "58546", "32116", "1821", "25547", "46561", "44213", "59591", "39834", "47311", "56646", "49822", "34877", "45714", "628", "28439", "8057", "2916", "13816", "51683", "15309", "6189", "54975", "8835", "29756", "40251", "9113", "24793", "22347", "23841", "32759", "6308", "3269", "6518", "56825", "42504", "26545", "10571", "44896", "26957", "45100", "39331", "12166", "17102", "19520", "25790", "49973", "31988", "39072", "33011", "56898", "38456", "668", "41862", "49938", "35873", "34784", "19348", "37289", "22017", "48723", "36471", "30992", "28230", "16901", "36454", "22666", "4141", "825", "41654", "13262", "58748", "52716", "34873", "46219", "36984", "57292", "36776", "38259", "10436", "29956", "26368", "56511", "49875", "34264", "14389", "19346", "12651", "39247", "49532", "22991", "30546", "4924", "45121", "45208", "18346", "28674", "4358", "46448", "54966", "23521", "7836", "19268", "34061", "55128", "13438", "18818", "14634", "9311", "35152", "49358", "50367", "29823", "33868", "28984", "51017", "11506", "58069", "56989", "18120", "51458", "54353", "25768", "32758", "14488", "51856", "21783", "16194", "16048", "4346", "20744", "55385", "36814", "52013", "30290", "22434", "46071", "28452", "19491", "57785", "19315", "1067", "6138", "3060", "43779", "38951", "2047", "55863", "45538", "56346", "49634", "7374", "36974", "33964", "43057", "47653", "55449", "18731", "6540", "39140", "37235", "23876", "2885", "9063", "21128", "50550", "15729", "45859", "47981", "54519", "48404", "35393", "49109", "20836", "394", "25275", "35223", "11075", "23381", "13500", "6712", "4286", "55045", "31174", "21462", "36756", "53050", "3587", "8326", "20466", "56741", "19106", "15648", "7793", "3858", "42241", "1634", "20082", "14954", "25160", "43205", "34862", "54767", "44489", "31405", "8879", "36706", "13342", "58932", "20204", "36123", "53346", "58954", "45887", "48704", "24529", "15504", "53672", "40136", "14793", "51083", "6335", "37224", "15311", "8784", "8841", "31251", "29265", "25178", "56535", "37701", "27738", "31254", "52762", "34420", "54600", "58200", "47121", "1582", "30884", "33197", "35918", "26904", "23474", "39445", "22335", "8964", "21061", "34713", "59578", "5893", "2082", "17300", "56249", "22314", "7252", "28598", "6316", "33596", "57394", "6440", "29430", "59530", "25031", "42626", "19624", "40231", "6790", "51840", "24315", "36079", "44033", "21277", "51371", "39350", "28539", "13273", "12593", "48388", "55767", "11897", "16231", "13543", "34752", "32605", "4950", "15800", "14746", "19459", "19695", "12005", "25932", "24584", "21367", "17476", "43441", "32588", "6646", "22412", "31946", "55303", "56826", "24897", "37939", "32751", "46351", "46212", "17840", "56133", "9343", "55011", "16516", "19947", "27156", "55275", "41507", "40208", "48064", "59076", "50376", "50492", "50593", "18981", "28864", "55904", "23967", "39760", "32024", "18392", "28440", "54967", "13198", "57412", "19562", "42752", "12477", "49197", "56235", "3631", "26776", "49416", "51450", "30706", "18084", "10026", "12100", "50452", "45981", "31758", "37753", "54614", "292", "9503", "9892", "35888", "52332", "3473", "39749", "35061", "12075", "53801", "59484", "34792", "28497", "26676", "11445", "30232", "18694", "52015", "3094", "18617", "34684", "56138", "2131", "18321", "46370", "40723", "19422", "52800", "4552", "9866", "37694", "16218", "6508", "11017", "24387", "11256", "38509", "46572", "45155", "18552", "16672", "15392", "6059", "7075", "26095", "11243", "35784", "23458", "31608", "42967", "58540", "31836", "15892", "6550", "47022", "43347", "46850", "26979", "58271", "20986", "2733", "15887", "3413", "11713", "10827", "9329", "24437", "28861", "2199", "12885", "49963", "40015", "2915", "28762", "35450", "56564", "11683", "6583", "16917", "37234", "32976", "47315", "22990", "12668", "38386", "31449", "32027", "34242", "13922", "11207", "33676", "47426", "4485", "31169", "24650", "27769", "1597", "21895", "353", "42073", "47901", "31597", "39918", "2566", "9574", "34625", "1541", "54164", "924", "9043", "32747", "44553", "19595", "41064", "8323", "9200", "37390", "17943", "44142", "44240", "59107", "23564", "32706", "50127", "36278", "50994", "48644", "21433", "8333", "23546", "580", "52310", "56771", "19908", "34959", "59492", "25789", "6572", "59675", "47192", "11452", "4278", "36920", "4857", "40894", "53026", "21501", "28498", "44895", "7603", "53501", "29560", "7790", "3470", "20640", "57207", "41764", "29506", "34485", "18670", "9269", "47058", "27184", "48670", "50457", "30634", "29479", "57910", "25039", "51078", "56394", "6450", "59654", "44820", "7382", "25446", "13154", "43969", "4254", "20778", "399", "57435", "9321", "45106", "7832", "55431", "12111", "1696", "2198", "28677", "31843", "25622", "30403", "53040", "49684", "32694", "24246", "30312", "28117", "14967", "42542", "55281", "40824", "35670", "37432", "20140", "54132", "18244", "19072", "3108", "40555", "26007", "27848", "16694", "50930", "43916", "11930", "25320", "33115", "43140", "32371", "34003", "44867", "5790", "3733", "2812", "12880", "11809", "23502", "44394", "16128", "38657", "16860", "35823", "59280", "23348", "13878", "17664", "22619", "6158", "29307", "34115", "55742", "45960", "359", "51675", "23647", "6058", "17637", "44824", "45247", "34637", "31869", "5880", "14253", "33767", "51042", "17067", "40400", "48594", "8285", "15664", "56018", "59217", "3186", "43147", "22220", "14646", "23241", "59306", "32586", "48696", "30541", "27746", "1255", "27691", "8203", "50944", "18183", "531", "30573", "18800", "24513", "32776", "29416", "40325", "38930", "17257", "52240", "195", "41893", "50207", "51913", "37913", "26830", "58019", "34619", "58500", "9532", "34241", "36180", "22418", "16187", "26660", "39982", "42715", "11424", "20935", "18006", "14411", "31333", "41642", "14664", "4112", "43629", "20295", "35329", "52753", "46402", "22113", "56516", "42329", "37139", "58195", "4404", "49255", "14512", "33205", "67", "17187", "48347", "36593", "45115", "16149", "31044", "40868", "21705", "4473", "28609", "36512", "56224", "53721", "34723", "30246", "3280", "5711", "26365", "56732", "51217", "22188", "24648", "10051", "28196", "54813", "13277", "1950", "30680", "32679", "41362", "23989", "35622", "32795", "30768", "41147", "27378", "24287", "3278", "16065", "9180", "27919", "9509", "56512", "17686", "56808", "51800", "25020", "55373", "57387", "28734", "42090", "34703", "42795", "46015", "4200", "56273", "39896", "40105", "49076", "26326", "56456", "9505", "35990", "20361", "44340", "44742", "19427", "17398", "26934", "24888", "4561", "13774", "19401", "14932", "27194", "53216", "56571", "7247", "50975", "56128", "5361", "8751", "1037", "1199", "53382", "26097", "47387", "46412", "22910", "33451", "15407", "18749", "38776", "21114", "49649", "25927", "26122", "37175", "40820", "49761", "47", "47818", "25123", "44480", "31219", "10251", "20441", "32301", "39639", "41954", "5075", "8047", "58174", "57741", "23229", "10547", "25696", "14911", "41073", "59261", "43139", "31523", "6359", "330", "4492", "20061", "28434", "57358", "18149", "34046", "18334", "43950", "9316", "19024", "20649", "22892", "29251", "46182", "49088", "26435", "40167", "20146", "47096", "22384", "13089", "6217", "29952", "7098", "7479", "17120", "33151", "41388", "55246", "18059", "35430", "18775", "56430", "20386", "27990", "13617", "52398", "40800", "55427", "35748", "17105", "31690", "2273", "16558", "10005", "2069", "30092", "46509", "55467", "55290", "27656", "35836", "40200", "24678", "19258", "47557", "17380", "36953", "42050", "38971", "49633", "33192", "42380", "42252", "38280", "1928", "6444", "49377", "33515", "27138", "49031", "34164", "18257", "28461", "26020", "15673", "56252", "13079", "26156", "31259", "46444", "53410", "11114", "53151", "6236", "28841", "59477", "27353", "23987", "35123", "50780", "51984", "38229", "6957", "28940", "42013", "52062", "55848", "44191", "11943", "12194", "59420", "10262", "4588", "6815", "35216", "15244", "46905", "30688", "31815", "54998", "52477", "13662", "49852", "38293", "45979", "57978", "12903", "25467", "50488", "17852", "37434", "57424", "10662", "40107", "14531", "8669", "1472", "1570", "34586", "59258", "7967", "5105", "57658", "20505", "45179", "23954", "42886", "21689", "9798", "57414", "45447", "26645", "1624", "57762", "8963", "52317", "18027", "55273", "40437", "56496", "42440", "31", "17255", "32665", "19292", "24040", "6952", "26123", "59475", "32293", "16484", "5996", "3986", "21109", "18697", "29000", "18930", "7225", "4553", "47027", "52912", "41606", "1052", "32907", "50735", "24830", "3717", "799", "59677", "6784", "45704", "26983", "47144", "40160", "55522", "19735", "35816", "47361", "56619", "9405", "22406", "33620", "47562", "1937", "45606", "23386", "32584", "34724", "6233", "41859", "22465", "17080", "5073", "31812", "43320", "49718", "44104", "33309", "14472", "49248", "39341", "49522", "9038", "8311", "8715", "35650", "2870", "47360", "27730", "46617", "29983", "4908", "22378", "4558", "18911", "7310", "13361", "13362", "31144", "5539", "27589", "47558", "8882", "35788", "51315", "46740", "58967", "23604", "37547", "55617", "41230", "974", "35264", "33352", "15778", "51468", "627", "29631", "58882", "5807", "28410", "3225", "51921", "19127", "50142", "38663", "21823", "36803", "19152", "18405", "16769", "54742", "4489", "52831", "18054", "8868", "26458", "30130", "1104", "25492", "31306", "41816", "16974", "57067", "13373", "48587", "50381", "41648", "45442", "48943", "46745", "20978", "868", "30097", "2893", "1153", "5815", "58370", "51474", "58437", "58898", "30698", "32689", "17200", "35540", "13987", "27421", "21666", "39335", "2825", "55553", "2815", "58518", "16828", "24162", "14221", "56897", "17314", "28362", "3910", "52482", "34215", "53464", "27866", "17170", "57496", "20137", "33575", "4496", "37550", "22462", "58547", "48497", "23481", "35768", "9273", "29168", "31882", "5739", "35305", "1883", "40642", "45181", "36581", "12829", "31720", "43034", "58516", "728", "16038", "2730", "4048", "52797", "34172", "55878", "20110", "8165", "20442", "21227", "7094", "4391", "23615", "16252", "28877", "39750", "14725", "46476", "56958", "24759", "32803", "2103", "45922", "21241", "18384", "5466", "36330", "9275", "49053", "57788", "55750", "2838", "33888", "45521", "20275", "25354", "43528", "15194", "18172", "45224", "15823", "3364", "58564", "38698", "25025", "40861", "31635", "16921", "56490", "46761", "30080", "54899", "8933", "49957", "7097", "24585", "33327", "46173", "7761", "43859", "43646", "42109", "56629", "1015", "21812", "25626", "4231", "37970", "15414", "38681", "19871", "1844", "25453", "40582", "8414", "44883", "45168", "21930", "27383", "41061", "49863", "41559", "35555", "15986", "46895", "6883", "36848", "28825", "13169", "25929", "51981", "24590", "46461", "21459", "7441", "1294", "49949", "51063", "47654", "51995", "51299", "25585", "15565", "2354", "4999", "56638", "2264", "52620", "24787", "23866", "21039", "49920", "43930", "7985", "50830", "44057", "52470", "27517", "10620", "25516", "2662", "38246", "7540", "18178", "43878", "38010", "15008", "5559", "24193", "140", "27886", "26839", "39397", "45232", "20157", "43746", "44766", "50299", "42860", "43179", "6861", "25619", "1125", "10097", "28925", "45786", "56786", "22143", "40537", "16619", "48036", "59912", "13715", "32281", "24885", "29896", "45883", "3193", "7437", "48766", "13750", "20378", "45392", "38451", "50174", "8738", "22835", "55919", "54859", "840", "24733", "28152", "12517", "27252", "57608", "59932", "25306", "40579", "19273", "55447", "47421", "24805", "54345", "35901", "44242", "14852", "6849", "13939", "5606", "41782", "55418", "38441", "2314", "51767", "48376", "6779", "40617", "46457", "44661", "16923", "41224", "40084", "19874", "42607", "44562", "14999", "52329", "28995", "12501", "44416", "4640", "33976", "57882", "11453", "35151", "39333", "10901", "24267", "33236", "10877", "2588", "14188", "36111", "13015", "30300", "19961", "53961", "22477", "59171", "56626", "47257", "50325", "54201", "33247", "1612", "21592", "49201", "22310", "10087", "19124", "56845", "15029", "35963", "9970", "54639", "27330", "10396", "42106", "11376", "30813", "39778", "35908", "9993", "3161", "48256", "732", "48970", "30611", "13505", "38257", "3708", "26453", "5264", "31203", "5339", "23390", "59571", "12923", "51605", "43951", "25935", "5132", "57809", "6333", "43817", "3457", "14564", "30746", "19104", "39175", "38005", "52393", "4664", "45926", "39787", "28041", "5991", "46888", "52350", "34049", "35364", "19953", "32363", "22506", "37202", "39619", "29924", "44603", "24350", "28141", "51321", "48988", "12936", "8665", "47400", "14570", "36416", "22203", "51833", "11922", "3485", "26736", "38243", "20192", "7630", "27324", "28606", "12200", "55986", "47207", "16863", "495", "24508", "44709", "246", "19956", "5000", "46802", "29431", "49581", "21083", "22184", "16608", "1899", "15463", "40031", "47880", "27896", "13839", "21569", "28003", "54241", "7900", "33033", "27051", "11981", "5520", "55666", "44115", "40669", "37586", "45260", "40848", "25426", "839", "52607", "7141", "52342", "26767", "9705", "266", "26125", "45898", "21836", "41006", "2219", "11381", "33435", "45804", "32889", "20644", "9883", "35219", "27560", "18505", "16662", "18077", "47877", "54414", "59145", "35032", "9911", "18254", "52584", "21983", "34929", "50427", "2806", "12048", "43584", "59321", "36543", "11765", "46626", "46078", "33662", "10729", "35328", "9177", "50126", "14036", "8527", "5183", "7108", "31603", "34799", "56400", "20949", "47523", "3726", "23266", "46248", "51116", "8858", "56367", "41182", "44134", "11470", "20576", "58386", "57192", "11538", "46857", "5033", "29470", "47863", "17696", "23196", "57920", "11365", "22181", "26846", "5567", "46581", "22120", "47511", "48281", "4447", "233", "58357", "38582", "54187", "49789", "4067", "35215", "1987", "19263", "38941", "6251", "55229", "15004", "32357", "2824", "55630", "21361", "2030", "24156", "49095", "16033", "25240", "21385", "3695", "2911", "7343", "37714", "23943", "17033", "50384", "56293", "2970", "22932", "49703", "53325", "12969", "55260", "49820", "36266", "11819", "50617", "18948", "59104", "22919", "45481", "14111", "40774", "6966", "40668", "7666", "48166", "7562", "2402", "46547", "29379", "38873", "43587", "48698", "305", "34451", "15265", "48952", "34826", "56195", "17359", "2460", "55859", "23847", "6110", "33061", "23689", "27532", "13681", "59315", "17420", "21392", "10445", "18484", "31844", "41096", "45127", "55814", "16517", "41513", "6948", "37884", "52452", "51388", "29278", "17899", "6596", "16859", "1342", "12836", "50793", "44740", "26449", "38271", "29664", "27177", "23545", "33945", "3808", "7289", "28536", "45571", "2357", "54064", "25847", "45746", "35730", "40509", "8815", "4143", "17419", "751", "7436", "3430", "15400", "29362", "31379", "26853", "35905", "32346", "32968", "4109", "31376", "7012", "46296", "45063", "42907", "13659", "5800", "57483", "42274", "6411", "36253", "8200", "58764", "21522", "14113", "24762", "27431", "25274", "26371", "27588", "46534", "34038", "35261", "10578", "17006", "20628", "47343", "52289", "17621", "32670", "9351", "41417", "48679", "12104", "50387", "20105", "35953", "40420", "53671", "52886", "18246", "31799", "14289", "10070", "2086", "27359", "7404", "11719", "34861", "49647", "15883", "7007", "2603", "48335", "9241", "10526", "10720", "49595", "17132", "3744", "56162", "30785", "36729", "12463", "26337", "56661", "15843", "30665", "27041", "46160", "36228", "20645", "58232", "17179", "25429", "7759", "53360", "28886", "25621", "14041", "15751", "47150", "23328", "42950", "53310", "18300", "47288", "16122", "10799", "56378", "23374", "50751", "32589", "14703", "31582", "50947", "17236", "20174", "1712", "27151", "49209", "42740", "19564", "10104", "18487", "26942", "45624", "36249", "35910", "26114", "12505", "2446", "48238", "1038", "40958", "58299", "57648", "33643", "49285", "33433", "22837", "12360", "9515", "25813", "22800", "4274", "19351", "14906", "17175", "15647", "8243", "33081", "7203", "28436", "5874", "9570", "28627", "42671", "29378", "54008", "13798", "6743", "30140", "32428", "34892", "16142", "1625", "7664", "26463", "24838", "37520", "17086", "6330", "18851", "14363", "21329", "58211", "33074", "29094", "33482", "48485", "17195", "36421", "20383", "46977", "56446", "29423", "14542", "50812", "17749", "59485", "47370", "25028", "3797", "45909", "23251", "8593", "18235", "1875", "28816", "40944", "17104", "48564", "4768", "31382", "15341", "25077", "24381", "52617", "5918", "8167", "57967", "36516", "35833", "41776", "19469", "12696", "54971", "39546", "18809", "14663", "9362", "44933", "44700", "15520", "55341", "36322", "46838", "55479", "59866", "46144", "51475", "18644", "7911", "51124", "35872", "47920", "3740", "11482", "51982", "38232", "39141", "35654", "6413", "37047", "45066", "7867", "57007", "52858", "42592", "53036", "4503", "37342", "17307", "45985", "29799", "3613", "42020", "19321", "15548", "55785", "46531", "9673", "23232", "38825", "38114", "55614", "26943", "46467", "9921", "27977", "39209", "54283", "55057", "36149", "2791", "26586", "26754", "48589", "45893", "5346", "58632", "7781", "49078", "54325", "23581", "54248", "8443", "56866", "23751", "21360", "51438", "8989", "42911", "30363", "5321", "31669", "26246", "26027", "47808", "2323", "40372", "47116", "52799", "24274", "22425", "48262", "25144", "55188", "17601", "18053", "961", "26766", "58317", "3820", "35489", "27533", "25377", "45153", "26349", "45983", "3704", "56754", "59140", "27872", "53735", "3680", "52383", "6255", "27935", "40118", "45713", "26641", "2614", "23995", "6194", "14677", "33225", "6081", "4041", "33758", "24083", "44321", "34326", "48049", "39206", "28009", "55114", "57190", "57250", "10623", "1353", "26600", "25211", "13152", "51817", "17827", "20065", "34109", "16355", "40370", "26828", "7635", "566", "54960", "5090", "41312", "21703", "28465", "38179", "28571", "53917", "20934", "5464", "11395", "39895", "1356", "55860", "22648", "22796", "33102", "21630", "15451", "43991", "25396", "53343", "20249", "35702", "923", "48671", "58462", "12014", "20654", "26322", "34306", "53316", "32255", "13084", "29062", "2786", "20139", "49223", "36879", "32860", "51632", "35635", "58103", "53055", "22044", "14823", "13866", "39528", "24186", "23654", "55887", "48314", "17034", "36340", "35010", "55490", "4835", "52252", "54113", "47908", "22251", "19776", "55976", "52528", "18297", "17986", "55109", "37581", "14600", "4049", "42495", "2739", "57538", "11103", "53498", "44628", "34937", "55105", "48414", "33189", "59981", "57527", "28106", "17454", "10934", "58725", "52102", "58925", "59233", "7682", "25443", "8357", "14249", "14680", "1829", "2317", "5123", "17032", "12582", "2355", "4152", "56275", "17689", "54179", "17570", "28701", "13385", "53381", "7394", "42395", "2245", "41645", "34222", "4985", "21887", "8863", "39197", "53599", "8064", "50950", "47528", "27940", "44801", "47079", "35180", "58252", "50858", "27044", "23042", "24252", "22363", "56891", "36030", "36052", "29708", "16871", "53601", "20750", "31030", "23839", "58013", "36566", "27159", "24717", "52834", "21112", "14396", "43845", "1450", "9417", "43369", "24607", "37739", "25304", "6105", "54900", "39543", "28443", "27197", "44003", "1958", "58641", "24642", "6788", "48577", "50659", "28551", "57064", "34800", "10797", "31692", "58058", "5217", "2854", "41801", "1921", "30708", "44434", "12816", "1885", "24149", "18813", "55261", "23255", "7880", "1618", "33044", "6983", "10142", "29981", "51966", "6898", "58049", "58193", "20677", "54865", "20586", "39455", "40267", "21026", "51383", "36898", "13804", "32768", "53659", "42175", "22651", "36102", "58877", "24410", "49977", "29116", "29322", "18855", "50211", "58945", "55758", "52699", "7371", "5504", "34632", "57382", "6968", "43949", "22721", "50690", "34726", "42873", "27382", "10520", "43487", "10105", "18166", "2524", "17826", "50640", "27756", "51284", "41714", "39248", "33789", "7095", "8808", "8946", "50318", "13249", "13613", "11700", "24864", "4944", "21914", "50485", "11427", "35982", "31407", "46875", "55798", "2503", "15876", "26709", "50036", "48129", "54448", "52727", "30961", "59411", "41098", "5213", "15069", "24700", "44720", "35285", "40359", "49054", "46163", "6612", "4421", "46086", "31544", "9833", "24490", "35778", "48875", "43224", "7947", "56534", "30433", "7328", "42108", "23784", "50727", "14778", "6042", "11465", "49815", "17811", "47996", "8484", "41326", "1803", "10223", "54007", "53868", "39218", "15296", "30866", "17260", "46562", "46901", "53421", "2726", "47067", "15411", "58091", "1808", "25792", "3863", "16081", "41583", "27568", "47163", "34223", "46246", "41726", "54989", "28373", "43250", "11190", "42138", "28023", "34823", "2954", "9867", "36880", "32030", "2482", "37476", "9558", "7577", "11867", "31470", "55996", "57329", "14826", "36847", "39050", "19689", "33111", "59055", "32782", "31399", "43081", "53224", "38863", "37150", "19931", "53078", "32894", "34614", "50522", "7264", "1975", "31272", "42797", "994", "52596", "23145", "28780", "22608", "34785", "11617", "29205", "2000", "37337", "35978", "3565", "19555", "45270", "3157", "3501", "38009", "58268", "59276", "5842", "32598", "41464", "19560", "25035", "24786", "11030", "16207", "17933", "43560", "49429", "45162", "29244", "33915", "4755", "45733", "10730", "8370", "18138", "55181", "51397", "48456", "6914", "10512", "49758", "8896", "8678", "14873", "36188", "9404", "18990", "2020", "50759", "18409", "2935", "7929", "54432", "27048", "17375", "10834", "28467", "18213", "15366", "19652", "44349", "29958", "3512", "48383", "42985", "15181", "30150", "45427", "20055", "27126", "40173", "32161", "32321", "42039", "43493", "45876", "42202", "45659", "33085", "41290", "35470", "47266", "9392", "25172", "5312", "7507", "36033", "8412", "14413", "6782", "27964", "39458", "53594", "42570", "38757", "42961", "45822", "11599", "35724", "27358", "9002", "4092", "15256", "59810", "6731", "45234", "36264", "44909", "31642", "18326", "58570", "49344", "20059", "46145", "44821", "12498", "27141", "4735", "20389", "39283", "19310", "24112", "57618", "49624", "17418", "55255", "53165", "55550", "20467", "33072", "57011", "45393", "54045", "48046", "45372", "59975", "20643", "13008", "45692", "43421", "45281", "48060", "11272", "57619", "46235", "44189", "59785", "36957", "39004", "59386", "26608", "38805", "58181", "56374", "7895", "44267", "983", "25960", "46682", "51247", "19909", "32239", "1952", "41452", "34401", "34817", "55512", "50265", "23800", "50051", "44169", "52066", "44088", "51581", "8912", "39625", "14131", "53455", "27207", "31038", "9606", "41820", "54611", "3581", "38311", "13801", "31753", "57030", "17647", "54584", "47852", "33082", "2125", "56031", "38265", "3063", "59787", "49777", "52416", "15926", "46644", "27615", "24355", "36350", "6716", "7217", "23665", "28494", "43767", "48336", "38573", "13663", "35435", "3802", "53564", "48685", "30904", "45430", "30679", "20887", "17556", "45071", "7621", "36165", "31978", "46997", "43108", "21389", "3523", "32215", "26893", "52205", "40386", "26443", "1229", "27472", "48169", "2109", "24413", "26486", "20435", "14150", "50140", "3531", "57591", "3469", "9981", "34390", "10733", "32019", "5966", "58263", "46301", "175", "18205", "46170", "40615", "50891", "20278", "4778", "26075", "35607", "19896", "20578", "48174", "21044", "23451", "45944", "35562", "3654", "52340", "29592", "44192", "52115", "32716", "22657", "50354", "45777", "8316", "33595", "29786", "12718", "55082", "6909", "52579", "36877", "40741", "59809", "30807", "33390", "53225", "33328", "7544", "6610", "28043", "50467", "55576", "10259", "33262", "48334", "40681", "16459", "37066", "7299", "46369", "58819", "34566", "42636", "3792", "34535", "33070", "16544", "57946", "58720", "30329", "3455", "7185", "56739", "14583", "44954", "5735", "54090", "33912", "16396", "15562", "1882", "47279", "36025", "13595", "59931", "22255", "51201", "50108", "58024", "42461", "4728", "48617", "48029", "13276", "21557", "3781", "53068", "29004", "13762", "20408", "54595", "28411", "26902", "24806", "11704", "33818", "57187", "49134", "26402", "3048", "17996", "47435", "40960", "28612", "21667", "47532", "35477", "46109", "3698", "30495", "32699", "43700", "44981", "21900", "9935", "38566", "12273", "35938", "29847", "26775", "41925", "36380", "49989", "16439", "24310", "30724", "31801", "19324", "10146", "30773", "19251", "16082", "21456", "18479", "17494", "13639", "57697", "30673", "54295", "53288", "51379", "8083", "12107", "30458", "43546", "32620", "41822", "27864", "36628", "14254", "21662", "13011", "50874", "19367", "23625", "2399", "39558", "36090", "7266", "34944", "5266", "10441", "41944", "1997", "26067", "18943", "21548", "14840", "50472", "1614", "41847", "36103", "56601", "21164", "1951", "50122", "49534", "18888", "46735", "10959", "11881", "30497", "44204", "50675", "725", "4505", "29736", "10525", "3504", "11997", "1422", "10605", "10531", "7948", "30670", "58527", "34284", "43540", "8930", "18871", "36470", "27994", "11344", "26612", "54207", "48652", "37744", "14066", "30185", "47477", "11118", "30756", "19746", "37732", "26002", "16830", "40096", "59063", "2904", "38771", "6987", "4297", "21149", "59507", "33446", "35907", "25246", "56526", "7003", "41278", "46716", "53470", "24115", "39342", "8918", "37768", "35521", "22455", "9192", "11241", "37124", "4155", "53431", "15031", "47481", "35138", "37934", "51271", "27978", "37481", "53748", "3915", "47378", "17869", "20007", "37001", "43516", "23883", "9586", "10265", "58551", "6336", "21993", "52218", "316", "12105", "59289", "33622", "31972", "16239", "11947", "26332", "10208", "15067", "45889", "40779", "13831", "11702", "30166", "2191", "42745", "16618", "52194", "51456", "49323", "12591", "29334", "21536", "10923", "36147", "11635", "3305", "54358", "30195", "5097", "48253", "56693", "2923", "14158", "15489", "10382", "9727", "1379", "6813", "24730", "49918", "16183", "53598", "25190", "56042", "17107", "21122", "51779", "43058", "44627", "48428", "55939", "21244", "7636", "11745", "6494", "52813", "55585", "43608", "47363", "23442", "22162", "34532", "38652", "43395", "16297", "32736", "42848", "637", "19563", "27981", "31899", "59378", "5972", "3656", "3297", "8263", "25494", "13019", "35992", "4897", "49230", "55837", "7818", "469", "3218", "36360", "40226", "54166", "43632", "49868", "25951", "27833", "26845", "54311", "54705", "47091", "26459", "22025", "23015", "33339", "58448", "23334", "36128", "454", "27717", "36060", "8603", "7199", "18435", "41297", "38484", "44198", "57350", "47889", "2259", "2865", "53898", "13478", "37871", "31971", "41644", "50487", "17636", "41465", "54458", "40606", "19530", "34327", "37343", "11249", "13242", "22729", "15082", "54083", "3542", "1609", "48624", "4533", "25513", "53894", "35376", "45815", "20874", "435", "39860", "1001", "19990", "12265", "31942", "53367", "38865", "30367", "29614", "42035", "54484", "14019", "50428", "25186", "51019", "35549", "33940", "43776", "30583", "51953", "53723", "54060", "40064", "17110", "44525", "30145", "32640", "104", "51183", "37177", "34271", "27426", "5513", "29904", "8445", "2767", "48140", "44731", "59721", "44219", "40852", "29337", "47153", "34735", "20128", "39606", "54112", "28227", "43164", "13131", "26597", "17480", "1764", "31125", "33312", "54477", "55765", "35590", "31160", "2248", "57561", "52726", "22198", "30384", "46955", "43048", "23488", "36943", "56882", "55068", "58472", "19065", "27787", "47838", "9756", "7282", "27071", "34912", "48177", "13417", "37519", "15977", "31385", "11768", "17241", "26922", "290", "17291", "13072", "56202", "50073", "53045", "35509", "5776", "51662", "34095", "22149", "48384", "303", "45267", "5294", "48062", "47624", "22381", "39211", "55867", "49256", "53931", "31657", "36408", "53411", "16534", "3543", "55669", "40955", "55656", "17656", "56903", "36809", "2193", "15030", "2292", "22252", "14305", "32857", "44056", "32440", "40047", "17670", "1685", "58119", "5903", "21715", "3713", "24256", "5256", "14103", "57038", "20602", "38558", "34796", "7144", "21668", "20709", "53745", "17329", "6007", "23116", "11154", "5257", "33083", "39256", "8833", "41608", "19362", "30949", "16186", "33058", "9282", "57816", "41637", "51218", "32028", "59761", "20109", "53450", "13897", "6878", "57494", "949", "54054", "28947", "28639", "33867", "30751", "41987", "33320", "30033", "9933", "5912", "7980", "30963", "15404", "23337", "58682", "7014", "43073", "34921", "53650", "12136", "36286", "24661", "23367", "8010", "14759", "22501", "19952", "17452", "20646", "9592", "22929", "19756", "10866", "20318", "31317", "37134", "9576", "10914", "33487", "419", "29623", "41148", "25678", "38429", "15813", "15077", "10964", "45122", "8794", "56651", "34544", "12366", "50002", "16685", "55828", "54817", "50152", "43260", "11008", "57526", "34412", "10023", "5249", "48713", "39991", "45573", "7383", "8430", "31704", "2256", "52578", "43937", "56291", "17347", "40171", "34598", "31159", "14700", "41633", "770", "36759", "52081", "36948", "23486", "41476", "25604", "8262", "46568", "5785", "16909", "35214", "29424", "8463", "39363", "34893", "57312", "5669", "25155", "41945", "37997", "8908", "10424", "6222", "9629", "28499", "56046", "51879", "25638", "7662", "35185", "54351", "25523", "41780", "56477", "46407", "11455", "42037", "58504", "35250", "59313", "28146", "27911", "15110", "1881", "42068", "23113", "38957", "4480", "48328", "21950", "44587", "20589", "43520", "29005", "31970", "54500", "46586", "33792", "45730", "28796", "34939", "27707", "31031", "27749", "41172", "19107", "12753", "32297", "45271", "20614", "24947", "30574", "56504", "25449", "23593", "16563", "24595", "28742", "44429", "42714", "12342", "58834", "21984", "23471", "56659", "11762", "20449", "41900", "42399", "43637", "4107", "47417", "619", "56223", "9215", "29616", "2866", "47502", "29212", "27105", "24955", "34353", "1213", "27950", "33055", "3514", "375", "36411", "11441", "6889", "12539", "25043", "33582", "27638", "27684", "39848", "48280", "42663", "12323", "55577", "51848", "4207", "22798", "47261", "13161", "19287", "57620", "39029", "42229", "48501", "3540", "24628", "13067", "10180", "39040", "10278", "18955", "2782", "48905", "39736", "32056", "44907", "41905", "55514", "54782", "38163", "1291", "14279", "2674", "44505", "4622", "53754", "36084", "16310", "51791", "5140", "32125", "36493", "16898", "28252", "39500", "27333", "15197", "50497", "26416", "7932", "12650", "18431", "2493", "40961", "17367", "56033", "46791", "27031", "18746", "47834", "50273", "36027", "5443", "59463", "1725", "59127", "8190", "54907", "52401", "8349", "10310", "31210", "59919", "10740", "33918", "29371", "12763", "39655", "23000", "42889", "29020", "7255", "57771", "52937", "16189", "9228", "20796", "21255", "30942", "26340", "22756", "59878", "7447", "30567", "41860", "40158", "25512", "24977", "57391", "25471", "3859", "47446", "25753", "21048", "50034", "24621", "14802", "17783", "26037", "32375", "53348", "8259", "13213", "28990", "15091", "44378", "58467", "30090", "16273", "34880", "6650", "27180", "5997", "5108", "35192", "54150", "37357", "12394", "29161", "11661", "10286", "58282", "1066", "56871", "4920", "55721", "55702", "45791", "55103", "55355", "15159", "53054", "3404", "32835", "40413", "45721", "27774", "47204", "25045", "44560", "30732", "2173", "6277", "15219", "53817", "30901", "48604", "35659", "11500", "29331", "57024", "7819", "18253", "42129", "54079", "19094", "10462", "54550", "50748", "21846", "24301", "23273", "3748", "5101", "14166", "39580", "59409", "32673", "25114", "23269", "48579", "22078", "45230", "13428", "33987", "31895", "1700", "42136", "13364", "2388", "40314", "54684", "30547", "4546", "38817", "34234", "48500", "15789", "36518", "5387", "54315", "33732", "9824", "46203", "7579", "35856", "54791", "5930", "30639", "4584", "10428", "48002", "39743", "51689", "28486", "1512", "6761", "43613", "753", "44611", "2787", "43844", "50137", "24748", "51574", "31545", "40945", "18237", "55898", "43927", "24704", "53768", "47309", "3171", "17700", "28860", "54778", "13455", "40678", "26617", "12976", "17185", "49329", "6107", "4629", "29054", "40300", "27544", "29665", "59526", "52217", "2034", "19322", "59905", "44507", "29287", "2537", "26348", "53992", "42187", "40714", "24021", "47994", "42503", "38182", "59150", "21832", "22497", "12712", "50364", "9379", "15691", "10686", "43492", "24434", "28962", "47167", "38641", "49574", "44143", "46265", "40108", "58815", "31028", "10243", "56932", "23111", "40786", "26539", "48077", "19549", "45354", "49503", "34739", "57861", "35577", "25858", "5041", "23695", "36801", "37910", "27195", "20261", "52870", "34489", "3948", "30014", "50150", "41918", "17013", "35530", "57315", "53208", "47441", "55454", "41593", "19787", "2645", "52239", "34560", "12762", "51305", "1155", "30004", "45661", "9306", "11898", "41544", "8367", "25037", "56991", "28228", "24080", "6586", "50730", "38630", "54498", "17088", "48187", "17118", "31905", "20182", "58684", "3018", "34753", "42681", "47683", "56985", "24203", "29637", "48822", "19941", "36095", "48427", "54902", "22207", "8273", "10161", "8999", "14182", "38475", "44531", "39696", "41375", "25327", "47581", "51624", "36439", "33459", "11581", "18221", "5858", "46069", "58264", "25571", "30056", "54162", "53177", "38883", "10828", "23220", "59146", "54476", "9414", "24486", "26438", "40007", "6598", "44498", "55946", "18556", "29234", "2738", "9428", "24217", "20210", "16572", "6693", "52527", "3484", "46565", "7644", "51401", "18708", "21967", "30229", "22069", "10591", "43066", "51515", "15543", "54685", "23828", "51092", "37630", "8789", "57110", "25798", "42098", "9068", "12094", "5035", "15879", "2013", "22165", "15967", "33122", "32895", "27439", "54507", "29627", "25308", "47209", "5957", "13844", "38209", "19360", "21034", "44199", "35248", "6555", "12306", "52628", "58212", "34750", "24097", "35600", "46897", "7736", "28030", "954", "26814", "4199", "50664", "46270", "47074", "43747", "49348", "33570", "28147", "30136", "49787", "51554", "25097", "29684", "41323", "5835", "38369", "56908", "40525", "41952", "30803", "12106", "9826", "55646", "48859", "8380", "44859", "10691", "34918", "52749", "40245", "42386", "52362", "18132", "44809", "31001", "14578", "28790", "23782", "15246", "40163", "39254", "24265", "51723", "45438", "43866", "26868", "55937", "3121", "42392", "25876", "21547", "34573", "3836", "15686", "1613", "42132", "23479", "14204", "42016", "5165", "34710", "49721", "58050", "11694", "10518", "40719", "24128", "2072", "21370", "34012", "8792", "18438", "3975", "24152", "18058", "48127", "46130", "53129", "39998", "11805", "23032", "15", "769", "37271", "19133", "56265", "2772", "59699", "1237", "3369", "56881", "39322", "20650", "4376", "20457", "51022", "32740", "42755", "15018", "58596", "35142", "9387", "31887", "47759", "20036", "21188", "5936", "18066", "36296", "15156", "40463", "25939", "17744", "38512", "36843", "1491", "26373", "7339", "5413", "6406", "38859", "55871", "31322", "16013", "33652", "14399", "30489", "28470", "28353", "52629", "18358", "41386", "31630", "37084", "14607", "29773", "55952", "6579", "22954", "33937", "7851", "30420", "52665", "16530", "25444", "41140", "26392", "57044", "7320", "4639", "28317", "46752", "5034", "34034", "36906", "29175", "8163", "16126", "47248", "29628", "40183", "39636", "23212", "37991", "7795", "2102", "29689", "37669", "1513", "3585", "25165", "50087", "26667", "7340", "19661", "59786", "30086", "5009", "57155", "36075", "35436", "52890", "53272", "32023", "13759", "56302", "15720", "44814", "569", "51415", "47380", "13473", "16825", "24377", "54374", "26094", "27822", "37862", "38846", "31493", "48353", "58424", "34831", "24925", "5283", "49760", "19845", "4007", "2353", "36472", "14598", "41498", "30989", "24841", "36042", "18739", "12137", "8241", "18722", "8102", "17548", "35487", "21929", "18476", "47841", "5974", "3564", "8219", "15430", "50307", "53341", "35086", "7709", "40199", "21683", "29916", "44307", "57163", "36134", "54697", "32444", "32278", "18087", "12624", "6184", "26877", "36400", "19112", "29758", "31941", "47313", "45374", "51757", "48782", "59680", "35196", "41321", "33481", "14235", "56338", "796", "45873", "49936", "3", "975", "6738", "22640", "8224", "9312", "36806", "56848", "47489", "36647", "49592", "8218", "32291", "60", "28911", "30836", "844", "99", "21070", "5927", "4028", "54663", "25762", "37007", "9102", "405", "507", "19758", "4902", "59162", "5181", "52960", "15581", "42894", "58737", "15131", "48128", "22532", "11489", "32117", "56343", "26053", "59257", "102", "17415", "23957", "27471", "57521", "739", "24639", "53685", "31677", "52458", "49335", "20357", "16391", "17446", "57506", "41689", "28978", "9692", "7623", "33523", "38572", "24448", "41311", "41941", "32720", "33991", "48518", "22005", "44177", "59517", "21827", "45036", "16725", "2937", "59767", "781", "15284", "1425", "31761", "584", "6742", "6881", "42113", "21898", "38364", "56412", "4524", "53445", "25579", "43464", "28298", "15915", "31226", "1831", "2315", "31218", "27637", "52946", "41222", "6714", "15726", "4834", "57508", "15360", "21448", "29504", "40417", "22085", "15207", "7601", "39434", "9631", "31062", "38580", "37327", "38412", "48966", "27917", "33804", "51156", "10209", "45776", "10233", "31242", "40197", "52351", "7240", "28431", "48593", "8625", "16628", "59067", "57969", "40209", "26131", "39777", "2720", "10630", "35937", "5175", "59531", "53221", "20722", "47442", "26176", "41001", "7019", "17987", "30804", "54203", "31263", "14590", "50336", "4477", "11414", "22342", "58827", "26153", "37631", "24372", "45557", "55582", "2488", "55510", "3499", "57669", "13648", "11501", "34835", "57708", "11969", "16949", "37567", "36306", "53025", "32919", "5797", "4637", "29273", "26727", "28190", "11751", "48937", "55312", "11323", "44520", "59684", "46298", "22672", "23538", "25704", "24785", "53902", "19675", "25597", "37546", "57987", "47473", "26078", "813", "42486", "5943", "53752", "10024", "4394", "35874", "57609", "46822", "53613", "43078", "16450", "12654", "50672", "15679", "17384", "40530", "54785", "45000", "59850", "1891", "38924", "31109", "36562", "50250", "27435", "41612", "23525", "12349", "53885", "25309", "13407", "5351", "22199", "15742", "43056", "55033", "5344", "24706", "55708", "39087", "3684", "17911", "4723", "5255", "12270", "32526", "37659", "50088", "38409", "55381", "23076", "45055", "34250", "51807", "26016", "37897", "8109", "20396", "442", "21923", "52169", "52682", "45966", "10696", "31958", "27783", "10002", "37778", "25276", "9774", "22728", "36932", "7498", "25930", "8493", "12703", "48934", "43591", "18322", "15479", "33882", "54522", "9560", "42595", "9318", "50288", "52091", "53626", "37091", "45773", "52924", "35237", "56017", "652", "10264", "45086", "2644", "3230", "16827", "43173", "37655", "34281", "20951", "3433", "45771", "2890", "43370", "39510", "46419", "618", "3292", "13606", "16924", "49733", "8352", "25300", "50243", "37461", "9088", "21322", "58552", "52792", "828", "59758", "15098", "23325", "29283", "52260", "12123", "26723", "24284", "6916", "15759", "55392", "29726", "42596", "16095", "3864", "1514", "5232", "10454", "34460", "4494", "12894", "40544", "21422", "34686", "41267", "16941", "7634", "1765", "41065", "30745", "55647", "1093", "26972", "1444", "28764", "54889", "21294", "49866", "20440", "4150", "53134", "4608", "32002", "59433", "14573", "52936", "59870", "30997", "53353", "44549", "34659", "85", "52583", "14132", "25774", "11980", "35381", "46576", "38723", "15100", "13252", "14361", "4957", "620", "17166", "27598", "34257", "9608", "3154", "9659", "57807", "43121", "53840", "46551", "44406", "42450", "37314", "56380", "14987", "52247", "1297", "39330", "4251", "52465", "19856", "13590", "19325", "24412", "21970", "15662", "27419", "5092", "33264", "47780", "40776", "21864", "18625", "34009", "10919", "432", "32355", "20455", "23971", "25349", "29629", "56902", "27392", "12864", "12532", "43934", "24867", "53106", "51188", "17804", "20307", "55302", "59770", "29145", "51945", "37710", "15128", "18542", "57602", "52734", "33866", "55588", "2548", "44070", "33355", "7838", "11526", "36693", "5857", "37855", "55808", "26404", "35782", "22901", "57097", "2346", "45265", "59237", "41832", "32906", "17366", "15329", "57236", "3136", "45630", "24518", "50089", "55678", "50410", "20777", "40024", "1324", "32548", "55409", "33422", "46171", "50631", "42351", "13918", "16286", "40982", "18740", "45176", "52478", "44195", "19729", "34563", "14653", "57156", "54333", "24612", "49010", "43406", "13210", "23122", "32120", "21043", "11553", "8865", "28576", "39979", "29527", "47182", "44862", "54863", "1761", "50949", "16075", "5131", "52235", "9187", "44231", "46843", "51273", "43192", "30943", "2623", "21885", "8594", "24729", "14554", "58908", "1931", "7698", "39474", "25135", "35409", "3399", "567", "35545", "55571", "30065", "18036", "2891", "57197", "45370", "58320", "23621", "32841", "51129", "31771", "44614", "23752", "57042", "56655", "10344", "31266", "57610", "14768", "54131", "46680", "8773", "8250", "27548", "7021", "18494", "50873", "537", "26996", "14924", "1818", "54022", "9455", "22942", "28386", "59674", "48668", "26588", "48034", "2690", "1999", "20057", "38749", "10153", "54027", "47327", "38686", "42255", "37043", "19976", "2242", "13661", "48928", "11882", "431", "57128", "50225", "49570", "45145", "37750", "21147", "26908", "29852", "33199", "46933", "52973", "19261", "58536", "25279", "29928", "2392", "18340", "2192", "39295", "10128", "13035", "22001", "54690", "41794", "23676", "54888", "30834", "27222", "52656", "26874", "23931", "4279", "31401", "22095", "44892", "21785", "39578", "13121", "22976", "45846", "56842", "6027", "14517", "38053", "17294", "196", "36315", "42991", "37731", "34247", "55", "18798", "14232", "47002", "11776", "7792", "42693", "38491", "43677", "34043", "17920", "7256", "46101", "24840", "19303", "57854", "42993", "19129", "13043", "25536", "54450", "59285", "7722", "726", "26166", "32722", "28759", "14627", "38795", "21037", "27962", "56578", "33897", "12151", "40067", "23112", "29985", "34052", "57153", "1756", "42510", "11556", "17617", "24311", "17628", "30467", "19506", "28712", "34858", "40694", "53465", "6166", "49180", "38138", "59542", "40442", "37491", "42943", "46468", "53956", "6617", "48552", "13628", "54242", "7489", "44309", "9545", "7590", "37517", "53114", "2996", "26264", "23633", "41843", "9350", "41795", "49527", "47319", "55127", "39129", "7452", "3277", "3715", "12209", "57966", "47385", "47458", "6679", "50980", "12520", "12656", "2586", "14486", "48375", "57346", "4583", "12357", "37262", "15439", "47681", "35203", "132", "1823", "42579", "58722", "15261", "46743", "16110", "4586", "50132", "48191", "46865", "16452", "5658", "28614", "51034", "49811", "35572", "12031", "36517", "33743", "5179", "10853", "43468", "19448", "23425", "3090", "3846", "47339", "10874", "9823", "41514", "43227", "24788", "54788", "48923", "26132", "16982", "59790", "13099", "35195", "36656", "9401", "46443", "41059", "12157", "40829", "48986", "47336", "12121", "34508", "38705", "54001", "30472", "38173", "14506", "57909", "22996", "13763", "12428", "36491", "37140", "42207", "35400", "43166", "29694", "59202", "28908", "36467", "47951", "13576", "7496", "46500", "35867", "31867", "2338", "57478", "29899", "12059", "54142", "58208", "48690", "22142", "47131", "44594", "54183", "39463", "17787", "59254", "41393", "8641", "55315", "13904", "35144", "50612", "55423", "5059", "44371", "27654", "18668", "48684", "29355", "54504", "45847", "3236", "32459", "50015", "3352", "14797", "30177", "54552", "3333", "13826", "32124", "26471", "14406", "22710", "55095", "37705", "40634", "46441", "22670", "38643", "42452", "13464", "42300", "51863", "39690", "20824", "42601", "53209", "49732", "6357", "51243", "30776", "47495", "53385", "8386", "44869", "25840", "48325", "57505", "59973", "46577", "41891", "32126", "17172", "44849", "29459", "29602", "25235", "16170", "36462", "45991", "53032", "9598", "12693", "19227", "43824", "271", "16504", "56943", "15134", "14945", "30113", "14897", "9714", "18055", "56933", "12421", "40371", "49837", "14483", "54312", "58665", "10702", "10679", "15505", "59695", "50646", "18455", "30873", "3350", "10961", "33418", "31383", "6993", "53636", "39962", "49208", "467", "16271", "10944", "38619", "23696", "49237", "36484", "49510", "50578", "51523", "7357", "27005", "2420", "5261", "18388", "12895", "26172", "17653", "22008", "45443", "767", "1867", "31262", "56268", "27907", "20946", "53489", "15844", "33421", "23875", "12934", "55605", "43422", "47815", "10904", "14658", "35675", "49541", "2156", "21607", "53770", "28916", "44202", "41187", "21559", "48746", "32261", "33850", "9968", "35083", "45938", "1641", "45149", "37143", "5518", "13405", "41826", "9952", "47465", "26598", "4736", "47452", "7530", "20861", "15487", "8528", "4506", "47080", "51541", "28004", "10501", "3211", "5576", "53262", "32399", "49513", "35437", "41548", "12292", "49038", "4956", "33591", "19128", "39681", "50166", "33829", "15204", "21223", "5571", "6752", "56061", "10422", "3586", "7963", "18572", "22080", "31380", "49512", "24576", "24609", "10280", "23043", "43784", "40317", "20015", "48553", "48591", "8774", "44125", "4851", "374", "49024", "54455", "12154", "45540", "53079", "40046", "42884", "59305", "54516", "54573", "43868", "30064", "50517", "8485", "7672", "4408", "54154", "7267", "56660", "48122", "7334", "45079", "59649", "6409", "9735", "2827", "53155", "1320", "52804", "23066", "53254", "59277", "57150", "48069", "23618", "45465", "18152", "32866", "22108", "7058", "32419", "19480", "21446", "40939", "22712", "56934", "12538", "27603", "10221", "54356", "1172", "48810", "28263", "26252", "40473", "45959", "2801", "57443", "18229", "28757", "9566", "47389", "22564", "705", "52748", "5176", "49906", "49519", "14912", "30797", "50432", "41333", "13979", "7756", "39496", "25752", "10291", "30631", "45286", "29666", "25999", "50902", "22724", "8677", "664", "3548", "14691", "9359", "40543", "27123", "18611", "36960", "15175", "7454", "20470", "58849", "10820", "21940", "50777", "30365", "15484", "8121", "3789", "31162", "35262", "47905", "30940", "16249", "3187", "45969", "27825", "967", "30577", "19959", "42845", "5289", "45031", "41099", "25642", "7229", "20359", "21458", "41570", "1260", "39062", "44295", "25554", "58770", "48300", "58695", "55005", "51425", "28733", "13709", "40423", "18901", "33348", "49653", "42999", "54305", "41922", "20960", "36392", "43018", "47909", "35700", "50396", "40036", "20966", "184", "27984", "51444", "48672", "12858", "4317", "54434", "43311", "53472", "45101", "58745", "13280", "18748", "5392", "48070", "58718", "51713", "15615", "58059", "8984", "52965", "17089", "43992", "14767", "9309", "40129", "38008", "13614", "45857", "7502", "12595", "55949", "56323", "38150", "34780", "14537", "59111", "57037", "49795", "304", "35682", "9108", "59603", "57892", "52096", "11959", "31302", "38670", "10921", "53356", "16444", "13409", "14977", "3135", "7921", "26625", "56886", "10932", "4526", "33212", "40055", "9668", "5888", "2247", "43331", "36271", "13603", "47777", "25379", "6172", "32483", "10909", "3782", "2202", "10305", "34291", "51830", "38207", "6577", "14460", "41812", "30431", "22237", "5366", "39520", "2244", "7192", "26072", "18212", "30328", "37833", "18950", "49457", "38238", "50811", "25145", "52236", "20113", "3832", "19151", "32788", "8084", "57543", "6299", "1539", "5103", "59605", "32932", "58306", "14712", "45474", "28157", "32383", "445", "48933", "56352", "33002", "28835", "5712", "10194", "48354", "30799", "59163", "58243", "29567", "16928", "40505", "28706", "25693", "56493", "57694", "33258", "44007", "1979", "58916", "47661", "2794", "50172", "38439", "10778", "28480", "32989", "18377", "58825", "27863", "18298", "7953", "39899", "21947", "50434", "46832", "21359", "32974", "50871", "4304", "42204", "30162", "29410", "47085", "55286", "15834", "2731", "19164", "23041", "40450", "49701", "47189", "37079", "53967", "46947", "36205", "43993", "25538", "16448", "48558", "10653", "25699", "59083", "8777", "2696", "40405", "47169", "51344", "29934", "47829", "17275", "59432", "9209", "55726", "38744", "54542", "16670", "45706", "50383", "25641", "31974", "719", "29311", "198", "58413", "49621", "45310", "47728", "9709", "38856", "55565", "17646", "24968", "36024", "35758", "44922", "4301", "22513", "23103", "52220", "34131", "7287", "42517", "26158", "47564", "48185", "33825", "42343", "15809", "7293", "28267", "54905", "35034", "890", "46417", "10606", "1449", "32925", "17027", "3956", "4223", "56918", "21243", "30439", "51103", "1386", "30181", "56276", "2359", "45952", "10508", "12098", "25831", "59910", "29481", "48784", "38353", "10419", "18526", "42032", "32206", "38844", "29976", "14282", "7575", "52920", "52222", "44248", "48880", "30423", "9991", "15142", "4078", "22153", "54399", "21146", "32630", "7490", "12790", "20313", "3555", "17201", "7742", "15721", "35114", "30105", "5243", "17704", "7617", "14894", "17217", "40666", "22652", "47395", "34757", "29342", "15539", "46465", "37442", "700", "36427", "38347", "539", "35820", "6227", "44032", "21055", "38778", "12659", "34229", "16866", "30417", "28236", "19439", "4310", "28339", "35914", "17138", "40116", "38450", "1672", "43623", "15005", "42798", "7539", "27955", "20618", "10742", "21517", "18916", "29788", "25722", "54829", "11597", "10418", "27594", "26466", "31673", "3510", "20783", "17361", "5971", "25417", "14043", "7622", "21782", "43504", "46703", "2410", "34307", "40286", "34951", "10057", "21527", "11174", "33275", "57241", "47795", "15799", "56715", "59187", "4214", "47871", "4221", "9381", "34626", "4864", "14588", "34787", "31146", "22156", "10707", "37370", "26052", "4211", "36258", "5875", "28617", "32412", "36285", "8358", "54034", "42836", "13032", "46051", "27042", "48519", "27386", "1522", "21245", "45930", "25797", "6003", "30041", "19381", "46931", "43142", "32503", "43689", "21247", "26824", "9954", "16244", "6094", "31392", "58889", "17644", "26966", "36712", "42763", "45336", "57032", "23968", "48573", "39264", "29218", "18440", "32934", "9042", "9330", "8052", "44154", "53331", "8141", "51645", "17377", "54259", "4589", "43796", "27327", "38754", "1913", "57705", "26515", "44870", "48418", "58757", "37038", "50284", "12783", "56820", "29690", "24598", "8781", "18000", "278", "17133", "4292", "39118", "49389", "54412", "10001", "30053", "26209", "22896", "26581", "55469", "33879", "28055", "48657", "12397", "5427", "48082", "29393", "48195", "41291", "28873", "35904", "37215", "39536", "41655", "50973", "15517", "27428", "43236", "16802", "53213", "40993", "24216", "25853", "58820", "59050", "26405", "21285", "58345", "39030", "18770", "48666", "28346", "31819", "9254", "8864", "53973", "10435", "48717", "4668", "7892", "19511", "56246", "26970", "29663", "35097", "31575", "56658", "3580", "43510", "27193", "1049", "40676", "56862", "3472", "20136", "11853", "34865", "40879", "24300", "30175", "38719", "25329", "40787", "13321", "41074", "46368", "6985", "44938", "28045", "6534", "54555", "30396", "23089", "30466", "37350", "18989", "4326", "9336", "56354", "403", "17518", "13354", "15184", "54799", "1908", "44671", "35474", "33708", "53939", "8444", "42210", "14830", "31550", "19400", "32807", "41030", "36131", "49774", "31339", "41692", "46686", "16097", "22514", "12350", "52381", "42509", "25570", "19299", "55997", "53047", "26851", "52037", "48919", "3432", "54370", "6432", "50681", "51269", "52638", "39602", "53253", "5894", "33049", "46268", "1218", "7228", "16024", "19470", "2184", "57476", "28249", "3459", "31767", "25289", "17789", "2331", "5664", "54390", "22488", "42193", "47548", "52369", "39931", "2512", "11730", "4091", "40134", "39270", "51053", "4424", "39940", "40810", "35916", "297", "7294", "30632", "27248", "7196", "51329", "30643", "23691", "52250", "15257", "19371", "33865", "19142", "24068", "14862", "20611", "5986", "23529", "10126", "20280", "49228", "2529", "29009", "16778", "23843", "41928", "54996", "19313", "6187", "14628", "26778", "46004", "9803", "9025", "33341", "28216", "18276", "38684", "1683", "3842", "22350", "3672", "17250", "10062", "55834", "19070", "52591", "19727", "54347", "17545", "41969", "39221", "47299", "50357", "21964", "11618", "9916", "45460", "5585", "34375", "37410", "38", "14220", "50913", "45515", "43000", "56079", "44503", "21803", "21652", "16986", "32405", "3493", "1753", "38221", "44924", "8934", "57530", "8037", "52711", "43175", "44351", "49599", "2509", "22650", "47676", "15227", "8398", "37835", "21555", "27583", "33245", "10402", "20018", "596", "6820", "18131", "9738", "48180", "2907", "20693", "58950", "32367", "35502", "10164", "53095", "27115", "6030", "32583", "50439", "57834", "21772", "33273", "26974", "20984", "41292", "15147", "41485", "41824", "41439", "54322", "45980", "16592", "56725", "58542", "47767", "4716", "2842", "49898", "50942", "27066", "51756", "20661", "18983", "57500", "47415", "27496", "14364", "8137", "36044", "10486", "162", "34189", "45425", "50668", "14779", "33034", "17249", "54101", "38225", "15815", "28739", "23884", "21224", "2183", "39166", "29247", "19100", "19245", "54297", "8536", "762", "53407", "26504", "20429", "23389", "44425", "13029", "23889", "35236", "49363", "6748", "14347", "28652", "34818", "58587", "16156", "17809", "15188", "2284", "15570", "1947", "8562", "11893", "27604", "48530", "44519", "5088", "15605", "51039", "16315", "44325", "9652", "19399", "1146", "49312", "27456", "36807", "39151", "38968", "27411", "57595", "10779", "1877", "43815", "28747", "35205", "27558", "32502", "54378", "24167", "30218", "1462", "34881", "9931", "41424", "23282", "14787", "17488", "56683", "7645", "39994", "58319", "11137", "47001", "39782", "22638", "6344", "4945", "35631", "50958", "4077", "573", "26551", "7221", "46812", "15367", "5265", "23636", "10483", "56277", "40624", "3844", "44468", "52932", "26768", "20884", "32854", "25653", "43404", "46223", "38046", "677", "29644", "52778", "42253", "1577", "35986", "48947", "51384", "58930", "48608", "59716", "48557", "11443", "47304", "23235", "31747", "37358", "35047", "56890", "13861", "56067", "30877", "40695", "43602", "51240", "12945", "1747", "30798", "18869", "40740", "23270", "57972", "43315", "56650", "54794", "4887", "59775", "12691", "8755", "3386", "35282", "12709", "9128", "47381", "30621", "24221", "47427", "3074", "51125", "7989", "37564", "46390", "20802", "3996", "30895", "14003", "35260", "5724", "44675", "46911", "54689", "29869", "37489", "29993", "49354", "58965", "33853", "14344", "53827", "58473", "51558", "28006", "41946", "18976", "46388", "31049", "281", "4277", "50395", "14319", "39414", "12754", "48559", "4881", "55390", "25301", "49433", "47146", "7278", "38315", "7899", "51832", "15919", "49891", "1373", "16625", "39124", "52051", "29805", "44891", "52471", "58892", "10076", "748", "27965", "19018", "23606", "21312", "20864", "9383", "31117", "6622", "160", "48139", "6717", "12278", "23918", "1841", "32111", "50752", "20413", "45902", "46470", "14141", "34545", "5801", "37268", "31601", "14844", "33383", "10322", "31543", "54069", "11787", "17965", "56040", "55240", "23412", "2422", "55542", "20347", "38325", "10654", "46943", "41524", "25955", "38378", "30313", "54422", "3690", "46556", "56450", "31424", "47142", "39128", "9296", "7315", "613", "44218", "7534", "42485", "42842", "12198", "22295", "50210", "11358", "28113", "21437", "1266", "24680", "33908", "15468", "50328", "36221", "51126", "9202", "26862", "2169", "2598", "13744", "54763", "24341", "20273", "38064", "33118", "47883", "17586", "56258", "247", "3774", "47977", "46277", "14175", "45386", "34846", "44963", "18838", "27745", "40784", "8727", "45130", "10538", "20125", "8296", "41639", "25860", "9658", "20776", "15931", "56893", "37889", "27046", "9486", "10657", "41683", "44863", "13284", "43865", "40622", "30881", "7690", "28272", "36032", "21375", "30260", "18289", "6253", "47711", "10553", "41154", "5240", "10327", "29070", "18478", "33771", "16574", "45485", "51155", "57453", "26987", "57216", "2119", "4666", "52318", "53435", "28879", "14788", "2756", "47786", "45531", "26511", "58340", "43735", "35800", "39483", "29165", "45328", "51720", "46116", "3765", "23245", "16031", "27512", "28806", "22704", "5337", "11573", "24971", "12723", "35427", "6800", "45974", "42669", "30299", "14689", "59375", "10671", "39873", "43267", "58082", "41445", "22304", "34311", "58280", "57310", "3050", "53903", "44984", "44648", "58574", "56755", "27755", "4130", "51596", "54953", "36010", "5162", "51990", "42527", "46996", "12249", "53279", "4631", "1850", "48965", "28197", "43808", "41740", "24646", "59937", "14368", "34717", "44970", "35149", "18648", "5496", "186", "11255", "56199", "18234", "55074", "27674", "58287", "10165", "6613", "6627", "9699", "4457", "27365", "38527", "35696", "54006", "59094", "9056", "10768", "52476", "3477", "57244", "43559", "37285", "8255", "11439", "46041", "18210", "31364", "12782", "38300", "19978", "48805", "54073", "12898", "6749", "28514", "2718", "18464", "41397", "21787", "49440", "25937", "8790", "16361", "18312", "27503", "3047", "36508", "18472", "5230", "14921", "13797", "57431", "3024", "56407", "8767", "27440", "3287", "55405", "38305", "14437", "12077", "51996", "48917", "41041", "46089", "39997", "51043", "1623", "20177", "35799", "24010", "12348", "17754", "24659", "398", "23404", "46028", "21650", "23791", "13926", "3250", "7227", "29949", "31506", "34564", "929", "24860", "3482", "4518", "9071", "37188", "24086", "30748", "57077", "22445", "13045", "1936", "16620", "27089", "50541", "39790", "32867", "43049", "36541", "19868", "37211", "1710", "59734", "57933", "52411", "10747", "54840", "29703", "39144", "7684", "5593", "4690", "20330", "53461", "7002", "25740", "26935", "56575", "44679", "1033", "48851", "13153", "18194", "10583", "19265", "59920", "14694", "42159", "31599", "56111", "57890", "13501", "2810", "34832", "54189", "3719", "40182", "8303", "55104", "11346", "4467", "26678", "40984", "6439", "38431", "28352", "4605", "28858", "39303", "25899", "33450", "11347", "46311", "59148", "39038", "27176", "24727", "54570", "59185", "22245", "33833", "29721", "13894", "49475", "13004", "58643", "382", "38591", "21434", "28902", "57229", "18363", "17462", "4435", "51766", "2882", "36314", "56049", "11407", "55135", "54268", "45472", "16805", "58937", "7026", "32273", "56134", "35367", "27753", "44341", "13660", "56405", "54704", "59211", "2959", "39375", "48909", "13920", "59033", "28149", "16295", "28049", "17450", "17477", "55366", "2740", "28283", "18821", "55403", "34986", "56576", "54982", "37119", "33978", "52338", "52353", "17995", "31963", "47132", "24627", "7209", "43997", "57655", "34769", "8804", "1990", "44569", "30800", "1866", "56527", "33751", "57640", "52504", "7345", "9635", "1311", "46606", "14853", "37850", "46877", "17824", "47599", "8802", "24222", "2792", "8889", "50804", "27027", "9929", "36745", "36779", "33373", "10515", "52165", "58080", "52014", "7703", "11176", "18830", "47177", "52688", "29257", "21625", "23329", "19534", "54403", "21097", "26700", "50134", "53558", "44129", "3933", "38153", "12872", "3173", "32884", "30986", "8910", "21481", "54429", "5629", "12238", "49384", "587", "50820", "35936", "46084", "58060", "4103", "50728", "28655", "8587", "27894", "19285", "33071", "21859", "28732", "53533", "19363", "48826", "16634", "13410", "3112", "20918", "36585", "26275", "21996", "53290", "59329", "55972", "34923", "41735", "44159", "51738", "26900", "34526", "49576", "55744", "42843", "25366", "55634", "25521", "28114", "47990", "13286", "11028", "44315", "5955", "23553", "9749", "23144", "18266", "2232", "4166", "6307", "51667", "59264", "43647", "39610", "53857", "3945", "56502", "13278", "33649", "18128", "43268", "29426", "43825", "58852", "13486", "32324", "11876", "33969", "47373", "11324", "4455", "24459", "26960", "12511", "640", "13674", "7973", "58434", "43161", "22259", "53437", "23678", "29140", "59524", "42530", "33317", "28586", "23726", "27460", "9049", "2036", "31509", "10536", "41823", "54629", "33305", "24011", "48771", "57383", "39716", "6013", "22738", "42086", "45187", "23540", "11834", "29072", "45821", "53019", "48906", "57633", "33216", "27881", "14643", "13205", "37941", "6144", "10673", "3458", "23984", "33153", "44361", "33499", "47462", "26217", "24764", "36358", "20017", "34135", "29705", "28338", "6404", "56300", "10046", "15198", "23086", "34641", "24385", "58524", "10397", "2998", "29159", "23190", "6249", "19318", "6899", "59255", "34336", "29282", "28357", "12016", "36559", "39469", "22266", "11724", "34620", "27608", "54413", "4760", "26858", "45984", "50220", "11975", "21484", "28945", "16118", "25591", "3530", "22614", "18499", "44945", "31394", "44148", "56608", "32336", "19649", "18057", "40034", "6527", "56793", "45986", "45344", "2581", "5246", "53339", "36206", "41108", "9944", "52153", "50475", "36714", "37074", "6422", "9680", "9729", "3248", "56663", "15651", "21651", "48677", "13258", "48838", "51087", "9646", "57448", "47583", "59861", "58266", "29970", "43038", "11891", "36207", "39794", "17983", "13495", "13577", "24749", "40529", "32270", "44200", "12464", "19452", "51204", "54944", "4899", "30536", "38219", "13596", "11237", "20081", "50052", "27086", "17991", "38938", "1272", "41731", "52304", "48628", "16537", "48006", "13117", "41200", "37891", "30558", "58404", "21331", "14679", "36394", "34915", "31041", "17904", "58549", "23639", "34121", "58796", "24277", "26729", "53477", "30554", "47224", "19329", "18719", "53359", "57479", "44696", "3767", "14435", "54848", "51836", "57278", "14792", "59556", "49490", "25728", "38344", "40728", "47642", "373", "51569", "30043", "33030", "47727", "38965", "49277", "28400", "47274", "15744", "38624", "10611", "16814", "16377", "41508", "37137", "24457", "55602", "19708", "20227", "23526", "14386", "8515", "41266", "12921", "20726", "22105", "36334", "31817", "21921", "34682", "7719", "3655", "35280", "38424", "11499", "26338", "42971", "32509", "35074", "45256", "30711", "18098", "55944", "30502", "20422", "568", "46546", "35238", "44540", "5952", "38200", "50824", "57829", "2637", "42430", "9488", "58368", "38565", "35156", "11731", "30718", "12633", "52755", "46044", "10371", "16222", "53442", "56636", "53101", "24520", "40637", "11218", "31313", "2734", "53606", "49150", "14875", "28460", "34550", "44621", "21725", "26308", "53772", "28847", "12529", "43356", "49514", "25362", "58192", "45695", "50003", "41602", "51862", "24108", "8171", "24911", "40236", "44180", "30564", "32381", "38542", "31023", "6847", "27867", "44920", "57650", "54480", "11109", "44658", "34438", "31806", "14586", "49387", "22561", "20688", "50737", "51211", "40566", "2364", "47456", "35634", "941", "34086", "4218", "33191", "14541", "32627", "21968", "7639", "13754", "28377", "49013", "52862", "24751", "59638", "13001", "6192", "11279", "24145", "14624", "59501", "29990", "35822", "35471", "39498", "49909", "31722", "31650", "25992", "57689", "3942", "26258", "2434", "3200", "57757", "26259", "49114", "51584", "25506", "31531", "46175", "53908", "49189", "23560", "15203", "34483", "23735", "36513", "58793", "6904", "10249", "41580", "8365", "52874", "27721", "4709", "59798", "47945", "45131", "15356", "42916", "3009", "40976", "54444", "57992", "44011", "59515", "30449", "52188", "18108", "10171", "55345", "37035", "41489", "40931", "16327", "55747", "35685", "5210", "34445", "6639", "10847", "38045", "54815", "29761", "44013", "43243", "49540", "36008", "22050", "49655", "43515", "50473", "54926", "34689", "31811", "16967", "44982", "53759", "10678", "44563", "58016", "36860", "37295", "57695", "54123", "12414", "48829", "42084", "44828", "12361", "2022", "10716", "23660", "54217", "26384", "24332", "36861", "34295", "12493", "46781", "42314", "28880", "16783", "19847", "4247", "8576", "16166", "9065", "42325", "40203", "59996", "13976", "43895", "29942", "8291", "4977", "12317", "14114", "14495", "42899", "5441", "8395", "58406", "15149", "45193", "45586", "57750", "25756", "17026", "50424", "33346", "37644", "21997", "35611", "20035", "45337", "38189", "13623", "13805", "9819", "10063", "42381", "44475", "24847", "9662", "44827", "15167", "27528", "13741", "45335", "55811", "22966", "40659", "59854", "37290", "40172", "34503", "47765", "45462", "6529", "59052", "21071", "33117", "14184", "24470", "58847", "33692", "44186", "21734", "37537", "28881", "57132", "4825", "3762", "17867", "23963", "14247", "11793", "26952", "45544", "53273", "40402", "4002", "53889", "56158", "52575", "35099", "21417", "50633", "11502", "12457", "49254", "34916", "36694", "10683", "32114", "45527", "17373", "43745", "11490", "43841", "52987", "27876", "52695", "9593", "44262", "54254", "57243", "31252", "59040", "9627", "15959", "1064", "17583", "15168", "1540", "41372", "27813", "48540", "49142", "52423", "1431", "7607", "45263", "28885", "42312", "23006", "44092", "34885", "51979", "4921", "47904", "39443", "13279", "48831", "20585", "25826", "14513", "3275", "5463", "34733", "13988", "1532", "46423", "38119", "49485", "1665", "53909", "9008", "26411", "17863", "26879", "52466", "35042", "46770", "21721", "53514", "59794", "2436", "39986", "41753", "8133", "42271", "5371", "28560", "14093", "38506", "6229", "39468", "9923", "35497", "59481", "21319", "8251", "30598", "33357", "15614", "18339", "3353", "41509", "37933", "2141", "1670", "29603", "48112", "18247", "5422", "31622", "18881", "17230", "36829", "55626", "6673", "50470", "37230", "29157", "56011", "18533", "31604", "39825", "33394", "16353", "20214", "57200", "5678", "7163", "37604", "12637", "59740", "42871", "39382", "40065", "42753", "548", "43509", "53832", "19536", "2117", "52474", "5853", "37101", "23121", "26682", "32603", "50208", "21158", "52072", "12545", "15560", "10924", "32013", "28776", "3120", "12567", "13600", "26374", "49826", "18080", "11068", "7978", "58369", "24308", "10954", "50173", "48155", "49376", "36560", "59645", "23627", "1992", "43534", "25044", "22569", "55862", "32638", "21366", "49050", "43100", "28085", "41247", "37403", "38080", "54946", "32774", "30355", "32187", "56371", "37220", "45637", "33790", "40851", "44985", "51416", "309", "41959", "58420", "16505", "49991", "55340", "39967", "4204", "16553", "8688", "35493", "9632", "7389", "9085", "22453", "3647", "10151", "55884", "51150", "22536", "50139", "40687", "25314", "32487", "4738", "22616", "21564", "22373", "46136", "56570", "32421", "56455", "40943", "58412", "50704", "44559", "34238", "59634", "48304", "33201", "42982", "43476", "32572", "26636", "47044", "10930", "51875", "21637", "9386", "52216", "44953", "16931", "53977", "23574", "11652", "17714", "5109", "41573", "46337", "10983", "917", "25569", "22805", "53401", "52058", "35553", "36940", "7428", "37056", "14660", "28871", "27910", "39758", "46917", "27328", "36361", "39836", "51939", "39502", "47960", "29179", "42276", "25830", "42257", "7016", "18877", "35826", "22463", "21601", "5196", "49322", "47255", "14544", "6036", "46429", "51633", "2545", "38110", "20880", "12648", "43847", "53317", "658", "45046", "49000", "29343", "47186", "26313", "30599", "47376", "14087", "14301", "23937", "13567", "3443", "33846", "23844", "33406", "29759", "58902", "55587", "20782", "54104", "44232", "39262", "40141", "37943", "7133", "46835", "16887", "35714", "39879", "57296", "30104", "40599", "14699", "6481", "35525", "16125", "43968", "23855", "26403", "9090", "2335", "29408", "17410", "6396", "24137", "17916", "17813", "29034", "20892", "33474", "50262", "23986", "25205", "57644", "43713", "57048", "56421", "20925", "49724", "22402", "13302", "5747", "36503", "10012", "57570", "11860", "20392", "48884", "46714", "2896", "44660", "26684", "3215", "17030", "9523", "28636", "4898", "14190", "7000", "25845", "43322", "13068", "5006", "50506", "17010", "36867", "21658", "17279", "42921", "2676", "47341", "33871", "41940", "38273", "50866", "39015", "53452", "58436", "893", "23827", "27939", "31046", "56899", "6835", "39564", "41797", "18540", "38411", "18121", "44255", "10876", "18223", "3955", "38935", "6133", "30023", "13186", "36972", "38916", "11214", "57596", "1504", "4145", "49463", "11201", "679", "52928", "35173", "57520", "37037", "21266", "17797", "17459", "4319", "18165", "56727", "25023", "18269", "14118", "319", "22429", "33108", "52006", "33475", "17447", "16362", "44472", "656", "11074", "35779", "4966", "34405", "3356", "14074", "28277", "59480", "35733", "34308", "2267", "51959", "56149", "31695", "38383", "31825", "27798", "28275", "24856", "52391", "38251", "55462", "40517", "18647", "58891", "50496", "47314", "32778", "23640", "5682", "35694", "32688", "37298", "48066", "55017", "52915", "24209", "51270", "3076", "19085", "20122", "51137", "26127", "49564", "20980", "30965", "31721", "55752", "13106", "5191", "7798", "41804", "49894", "49972", "54070", "49692", "29940", "59450", "37715", "56060", "10121", "4678", "20462", "33408", "42566", "28321", "40916", "43400", "53065", "41225", "23985", "28370", "38848", "515", "4875", "28139", "10735", "21325", "10232", "26498", "28792", "41977", "19161", "57371", "52950", "1482", "28722", "32014", "26379", "26479", "2707", "43821", "38952", "2627", "47065", "24502", "27114", "59763", "40986", "47226", "23909", "8913", "28244", "9506", "59425", "11547", "33911", "45134", "38621", "13468", "40376", "54026", "34077", "47840", "59552", "22925", "49108", "39969", "52291", "42127", "7919", "4029", "36993", "37573", "44210", "49200", "23406", "4741", "59505", "20746", "58180", "42897", "36865", "43069", "25515", "24345", "30298", "18043", "12454", "3451", "2599", "49908", "14259", "41911", "53642", "17061", "37838", "45059", "59323", "51455", "33958", "53616", "18615", "56211", "53524", "50321", "13565", "13378", "36017", "31276", "57917", "18537", "57108", "14318", "32352", "31724", "18277", "58879", "6903", "52508", "30079", "39832", "10065", "9856", "49383", "57452", "52903", "29800", "9840", "13044", "50285", "47605", "8839", "44139", "18728", "47337", "20012", "55500", "31950", "44090", "7029", "27410", "21441", "8948", "231", "2929", "21178", "41560", "12216", "48886", "47787", "26106", "28116", "55310", "16689", "50241", "57790", "3823", "14464", "5389", "19138", "35112", "55321", "17355", "51952", "58355", "11800", "45240", "11224", "23188", "18906", "14946", "46262", "12308", "35001", "23257", "1846", "55466", "391", "15642", "41436", "38898", "19185", "12683", "43126", "29851", "24667", "33427", "36662", "46286", "20126", "56927", "1995", "33184", "37634", "56303", "41670", "17144", "40911", "59779", "54908", "28024", "333", "41675", "44196", "37530", "53379", "15151", "31299", "29941", "20695", "46125", "40990", "42149", "8950", "21955", "49970", "42457", "4437", "9578", "59906", "33933", "39929", "29253", "14929", "56373", "31166", "529", "2785", "3705", "52941", "46719", "27113", "8256", "24302", "32639", "17503", "12599", "45459", "51655", "9927", "30860", "46475", "59640", "35588", "58601", "8559", "30485", "37700", "835", "41002", "43683", "10331", "29577", "44107", "17675", "28862", "3643", "31651", "44517", "25474", "809", "9682", "47625", "19501", "47725", "12324", "26797", "55711", "15511", "44677", "14350", "19912", "15903", "44669", "4010", "20354", "4359", "26524", "42463", "23679", "30886", "30705", "49033", "47200", "57473", "54184", "557", "40316", "3840", "55568", "45869", "2869", "38143", "20927", "47285", "41446", "42322", "41169", "58391", "25416", "53701", "46172", "23407", "10009", "4011", "23682", "3156", "15491", "38211", "35744", "57380", "20437", "12122", "44708", "58079", "35388", "5318", "2555", "14749", "53800", "50279", "26503", "23011", "8743", "57911", "43918", "35857", "40261", "8722", "44759", "57814", "16754", "13489", "20619", "35895", "51264", "15708", "28079", "28970", "46871", "18754", "9864", "34567", "860", "50754", "8966", "10473", "49166", "2347", "55707", "6924", "40165", "42741", "39506", "55026", "58894", "4785", "46421", "32567", "35771", "41462", "1085", "11674", "12702", "48721", "48476", "25064", "54198", "33441", "48287", "20403", "59834", "44235", "36485", "29882", "19524", "11136", "3029", "59296", "3839", "54680", "37966", "47910", "19556", "56495", "55415", "9054", "27384", "37801", "6195", "24699", "25014", "45413", "42828", "12543", "28929", "34243", "29231", "29195", "37574", "54510", "53195", "36468", "23402", "29110", "44683", "33040", "50063", "45085", "46491", "30323", "46157", "26327", "47273", "817", "4965", "41779", "13800", "43465", "51131", "21635", "19733", "980", "22549", "59069", "10625", "37458", "35896", "26708", "16770", "8393", "25583", "54493", "58651", "28982", "57873", "22530", "17864", "25011", "48132", "17473", "7843", "26351", "55249", "39946", "31732", "56225", "33087", "6390", "26986", "49011", "25518", "37597", "34358", "7051", "55912", "59262", "2038", "38697", "23070", "38781", "9162", "17600", "5922", "48316", "26761", "29384", "22422", "6860", "5039", "15146", "29239", "19971", "34023", "36558", "29028", "53315", "15072", "18342", "21799", "31853", "58215", "19631", "53934", "25412", "39181", "14392", "27370", "15621", "23373", "21849", "38384", "31474", "6839", "34504", "58508", "29767", "11368", "41045", "54208", "42499", "3488", "1397", "27003", "59319", "30662", "22944", "49133", "30835", "41", "42564", "6972", "3962", "18064", "5783", "1045", "41016", "55439", "45867", "26930", "32275", "23160", "27734", "42122", "34486", "5792", "59199", "27705", "20349", "41314", "25530", "29044", "28394", "43377", "26915", "22509", "55205", "4745", "12932", "26668", "23738", "14042", "20268", "54596", "28061", "20647", "41405", "40919", "56869", "41043", "55148", "4591", "46049", "1270", "27442", "2115", "44515", "50653", "52211", "38258", "37844", "21295", "51566", "2764", "43469", "48522", "26995", "49802", "53536", "11779", "8705", "23753", "4427", "53485", "20969", "13712", "37448", "4657", "43274", "11900", "32796", "49337", "48239", "4971", "20350", "47160", "59158", "36853", "34642", "56313", "22953", "12284", "58238", "21747", "29596", "24523", "21542", "956", "27296", "40627", "53583", "44518", "8196", "13635", "6266", "2300", "28507", "20252", "53478", "44008", "24417", "52139", "13818", "37026", "7096", "1478", "36570", "12371", "37703", "41081", "59837", "10770", "926", "10837", "46005", "57266", "32359", "58620", "36951", "43539", "44225", "9533", "4973", "20732", "1217", "44597", "14835", "12408", "33671", "58182", "2130", "34114", "34611", "53142", "6785", "15267", "44477", "20669", "49056", "10572", "2465", "32106", "48021", "17735", "6024", "1840", "39154", "11507", "40762", "6547", "40112", "31135", "31991", "30794", "39791", "39754", "32500", "24467", "35212", "23856", "20659", "25546", "55806", "48739", "47864", "49593", "55013", "12369", "21780", "33554", "29306", "24324", "45312", "42148", "52111", "22573", "20916", "34205", "54508", "26366", "25337", "33830", "55146", "3547", "13216", "52432", "41329", "36424", "8894", "14372", "23936", "11748", "1787", "47496", "49914", "19377", "6824", "7422", "16598", "26535", "58241", "23183", "5208", "54134", "29564", "5023", "49845", "6995", "13895", "17531", "4167", "33616", "5", "58838", "8623", "39833", "18922", "16012", "27616", "48568", "49385", "30757", "42491", "42704", "28547", "34156", "6858", "50894", "16314", "11336", "22708", "14079", "34122", "51237", "31977", "21699", "8843", "9941", "17959", "42425", "10457", "51443", "55923", "1140", "57781", "19033", "36547", "51695", "18497", "41212", "19410", "33737", "12967", "41301", "5260", "35495", "48003", "53476", "21170", "42421", "30358", "241", "144", "22793", "17551", "31129", "8653", "51221", "37443", "16875", "42056", "42243", "19812", "26247", "54912", "10475", "52492", "24636", "9142", "19665", "41418", "21643", "26444", "36353", "1984", "2694", "43271", "22254", "35591", "955", "44301", "42608", "47659", "28064", "56856", "20424", "14721", "55579", "29097", "24369", "34894", "24803", "29964", "24314", "14408", "13449", "30456", "31126", "35776", "8253", "53409", "28838", "56676", "11383", "23346", "44046", "9470", "29498", "28432", "36497", "28345", "12714", "42296", "39924", "6607", "3261", "26541", "2122", "46165", "11098", "4693", "22307", "51459", "54416", "30646", "12939", "16224", "7896", "16952", "42041", "33656", "29289", "37251", "24205", "37923", "9942", "3291", "37216", "27390", "53222", "42622", "22253", "42168", "28193", "26527", "43754", "19495", "25157", "35510", "27219", "34863", "46858", "32168", "2031", "56160", "15997", "48241", "28222", "49545", "59294", "44579", "3117", "23915", "22696", "32319", "43306", "58281", "12012", "48819", "47846", "21235", "15233", "55538", "41430", "33769", "17968", "51024", "51054", "34208", "30265", "59835", "39533", "58743", "48016", "19807", "54959", "52560", "32709", "40318", "23411", "10895", "1759", "19671", "23155", "14668", "2099", "45726", "16043", "33397", "36728", "33706", "8487", "29118", "43965", "2105", "3553", "32151", "42441", "33489", "17906", "49715", "27301", "28771", "15290", "54671", "6516", "5572", "13081", "6658", "26561", "27657", "38694", "29554", "2625", "57944", "35535", "49071", "22333", "13892", "2879", "54545", "50722", "265", "59209", "43576", "16281", "2014", "41555", "56227", "31293", "5619", "3610", "40162", "37572", "8314", "20996", "33624", "51601", "30982", "40645", "1488", "25760", "27768", "14545", "50887", "51491", "6649", "9236", "601", "470", "37800", "31590", "26425", "9955", "24718", "23910", "28876", "14818", "25746", "15143", "51094", "54335", "51705", "14575", "55494", "26932", "47832", "30980", "55682", "31095", "2306", "41450", "42906", "25260", "31429", "39168", "53705", "4883", "54894", "4812", "30438", "7787", "17091", "31106", "12628", "24778", "47032", "58723", "56597", "38923", "1599", "19155", "25735", "35709", "51703", "17946", "1442", "27592", "13172", "42557", "32576", "20556", "38032", "14140", "39984", "41086", "4894", "58510", "44059", "43648", "19206", "6400", "12833", "9007", "7728", "38967", "28969", "5795", "15586", "38397", "52633", "59251", "14370", "40546", "50538", "17988", "16719", "20827", "30525", "34323", "37932", "51851", "5062", "58339", "31949", "9977", "5714", "49325", "50154", "24555", "43719", "27538", "34749", "1581", "47812", "37529", "25758", "11401", "29837", "58442", "7869", "36919", "49153", "34333", "16960", "34380", "10762", "7184", "45874", "43275", "3043", "2654", "8575", "40154", "33511", "15675", "54795", "40804", "7326", "14228", "36246", "52854", "9623", "47656", "12826", "36352", "18041", "44233", "4203", "30612", "35913", "50001", "34317", "8693", "10563", "48925", "6121", "40799", "21455", "793", "7045", "13675", "38236", "45917", "50237", "31517", "12938", "44670", "11541", "51822", "40232", "17882", "56066", "39189", "20509", "13333", "5262", "45829", "258", "59295", "26718", "26714", "2290", "42616", "34280", "49876", "20067", "45103", "22906", "59092", "29354", "38881", "52261", "37099", "22131", "42288", "59560", "38171", "3568", "29935", "15145", "8206", "53566", "44976", "21839", "27404", "52555", "53139", "36582", "53713", "2223", "24548", "37098", "28083", "58789", "15432", "50259", "56950", "26724", "8719", "34593", "42294", "31088", "37032", "43933", "18695", "23243", "26089", "57709", "48399", "47219", "43244", "47975", "41159", "29713", "23873", "46111", "16083", "7248", "37405", "48072", "11777", "24588", "47129", "27354", "21479", "37742", "10567", "13518", "35240", "44463", "33765", "50200", "51983", "21779", "40447", "9708", "50097", "55875", "22323", "33119", "1096", "31287", "55616", "32423", "36657", "55065", "4711", "10936", "58429", "969", "13255", "56614", "40675", "38581", "7250", "36143", "26478", "26649", "45693", "55396", "26452", "48634", "47229", "36796", "26278", "33909", "35105", "13056", "18154", "52065", "24755", "55347", "11949", "2152", "57767", "37142", "55655", "7180", "11989", "43263", "4702", "12974", "25423", "49783", "37288", "36153", "52193", "56159", "46583", "14702", "17158", "7280", "17802", "107", "26849", "9624", "38646", "17728", "56633", "24567", "17348", "14741", "13256", "11124", "52898", "21418", "1798", "44356", "11634", "47342", "26652", "23587", "55506", "43358", "44537", "35129", "43628", "54664", "4037", "36171", "9691", "12266", "3146", "11191", "56090", "18004", "12937", "48250", "17351", "55185", "24783", "30484", "50415", "42513", "13130", "27724", "11986", "36388", "47618", "24550", "18502", "16641", "22887", "20899", "40155", "59446", "37424", "53206", "16435", "50067", "54870", "28899", "39761", "22903", "4470", "25576", "48745", "33300", "59416", "31140", "1982", "56624", "2263", "17764", "31464", "25828", "24510", "38549", "27346", "13465", "49600", "32311", "3620", "36500", "45479", "9374", "8832", "16614", "54556", "9450", "56023", "47178", "29507", "50421", "16734", "47544", "10587", "58484", "21688", "27821", "47328", "13834", "49790", "28112", "28487", "14068", "15669", "51414", "54257", "20511", "22576", "32310", "17640", "34775", "19500", "35133", "17402", "31668", "39274", "43568", "51610", "17623", "37053", "1316", "58301", "11620", "22560", "43104", "27429", "54641", "32840", "37864", "14923", "33544", "36851", "1278", "32103", "52604", "10336", "36996", "10786", "28288", "1223", "10213", "46187", "26035", "9183", "10463", "43254", "26472", "56482", "56922", "30996", "25871", "17760", "12940", "38363", "49141", "22909", "2710", "13080", "24916", "17280", "56879", "3425", "51012", "24837", "36307", "30717", "12126", "3265", "50238", "5066", "5110", "21986", "38915", "22502", "32852", "11049", "10362", "33924", "16434", "58037", "50861", "52554", "50717", "24339", "14046", "3055", "7783", "31964", "28996", "49085", "53932", "6565", "47805", "38224", "2919", "34209", "53042", "43598", "50161", "10704", "54580", "41886", "3435", "20475", "40819", "34623", "14354", "37531", "3596", "51663", "58158", "48876", "40873", "21110", "31352", "51831", "59167", "39429", "2462", "24406", "47265", "7271", "31396", "16227", "34379", "34652", "5332", "49672", "40224", "50700", "29848", "38365", "48115", "31369", "24138", "44481", "33086", "49107", "44781", "16008", "36255", "34565", "12887", "10116", "7930", "12426", "12854", "21145", "59700", "33560", "36894", "8281", "13091", "3080", "58681", "22400", "42100", "51821", "57564", "48957", "52162", "30931", "7854", "18558", "17382", "25450", "54161", "32342", "47839", "47461", "25343", "59417", "50440", "55348", "47843", "31813", "4100", "2940", "33821", "48201", "50009", "44716", "44118", "25317", "16819", "10032", "22914", "39652", "4974", "50039", "15376", "4990", "59138", "2642", "51374", "50845", "28832", "13476", "43436", "45566", "29356", "48114", "27564", "58575", "35405", "12470", "16947", "24965", "3452", "51928", "47475", "28361", "58798", "55562", "39178", "17671", "53700", "25061", "48727", "19916", "26344", "42414", "51971", "24067", "16968", "38362", "39163", "39978", "51496", "52172", "9059", "37811", "52002", "6937", "53198", "12471", "7823", "55101", "30587", "50670", "58598", "52822", "40422", "38699", "49001", "50823", "19532", "36137", "48612", "26547", "29436", "27099", "38140", "16381", "8481", "30533", "38658", "50153", "22495", "16235", "51398", "53260", "44491", "8753", "20826", "20632", "6159", "42484", "59437", "52060", "3500", "40085", "57899", "44486", "9325", "59241", "44362", "17443", "31414", "12948", "57753", "9232", "20819", "2692", "7385", "52442", "39559", "36385", "14553", "32700", "59781", "24828", "35312", "41985", "34907", "15361", "30499", "19357", "32180", "59256", "7740", "29081", "1508", "49812", "49068", "36342", "59290", "14794", "15768", "19080", "37816", "2342", "46121", "41622", "31111", "11714", "51018", "48629", "11955", "35662", "18292", "42334", "4948", "19334", "33648", "35699", "4196", "38745", "20402", "22891", "56668", "8991", "56620", "28749", "7378", "5317", "21752", "3868", "40388", "6276", "47729", "29765", "1280", "30256", "45561", "36193", "32046", "53462", "3442", "5710", "51318", "54479", "37648", "55723", "23244", "47072", "35995", "58563", "23797", "35467", "1098", "21810", "36269", "2064", "16565", "2293", "947", "42454", "52613", "42781", "38420", "7277", "659", "9520", "46929", "24768", "38116", "8002", "49867", "11655", "44061", "28973", "29465", "51731", "3095", "11326", "6637", "54025", "33807", "24792", "32008", "39786", "32282", "23777", "31547", "55821", "5238", "12838", "4679", "426", "11460", "50437", "15878", "48121", "47414", "56916", "36825", "10867", "17123", "2194", "26958", "19551", "40106", "29124", "33536", "49186", "52389", "41834", "43442", "3371", "24663", "53734", "38537", "28511", "55386", "53375", "43206", "34427", "50355", "35398", "525", "36531", "47106", "11637", "29795", "52801", "25782", "22066", "49902", "36546", "14576", "32770", "32053", "16356", "48655", "11584", "25418", "26961", "18554", "58822", "33714", "48495", "1903", "38432", "17485", "42867", "39183", "36673", "27504", "52956", "32533", "55567", "39070", "23469", "30750", "4371", "12379", "55768", "28166", "40346", "32071", "20279", "28334", "47118", "35571", "49324", "58046", "19235", "41919", "8193", "23205", "31441", "19032", "22183", "59692", "27258", "1239", "19741", "11261", "19636", "27309", "3853", "43443", "9881", "42101", "24614", "33791", "30610", "35950", "26717", "55358", "40466", "26309", "15941", "51708", "43070", "23009", "57219", "11631", "46245", "59799", "42055", "58221", "47842", "30085", "9847", "48044", "10717", "10429", "8127", "52756", "59646", "5118", "28579", "34272", "33998", "76", "24833", "4012", "52455", "29098", "55093", "27807", "53119", "42856", "41845", "44352", "56696", "19704", "5809", "17574", "15519", "38051", "12384", "57059", "29194", "38062", "29107", "45968", "18169", "54525", "36791", "59120", "23477", "44815", "56860", "986", "31961", "33530", "24243", "43023", "22228", "4707", "27920", "58769", "8300", "3372", "57868", "52830", "34529", "43301", "17940", "44209", "8385", "51718", "55596", "21561", "5106", "53219", "47210", "5160", "53268", "16677", "7848", "51338", "3266", "42178", "55515", "48176", "52400", "16174", "19415", "36191", "16824", "58693", "46662", "29678", "311", "31408", "9390", "42638", "8126", "9333", "8928", "13218", "26218", "33090", "12404", "47898", "10497", "29057", "19650", "2129", "43470", "39016", "58102", "37738", "18037", "40689", "11516", "26228", "57928", "2481", "52821", "21972", "8042", "2958", "5823", "44593", "26049", "56685", "2793", "35324", "45742", "44776", "6701", "35062", "7926", "57091", "15353", "15373", "31403", "25371", "14241", "38087", "814", "31706", "14035", "3258", "20296", "16850", "38540", "45802", "43270", "17194", "59774", "55304", "3036", "38223", "9327", "6114", "3293", "1752", "40152", "55448", "40500", "39215", "58031", "11708", "25199", "52602", "49395", "2476", "17290", "26212", "33872", "51754", "17850", "19567", "22714", "41657", "31480", "5124", "17272", "32805", "4284", "35760", "21829", "36303", "22586", "59271", "8321", "16376", "4798", "21038", "3288", "9119", "4673", "28685", "647", "24433", "39225", "28608", "47930", "27313", "51672", "58727", "17820", "59969", "30640", "7467", "13891", "27956", "40311", "53039", "11442", "33485", "30167", "58073", "212", "15363", "36480", "53440", "37786", "2651", "24617", "501", "7814", "16020", "808", "37834", "30143", "59980", "43989", "30568", "51248", "46415", "17223", "53980", "35193", "58314", "485", "15195", "17604", "29701", "24041", "24446", "28795", "57321", "17215", "40502", "37470", "10112", "7589", "37987", "12436", "50101", "38919", "7784", "45245", "58183", "26611", "25390", "18383", "10195", "3852", "44721", "12390", "41547", "27880", "358", "49271", "16455", "33483", "50839", "13485", "44384", "29967", "7513", "34411", "23014", "28138", "18074", "19432", "20772", "12906", "31022", "113", "14424", "29382", "58029", "27397", "56024", "18842", "34293", "38659", "14023", "12476", "52167", "54468", "12614", "57509", "27444", "7912", "46271", "54317", "13616", "29173", "19581", "52150", "26722", "26476", "47574", "43990", "57973", "9433", "46881", "54602", "58095", "42007", "22985", "38522", "15803", "14197", "51803", "58191", "55166", "35794", "42021", "5415", "39384", "38928", "59030", "37741", "55089", "26968", "6282", "20997", "10207", "25126", "24790", "29380", "5032", "23608", "48113", "17999", "38950", "27712", "32130", "53389", "16545", "40056", "27338", "56255", "39934", "37161", "12794", "2673", "17376", "1650", "39948", "45391", "18753", "55237", "54764", "39096", "25036", "20535", "55793", "45440", "42088", "11760", "21297", "43200", "37690", "43314", "23649", "15196", "31358", "22518", "22789", "47762", "19893", "52606", "11524", "40995", "12009", "49729", "18319", "25085", "56847", "4826", "43681", "23375", "53011", "4767", "26967", "3498", "36257", "20866", "40343", "2941", "20342", "35402", "46915", "42887", "37829", "10158", "19664", "51907", "2754", "27205", "7260", "20216", "49700", "55906", "23763", "10888", "38443", "26760", "41216", "21226", "749", "54111", "24363", "39332", "26680", "28944", "51296", "20397", "4686", "39679", "27571", "4646", "44877", "13560", "45415", "16429", "10078", "14756", "54080", "35765", "38435", "27501", "16236", "18566", "51164", "43399", "4918", "25218", "22810", "42723", "58721", "56340", "8011", "6544", "12685", "3227", "48689", "39916", "36963", "2925", "14503", "25476", "17759", "40601", "11244", "16251", "38525", "45529", "59519", "53104", "36675", "35213", "3329", "34676", "18167", "26800", "16241", "56305", "21401", "39170", "45478", "51503", "42040", "18544", "33838", "14300", "35383", "48956", "35031", "45641", "9", "5308", "51898", "36475", "51088", "19466", "21556", "26832", "54551", "41539", "32083", "19238", "28650", "50678", "16699", "10805", "33501", "4910", "18604", "13462", "19061", "24322", "27091", "343", "52737", "48687", "9191", "45870", "54028", "7030", "59796", "18629", "55973", "35787", "55766", "48350", "48656", "10557", "36842", "50511", "49828", "9060", "28476", "22599", "42078", "18638", "46087", "38290", "44846", "44971", "50840", "57860", "45276", "31538", "29839", "46376", "5052", "11440", "11298", "25908", "46724", "33032", "49807", "5923", "10349", "27809", "850", "32380", "39793", "19207", "17305", "3618", "21269", "10821", "43137", "59491", "37132", "23949", "46840", "22493", "53614", "50615", "44421", "2897", "45331", "27215", "27904", "21862", "56", "47527", "30790", "33728", "20605", "40497", "27803", "10184", "2924", "5014", "6147", "33512", "50430", "14582", "15666", "22482", "21314", "35715", "28620", "38861", "42387", "31863", "30689", "30522", "11962", "13108", "53553", "36114", "49523", "9496", "21583", "39180", "11690", "45012", "17092", "42802", "46254", "4601", "22615", "44089", "15225", "32946", "22216", "15025", "10038", "44", "19121", "7218", "22414", "41739", "21714", "15552", "28714", "37036", "31326", "27762", "2212", "46054", "35063", "56652", "18876", "57778", "29619", "35434", "34418", "5166", "6685", "42206", "5286", "45955", "18649", "18303", "13261", "996", "51881", "46323", "14138", "26707", "33661", "27679", "34286", "3968", "11913", "21309", "54424", "14731", "29831", "21047", "12977", "3518", "7906", "22867", "38390", "94", "51497", "40079", "504", "28002", "25191", "36597", "3041", "19635", "28221", "935", "40238", "11529", "13757", "205", "26757", "18248", "2352", "2104", "31187", "24278", "5372", "48638", "5333", "39109", "31188", "33490", "9136", "7432", "37128", "46856", "9320", "51504", "15187", "27685", "33812", "31881", "19078", "43785", "24641", "26043", "29187", "54597", "5409", "27898", "1906", "26369", "17527", "40863", "20426", "23045", "41149", "30928", "19837", "58854", "13144", "21236", "40410", "53736", "20427", "30566", "1087", "34059", "40039", "6297", "49147", "48972", "38766", "28537", "32850", "21806", "26378", "58346", "19559", "38410", "16221", "25743", "59752", "12241", "48659", "59886", "41062", "19200", "19066", "21999", "34989", "3798", "33410", "2220", "32247", "33137", "27595", "21717", "50189", "59619", "13520", "35380", "55692", "9568", "34269", "40469", "16749", "14422", "36983", "23393", "31264", "30020", "31189", "44950", "9807", "31995", "36227", "27073", "49691", "11090", "49364", "38082", "54745", "18735", "25500", "15779", "17776", "18931", "30946", "51336", "33281", "7788", "53220", "37416", "23992", "21119", "28906", "30668", "7311", "9154", "59252", "58953", "47673", "32982", "28414", "17320", "58676", "16903", "6085", "17286", "40983", "4537", "46353", "39003", "20845", "26054", "25785", "12566", "18306", "10248", "44454", "22727", "52003", "1730", "7614", "40030", "58795", "18462", "17818", "40701", "38871", "30223", "13964", "38886", "8245", "1966", "29225", "28192", "8132", "23437", "19616", "49351", "17114", "55438", "1393", "15210", "54369", "34062", "44545", "44612", "26311", "52823", "35665", "40221", "54819", "15199", "48448", "2057", "35601", "14737", "35567", "38375", "53849", "49352", "40481", "46953", "2322", "42183", "5134", "8870", "28379", "54995", "26277", "1215", "43482", "30062", "1070", "41273", "26886", "1675", "59527", "57713", "30296", "31163", "3419", "12741", "9961", "32452", "7118", "36789", "39174", "43976", "58296", "8915", "52385", "651", "57014", "14886", "53227", "277", "36820", "6729", "34414", "28205", "21397", "43885", "9495", "49794", "21543", "39361", "6895", "39320", "7800", "15792", "20692", "24737", "23279", "54604", "26474", "38539", "15542", "18078", "38426", "40390", "57039", "6365", "31893", "27167", "54816", "13229", "17812", "41618", "27617", "30151", "21771", "46316", "44620", "42658", "30754", "49360", "55700", "45353", "32566", "40009", "4185", "30285", "26861", "6941", "31292", "51545", "44087", "19333", "41810", "52866", "11163", "53641", "30011", "37291", "858", "8090", "16766", "42804", "39242", "56709", "379", "57384", "24316", "29327", "14339", "56004", "20021", "31954", "47535", "43536", "54717", "30506", "13968", "11132", "45881", "32455", "34103", "38407", "46805", "55650", "39572", "13555", "55987", "22043", "32749", "44664", "17577", "20432", "2475", "7536", "13771", "23362", "37642", "56974", "19614", "8113", "38583", "9265", "51589", "10067", "47051", "32265", "8586", "37474", "35341", "5980", "37130", "29683", "33088", "5094", "13865", "35606", "51245", "22630", "5126", "40432", "47545", "43547", "54651", "15925", "59538", "13545", "53066", "5353", "3801", "912", "39797", "20003", "51868", "46540", "36660", "32221", "34126", "52990", "5497", "41651", "946", "32681", "283", "15816", "49614", "53080", "10202", "16335", "13776", "54732", "44229", "25958", "55625", "18226", "34313", "12074", "25674", "24392", "4255", "14917", "4478", "42962", "31918", "22825", "52293", "1420", "50946", "39139", "43906", "7088", "36273", "36031", "58501", "50785", "3539", "19521", "4627", "10022", "54252", "30125", "58698", "20218", "50637", "37003", "46952", "2701", "22138", "46140", "22643", "19196", "14877", "52103", "27299", "48839", "43245", "56335", "14734", "4628", "20706", "5444", "30480", "38999", "27980", "46983", "28728", "56205", "24912", "32543", "55351", "1737", "50799", "2979", "31820", "14462", "6944", "16791", "17374", "51967", "19593", "16750", "44165", "50662", "855", "33084", "20492", "24122", "46674", "10069", "245", "33182", "23310", "39207", "16695", "22439", "10307", "25713", "59855", "29303", "23666", "46916", "3142", "37895", "11967", "20450", "9874", "27332", "26284", "29855", "10962", "14743", "45793", "23671", "38802", "4832", "50647", "39251", "1616", "31514", "54365", "4047", "9332", "25026", "46796", "29576", "22242", "11993", "5258", "57031", "33564", "35858", "49039", "17680", "8962", "30413", "783", "56919", "36561", "773", "17725", "48426", "31680", "7652", "33154", "45723", "46984", "35420", "53562", "43892", "20526", "20053", "20846", "1571", "52524", "27635", "29437", "43193", "59009", "37367", "51066", "46699", "37704", "52152", "35331", "17358", "44568", "46610", "10189", "4862", "2845", "54417", "34289", "6327", "52278", "24141", "36376", "51165", "57354", "53991", "7785", "38376", "50606", "22109", "18239", "53551", "646", "11111", "48105", "10447", "31741", "10431", "15009", "52187", "2425", "11932", "24573", "6977", "59858", "185", "55789", "27536", "52367", "6254", "20224", "58896", "33527", "48600", "27814", "32781", "32392", "1586", "52055", "15099", "28830", "10426", "5752", "27343", "23387", "39714", "16981", "38335", "48322", "36553", "5470", "21920", "45213", "47345", "6831", "46736", "19904", "31783", "45575", "36651", "8099", "648", "35443", "22654", "20856", "36054", "54147", "9564", "39539", "46948", "6837", "1783", "48730", "15480", "15337", "58830", "1555", "36934", "21910", "13572", "41132", "16368", "26510", "6281", "2312", "4849", "12140", "56418", "22272", "12264", "55222", "27369", "24466", "35019", "5862", "49437", "14645", "17243", "49567", "945", "56120", "38528", "3159", "2297", "33858", "43887", "44247", "49913", "4732", "6910", "43187", "53022", "35532", "15912", "55230", "27316", "6926", "32287", "59887", "48327", "4764", "779", "47924", "2942", "1596", "16069", "47893", "25881", "7580", "39649", "36846", "28129", "2424", "22683", "21649", "55024", "12208", "58279", "27681", "24415", "51722", "13702", "41658", "6434", "19482", "15907", "43490", "35612", "7516", "5274", "13236", "52925", "46694", "45068", "1525", "44113", "150", "54917", "48550", "42942", "34952", "56717", "4021", "30037", "56951", "36739", "9869", "41479", "50865", "22285", "19079", "40093", "28372", "20823", "23739", "20411", "59101", "39565", "27030", "114", "4748", "12512", "59893", "51101", "13524", "8745", "2706", "30107", "8269", "55593", "41289", "28789", "21600", "21090", "16509", "11868", "43479", "9724", "3209", "7377", "53122", "7130", "6196", "6118", "58177", "41725", "19254", "12063", "57006", "42612", "8174", "34436", "34999", "25034", "13859", "53545", "4495", "35064", "3201", "43410", "25692", "27251", "41117", "1459", "10379", "6885", "36001", "38834", "45657", "31177", "8078", "23507", "2160", "45249", "37019", "28450", "8612", "47632", "18891", "20514", "59819", "46418", "28956", "13345", "52953", "24383", "13516", "29745", "34866", "57282", "37560", "45320", "11159", "16631", "29871", "40748", "5836", "26813", "55717", "47241", "52131", "34595", "45904", "33667", "22774", "59812", "29870", "37501", "15469", "13320", "25553", "40991", "14654", "51413", "34256", "49924", "12773", "24030", "21537", "26193", "46687", "18945", "13220", "16288", "33966", "44649", "25993", "59945", "22697", "57499", "26677", "35854", "53188", "24077", "7878", "39031", "18567", "31975", "27573", "8147", "48151", "44699", "44376", "13071", "30702", "51527", "5696", "5396", "14144", "10698", "59596", "41253", "47931", "29420", "32575", "44698", "53557", "36037", "30627", "20456", "16730", "5965", "5621", "48718", "23181", "14461", "21485", "27969", "43960", "11119", "30473", "30857", "17658", "47884", "37249", "41711", "59555", "31557", "41993", "36616", "47447", "24045", "49246", "21833", "45570", "43196", "7586", "16078", "20713", "31257", "12044", "21148", "3927", "31116", "26215", "55710", "47721", "20176", "35929", "49079", "15940", "6371", "51375", "47645", "11474", "25263", "21169", "34360", "18939", "58076", "56713", "43417", "30294", "5382", "37637", "39827", "57738", "46630", "52331", "51537", "28495", "12647", "11533", "16874", "22597", "17836", "37878", "23225", "48452", "39043", "12705", "54810", "28291", "13542", "9382", "48528", "22558", "51651", "14771", "25700", "33534", "18745", "26432", "55476", "8860", "45359", "9590", "39727", "9112", "50115", "40026", "45884", "58841", "28859", "40319", "6517", "30292", "2992", "47468", "47490", "25047", "5904", "13553", "34680", "31389", "8525", "46940", "28010", "24495", "50507", "2453", "9385", "30500", "33112", "33538", "49261", "49813", "13529", "43582", "19720", "13049", "28684", "42470", "46196", "1550", "35222", "35651", "22020", "44277", "10037", "47678", "24990", "47675", "46672", "50016", "11276", "46647", "50798", "57195", "41910", "24052", "35189", "16648", "13471", "55069", "47307", "30475", "10771", "7469", "6420", "8805", "23632", "28183", "49326", "24630", "13697", "40586", "12480", "56184", "40954", "46341", "7023", "43234", "51358", "34705", "3780", "14900", "48232", "30501", "35096", "41835", "23027", "10897", "36238", "33042", "31221", "13673", "40287", "26492", "4970", "12737", "28210", "698", "34510", "10577", "6543", "32344", "46021", "34026", "54430", "56436", "8608", "13148", "49587", "50945", "10841", "4938", "2135", "51556", "55623", "34368", "33875", "38217", "35433", "42374", "48147", "36366", "25285", "35652", "579", "161", "30364", "5221", "35892", "4001", "45956", "34213", "17847", "11357", "21445", "42309", "41676", "23774", "25632", "42951", "56837", "55369", "23132", "37445", "33468", "7200", "5691", "50361", "3194", "14021", "3896", "41682", "24938", "25141", "53093", "29879", "9077", "33566", "49401", "19465", "56765", "16102", "41259", "20331", "26770", "35307", "21577", "20088", "57254", "58435", "42256", "55674", "36145", "41298", "59709", "37258", "33834", "12409", "25631", "57607", "13972", "5273", "11830", "42186", "53446", "9614", "39378", "54921", "17068", "15620", "23568", "27892", "3351", "40138", "10385", "46133", "19973", "38476", "59462", "41681", "786", "9924", "56242", "30343", "35229", "46844", "19817", "2095", "36680", "17918", "30778", "12263", "22485", "29606", "58977", "25749", "40384", "7413", "10053", "1079", "45885", "13730", "31202", "7873", "32445", "56088", "1076", "55714", "9169", "50329", "8021", "36354", "15176", "1321", "14812", "36589", "45866", "44327", "3086", "17939", "39845", "42847", "28680", "19113", "1692", "6448", "48738", "2527", "59732", "41127", "58655", "33314", "38040", "24900", "7244", "14595", "19010", "57739", "26604", "44442", "53143", "9922", "7157", "48763", "58752", "55147", "46851", "39645", "10929", "17687", "19988", "9337", "53162", "59001", "38734", "12276", "12250", "7910", "30388", "4110", "6125", "55609", "3716", "51639", "42792", "23206", "6089", "31423", "13283", "26842", "28837", "19702", "26287", "38970", "56076", "38131", "22106", "32870", "49935", "4393", "35234", "12002", "468", "58918", "3646", "44635", "14352", "53940", "1914", "23236", "32801", "36755", "29818", "55960", "51793", "7835", "25793", "43068", "59091", "25504", "14988", "30716", "47191", "28531", "34192", "56580", "36911", "11405", "57425", "47895", "53852", "641", "19678", "1659", "30415", "23534", "28286", "17221", "13012", "46564", "9757", "5135", "7663", "58864", "34522", "55442", "36618", "51222", "4354", "12087", "54016", "29158", "18613", "34511", "42348", "27494", "10333", "26071", "38900", "55088", "52802", "50414", "3054", "11744", "28126", "14234", "54485", "35935", "46739", "42794", "31507", "19192", "51045", "1128", "44902", "53767", "37012", "37210", "2415", "51841", "38729", "33252", "24957", "9157", "25271", "4360", "26184", "50963", "1590", "19937", "21381", "1773", "3794", "16147", "33238", "19932", "15973", "14569", "52355", "7439", "32874", "51630", "2083", "11629", "19820", "57164", "36854", "2458", "30810", "58726", "23877", "46833", "50000", "4198", "15241", "25677", "38740", "12220", "45797", "4516", "19481", "38339", "38914", "2902", "32250", "4329", "54533", "47201", "6744", "31435", "10784", "24981", "32568", "50104", "13742", "13149", "58584", "59948", "36106", "30488", "24346", "30235", "26534", "15914", "43870", "24533", "55154", "21131", "2023", "57245", "9122", "9170", "35805", "28433", "44802", "36040", "48950", "6539", "27647", "14120", "12800", "36579", "57684", "9133", "33725", "2714", "34617", "44258", "6190", "52406", "15649", "7316", "33645", "22504", "43361", "51737", "20108", "48008", "37554", "6154", "43610", "20142", "39861", "19945", "29041", "57884", "2508", "34605", "24263", "24429", "50765", "57656", "49047", "50767", "14824", "8917", "9058", "47073", "12468", "1327", "59372", "4705", "8442", "32732", "30394", "34822", "35499", "53706", "28869", "47000", "24929", "12615", "37940", "44116", "34990", "24013", "53622", "34067", "5973", "37977", "52445", "3756", "6383", "23100", "39800", "20172", "37917", "47868", "46007", "26777", "11946", "510", "33174", "2582", "37887", "36910", "4380", "36891", "25732", "44082", "54792", "9429", "16232", "37076", "20078", "30049", "38016", "57823", "19277", "9725", "10703", "15130", "42833", "51777", "19601", "6657", "27547", "511", "15917", "39553", "13113", "57318", "33677", "14246", "14903", "11477", "1100", "27953", "15038", "37927", "57212", "56819", "34314", "15307", "14007", "18032", "2417", "41646", "20272", "14054", "52847", "16482", "18270", "37365", "24640", "31814", "56178", "27434", "54939", "33984", "34383", "19042", "2841", "40592", "38309", "50955", "9279", "22658", "53655", "56870", "29669", "8330", "20431", "9430", "1337", "51909", "7920", "3096", "50195", "25006", "33772", "9046", "6385", "36368", "4891", "12223", "57936", "43757", "17354", "1938", "25491", "39045", "46344", "240", "28509", "19834", "26734", "46377", "14033", "18702", "520", "36915", "39505", "43051", "47165", "27096", "48106", "50711", "55411", "53292", "21205", "24964", "38794", "22734", "43168", "40057", "55245", "42451", "32101", "14915", "21062", "59814", "52201", "35044", "40559", "48920", "11140", "27267", "32649", "17582", "12328", "28562", "51023", "33243", "41942", "19194", "34734", "1820", "40377", "18761", "46717", "829", "35943", "45894", "22176", "23371", "25367", "34820", "42642", "12261", "35706", "45163", "11761", "2467", "4969", "43155", "3509", "52877", "51858", "15299", "20454", "51512", "9550", "38788", "56151", "35464", "11095", "19643", "56059", "23823", "46774", "31880", "59947", "58910", "51215", "45848", "59804", "52573", "57623", "50240", "14010", "31475", "758", "22764", "35408", "57663", "30322", "7288", "31225", "45272", "59974", "27706", "20982", "5129", "17642", "56253", "15242", "39961", "141", "49432", "22247", "42550", "17012", "16188", "3903", "37898", "55334", "50098", "37813", "34694", "30604", "29833", "14839", "3928", "57109", "41257", "50881", "42621", "31204", "15001", "10764", "53070", "44761", "39762", "7888", "45201", "32153", "59955", "14376", "25894", "15855", "29987", "48301", "3718", "45209", "35318", "45301", "30210", "4934", "14083", "17829", "29763", "13961", "19188", "21383", "4750", "7569", "17404", "32875", "5781", "57832", "56604", "50530", "6630", "57070", "1497", "34631", "18560", "37181", "23332", "21030", "19305", "19620", "18220", "27974", "39959", "23994", "52214", "39252", "52852", "30933", "27888", "1352", "30628", "40353", "37985", "8134", "34217", "37231", "48812", "17023", "44483", "29442", "21126", "16935", "4234", "30641", "6705", "1832", "18734", "22534", "56799", "59444", "38556", "18889", "49207", "22391", "25170", "49869", "51664", "11308", "16413", "11198", "48425", "52694", "43393", "47854", "15859", "58528", "11402", "27542", "11613", "40252", "25217", "22476", "16953", "17345", "16623", "7142", "27374", "29891", "6969", "28396", "32897", "20028", "15273", "25734", "53600", "42427", "13061", "2257", "27056", "20257", "18506", "36474", "54544", "11121", "32618", "26441", "54649", "2789", "16212", "26388", "50952", "22874", "8056", "46785", "38161", "21192", "46999", "38213", "40453", "4793", "12242", "56244", "36214", "36409", "11769", "12851", "3889", "20297", "50624", "42700", "30270", "36327", "58765", "36935", "45285", "46876", "2495", "46801", "46352", "43255", "23583", "31558", "18357", "11850", "4756", "36435", "5186", "16157", "24224", "11651", "44182", "49695", "50466", "334", "14718", "11196", "11905", "2844", "56910", "56761", "38792", "5554", "41849", "17442", "22791", "37455", "1325", "44261", "31301", "27575", "14956", "44399", "37538", "49575", "52008", "43917", "42501", "6563", "37955", "47188", "39949", "38926", "12032", "57963", "48695", "1030", "25979", "9315", "23174", "12556", "7918", "56889", "59784", "54148", "16386", "22413", "13221", "8125", "19565", "51176", "45493", "3326", "27715", "2755", "56068", "13051", "25646", "34334", "11156", "13896", "41468", "39374", "18081", "51734", "654", "26461", "17805", "48699", "7072", "7549", "5270", "4831", "42281", "42649", "19047", "26817", "22362", "39658", "13469", "17585", "48312", "31244", "55948", "50330", "22933", "20338", "7125", "25729", "42552", "27163", "44073", "19420", "5560", "2101", "39698", "10843", "28718", "24697", "58197", "43084", "3118", "20522", "50054", "47941", "32917", "11841", "7933", "19639", "30109", "52388", "31723", "5978", "2363", "47619", "20070", "38164", "58078", "23340", "54040", "13645", "22239", "29558", "18949", "38065", "7210", "41565", "11485", "26829", "16374", "34468", "8421", "42051", "10631", "12874", "15023", "26134", "42944", "39160", "33257", "37566", "43116", "10476", "57840", "3125", "5280", "3657", "4991", "30370", "15470", "52770", "8038", "5827", "32961", "14414", "34766", "12064", "36890", "2886", "55176", "5475", "58017", "33938", "18147", "12764", "40649", "45088", "56081", "56581", "23919", "59113", "52697", "25391", "6242", "25451", "53571", "28892", "22702", "32366", "7085", "5338", "15421", "55460", "21040", "20315", "25665", "52129", "22457", "42063", "21746", "12551", "13387", "20553", "19647", "22211", "39465", "18281", "10113", "51467", "29859", "53092", "45340", "5546", "57735", "57226", "28640", "24845", "25", "45332", "43684", "9769", "7786", "18048", "30352", "34391", "44506", "31653", "6614", "33163", "56434", "13270", "45284", "44930", "9481", "14185", "7354", "8336", "55072", "31584", "30865", "18445", "44440", "24364", "10148", "15549", "40672", "27447", "9465", "47976", "8249", "26946", "10276", "25200", "23214", "1549", "12373", "30073", "49582", "7625", "32038", "44514", "59503", "57691", "56457", "33003", "17242", "34419", "31510", "41762", "51788", "43867", "49829", "37408", "37896", "16406", "55422", "41833", "24903", "52622", "39292", "24100", "29620", "48692", "27694", "3132", "6832", "34687", "42376", "5609", "16642", "8567", "37510", "47789", "9514", "32279", "51505", "7069", "53422", "48358", "30250", "14867", "57719", "40661", "45006", "45649", "34640", "38458", "26198", "47560", "35660", "41317", "10603", "4845", "48148", "42275", "1888", "23267", "29359", "40909", "20683", "31519", "2973", "121", "42496", "32525", "7457", "47782", "19187", "34509", "3938", "3943", "45366", "47720", "6484", "44595", "58152", "14016", "31891", "4248", "59859", "53803", "58663", "56453", "21429", "7584", "3228", "42372", "34777", "36065", "8916", "13827", "46321", "51582", "6044", "29249", "48407", "47105", "40444", "4190", "45958", "12699", "20610", "43862", "40192", "33577", "52661", "42349", "8819", "9491", "37565"]]} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist-test-split.txt b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist-test-split.txt new file mode 100644 index 0000000..548e17e --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist-test-split.txt @@ -0,0 +1 @@ +{"xvalid": ["int", ["9927", "7226", "7711", "7504", "6015", "2924", "5588", "9080", "6835", "548", "41", "9349", "2134", "1315", "1953", "8430", "953", "5563", "8696", "6879", "9010", "2782", "6518", "635", "8580", "9817", "426", "9464", "1157", "3532", "6066", "992", "1426", "3371", "2983", "9727", "1490", "2010", "6199", "9563", "6650", "520", "8246", "5174", "3344", "8539", "970", "3370", "545", "4062", "9741", "7653", "433", "2949", "3177", "4083", "5556", "9023", "2054", "7721", "7119", "7608", "7098", "6761", "2188", "1358", "4420", "6317", "1413", "4702", "7399", "3084", "2856", "7931", "8789", "1999", "4289", "2009", "7791", "4437", "1191", "7544", "6788", "5706", "6814", "2557", "9432", "9509", "6830", "6176", "7836", "8787", "157", "3881", "5022", "5857", "5462", "9812", "6467", "2569", "5746", "2376", "6551", "3009", "209", "2437", "7890", "148", "3321", "4701", "3555", "2212", "1554", "3143", "3223", "155", "4848", "7845", "2071", "8945", "7740", "8069", "9425", "3041", "3266", "2661", "5376", "3104", "7457", "4528", "5110", "6870", "9898", "3521", "5846", "2220", "5085", "8232", "5525", "9859", "4309", "3579", "8281", "3526", "3199", "5675", "7333", "3191", "9700", "3896", "4564", "6533", "4462", "4846", "7884", "3500", "3294", "9180", "7825", "9461", "156", "6065", "8200", "9532", "7222", "4078", "1208", "2191", "5067", "4056", "8169", "332", "6614", "5815", "8487", "8483", "3608", "5480", "740", "6815", "2099", "745", "489", "6138", "6827", "1704", "1596", "7509", "3938", "9539", "3386", "3567", "8175", "9137", "8163", "9490", "8999", "3150", "1087", "3355", "5354", "1738", "785", "7246", "5414", "3776", "8237", "1683", "7956", "4439", "614", "3835", "3156", "3721", "667", "8797", "7675", "1128", "6203", "2653", "8009", "8274", "4512", "8523", "9925", "8090", "5107", "9462", "4535", "2243", "3292", "8809", "3771", "7586", "1135", "7948", "3954", "5801", "8638", "9717", "1346", "7525", "702", "3250", "9558", "2532", "8844", "972", "9139", "6012", "5729", "8283", "7448", "7773", "9554", "4816", "5551", "2852", "993", "7599", "1480", "3667", "1581", "8260", "9419", "2961", "4854", "3048", "9298", "4291", "7701", "3987", "9627", "487", "1621", "179", "595", "8938", "7412", "990", "1296", "229", "9525", "4540", "4130", "83", "113", "1796", "5736", "4328", "9493", "8161", "5", "2977", "9096", "6576", "4249", "5647", "2593", "6487", "6027", "6226", "2187", "1449", "6101", "123", "3269", "4847", "7708", "134", "9619", "8769", "9099", "8435", "6850", "9428", "7765", "7651", "6940", "4678", "9353", "1824", "8746", "7938", "9348", "5189", "7941", "9151", "5302", "4578", "7407", "7572", "2475", "7735", "461", "7492", "300", "7989", "2202", "6667", "4007", "9517", "8499", "7562", "5942", "5874", "9453", "9389", "8923", "6420", "4352", "4442", "3242", "9971", "7332", "2289", "6334", "6007", "9826", "5446", "2273", "89", "2883", "2316", "7288", "1775", "3458", "7219", "3718", "6961", "9198", "1685", "5805", "6753", "5844", "6781", "9444", "3174", "7234", "3057", "7555", "4667", "7617", "10", "8327", "1520", "2635", "1352", "2216", "2975", "410", "2183", "4253", "2686", "6574", "6676", "7058", "6654", "2693", "8121", "9057", "1696", "2474", "1669", "2979", "131", "9637", "7051", "653", "8572", "9393", "8693", "1096", "6578", "8753", "8538", "6395", "8955", "2038", "908", "3983", "5850", "1148", "9787", "9930", "562", "1679", "9739", "8340", "3744", "6605", "6216", "1797", "4990", "8035", "7451", "8907", "853", "6941", "4031", "4739", "4299", "5829", "3221", "2022", "4964", "3044", "1505", "6262", "5546", "1850", "7878", "3559", "8374", "2307", "5267", "2966", "7606", "4189", "2560", "9820", "3280", "1458", "3573", "6048", "4502", "6315", "8612", "3887", "9229", "4579", "5954", "8255", "9413", "5725", "2324", "6653", "611", "4193", "8802", "3910", "8917", "418", "326", "1793", "372", "1070", "4153", "9729", "5528", "1494", "8288", "6303", "8047", "1102", "1323", "9078", "6225", "6726", "9847", "5745", "8725", "6923", "4250", "6696", "8347", "7786", "6450", "9395", "1445", "9417", "2627", "1607", "2955", "7263", "2770", "6580", "7648", "3442", "9447", "9863", "6001", "6316", "5822", "1369", "7456", "2471", "9723", "8140", "3759", "4030", "9359", "1216", "2588", "1023", "8051", "5499", "4797", "5603", "9150", "3980", "6463", "3146", "634", "6069", "6906", "7829", "5644", "1921", "2320", "1175", "561", "8491", "8515", "2353", "7667", "8381", "9206", "6254", "6857", "7201", "3086", "2638", "3591", "6049", "9060", "4754", "2675", "5601", "6190", "5612", "6688", "8636", "2622", "5720", "9917", "7835", "1174", "6126", "2252", "4436", "3615", "4146", "3372", "9094", "4961", "8502", "3017", "5761", "8975", "21", "3356", "7236", "6881", "346", "3735", "2290", "6143", "3101", "4292", "2809", "9996", "922", "6025", "4150", "6256", "2854", "7539", "6666", "8633", "9324", "421", "6109", "4473", "357", "5540", "6706", "5604", "9630", "122", "6035", "5296", "8227", "1946", "137", "3284", "2527", "8153", "9255", "791", "1084", "636", "6607", "3408", "5534", "5967", "1500", "9570", "4529", "236", "2024", "6708", "2394", "5496", "6374", "4693", "9830", "1933", "730", "2893", "2887", "9888", "975", "6020", "5891", "2904", "6051", "4722", "9079", "7955", "831", "1097", "7980", "7837", "1044", "1354", "4819", "2836", "6983", "815", "1755", "2659", "7545", "6370", "361", "3956", "6164", "9824", "3991", "5620", "9387", "9672", "1889", "4143", "4196", "6010", "6366", "2462", "8928", "2873", "7866", "4527", "8549", "6584", "1138", "6246", "2718", "2057", "3699", "9496", "4463", "4246", "8517", "7928", "9695", "8698", "3493", "9231", "6560", "7640", "2441", "4050", "6843", "9641", "6895", "4247", "9189", "8775", "1153", "3720", "445", "59", "8306", "2764", "1836", "7804", "5547", "5562", "6107", "2886", "4531", "5033", "5199", "2082", "2765", "2456", "3001", "2756", "6922", "8729", "6352", "2603", "3528", "1492", "6208", "2176", "951", "6494", "280", "8776", "3697", "8257", "499", "5087", "8958", "7902", "9515", "6358", "3981", "6248", "1725", "2704", "9823", "3303", "9289", "8440", "1552", "8224", "652", "5134", "3645", "5527", "6475", "3210", "9303", "7561", "5981", "2799", "1250", "497", "7762", "6723", "5120", "7681", "6061", "2812", "4243", "6037", "5778", "6006", "1561", "8641", "9774", "6709", "7537", "80", "7799", "1842", "7524", "2385", "2680", "5584", "1594", "50", "5976", "7839", "7322", "1985", "3994", "6477", "9840", "3830", "5315", "5569", "3401", "9883", "1863", "1201", "4956", "2841", "3301", "6582", "2950", "1254", "6083", "6917", "599", "7392", "818", "2587", "9959", "6659", "995", "9682", "106", "6394", "1822", "7010", "5657", "6639", "5121", "9284", "3216", "1234", "5316", "6112", "5969", "1419", "7526", "3359", "5781", "2281", "7679", "9606", "3024", "9956", "4913", "3595", "5517", "3879", "2802", "2604", "8339", "8295", "9429", "9136", "3404", "8036", "5564", "2523", "9127", "4812", "6894", "2393", "4177", "4381", "4271", "764", "6916", "9046", "1592", "2709", "4144", "4612", "5438", "6842", "7505", "6009", "891", "6215", "3923", "3384", "9655", "2796", "7830", "3622", "6016", "5670", "8284", "4341", "1785", "9588", "3314", "1059", "806", "1588", "9598", "4164", "8012", "8500", "9158", "9599", "9758", "7242", "8632", "3172", "7581", "4082", "3262", "56", "4277", "2426", "2888", "1132", "4331", "3554", "6738", "5904", "1264", "99", "5395", "546", "5627", "8354", "51", "6981", "5400", "9557", "7776", "233", "1538", "4303", "2162", "1222", "2785", "8142", "7092", "9716", "8503", "5420", "3846", "6545", "1005", "1632", "9740", "948", "368", "9452", "2206", "4232", "754", "4906", "1281", "295", "3635", "5924", "8454", "3704", "3192", "6206", "2368", "3482", "6241", "2577", "9833", "4451", "3122", "4113", "9526", "4625", "5876", "9873", "4872", "4112", "1595", "5304", "651", "9114", "7703", "390", "5531", "4751", "6105", "491", "6271", "5794", "7155", "8165", "7998", "495", "9130", "7256", "3380", "1195", "3224", "1137", "5368", "3377", "6573", "6784", "4719", "2610", "8588", "4393", "6565", "1031", "7657", "5309", "9187", "8249", "2611", "4295", "4282", "1080", "9405", "7595", "4026", "6925", "7696", "2163", "6924", "8265", "4338", "3644", "1344", "2582", "244", "964", "6740", "8755", "3917", "7317", "5948", "8476", "9356", "3750", "3234", "7780", "203", "3965", "7557", "6811", "4343", "153", "1789", "6954", "3293", "7273", "2642", "6508", "8837", "2280", "2132", "3722", "7286", "1885", "183", "1633", "1068", "2460", "6966", "3815", "8369", "46", "6124", "9311", "5254", "4748", "5079", "6880", "7188", "2998", "4841", "2810", "3212", "7142", "688", "7751", "6568", "642", "4938", "4583", "2906", "1781", "3213", "5300", "4274", "5108", "2772", "5858", "7593", "1348", "9436", "1579", "8873", "6722", "5716", "6949", "4728", "2978", "1278", "4714", "7239", "9085", "3825", "86", "1409", "3547", "9251", "4301", "8338", "8496", "488", "2780", "9743", "1523", "6570", "7748", "5125", "5383", "8497", "1028", "6982", "584", "8194", "1307", "1107", "3075", "7893", "8551", "7402", "264", "3050", "6428", "7021", "8287", "9802", "2234", "1433", "8253", "7131", "5018", "4618", "6489", "4692", "7203", "8603", "6774", "6288", "7495", "9310", "2379", "1406", "9771", "8624", "7371", "7724", "58", "4744", "4018", "344", "382", "4698", "6567", "775", "2089", "9703", "1371", "3679", "3437", "8024", "6419", "2155", "6219", "3190", "5377", "2798", "5605", "619", "8059", "4984", "6305", "1930", "2985", "303", "1768", "1926", "5052", "6279", "807", "8650", "7535", "7551", "8070", "7831", "5916", "9777", "6439", "8417", "6313", "9089", "3717", "1342", "6221", "6380", "8286", "9318", "3779", "9492", "8139", "7880", "9944", "276", "6848", "9227", "9849", "5195", "6698", "3353", "9065", "8716", "9954", "6082", "5910", "2205", "6036", "1979", "4332", "2287", "6437", "8248", "7589", "5837", "1124", "565", "6399", "8158", "3758", "2553", "8209", "9800", "2820", "1156", "8461", "1337", "366", "7", "324", "1565", "2540", "1257", "8543", "8851", "4344", "1364", "3028", "8840", "3511", "2947", "2391", "5669", "6345", "8008", "6575", "1333", "3230", "386", "6914", "7116", "3958", "1303", "9209", "3061", "7832", "5374", "720", "2292", "9747", "8477", "4557", "7009", "3797", "8841", "3052", "9113", "260", "8250", "1935", "2928", "761", "2524", "8611", "3587", "4837", "8931", "4132", "9620", "4882", "827", "8779", "2037", "6360", "3862", "7467", "4372", "8731", "7264", "6934", "6033", "856", "8530", "425", "1209", "2803", "3728", "1829", "9953", "7781", "4864", "710", "1006", "2305", "3426", "145", "5558", "9334", "7785", "2354", "8969", "1420", "4522", "2377", "9259", "5913", "2448", "3285", "2261", "645", "2053", "5909", "8093", "6402", "3624", "4903", "8054", "2514", "5116", "8447", "9706", "1774", "4725", "1763", "2539", "5062", "7811", "4815", "7339", "7499", "4768", "4985", "7588", "7150", "1381", "3913", "1577", "6778", "8599", "146", "5149", "8310", "7170", "3782", "4116", "4327", "2390", "1560", "5961", "6792", "3590", "5000", "5622", "8987", "8691", "7997", "5911", "2118", "8690", "5749", "1360", "7915", "4770", "380", "9213", "4785", "2033", "160", "5680", "8508", "855", "6945", "8178", "7634", "1073", "4647", "3171", "400", "5919", "8373", "6873", "1244", "5668", "9985", "475", "5580", "1114", "1470", "456", "3439", "8821", "288", "1422", "8429", "2835", "9336", "4006", "8902", "6142", "7347", "379", "5554", "9407", "196", "6631", "5667", "758", "6269", "570", "9810", "3054", "8408", "2833", "4080", "2119", "8831", "8724", "2223", "5478", "6149", "4425", "6137", "5009", "7318", "4835", "70", "1945", "6184", "6936", "7553", "3533", "3823", "722", "5045", "6672", "7216", "7777", "6068", "2571", "1172", "3874", "3175", "1411", "2689", "8390", "2777", "7335", "1192", "5281", "7094", "8399", "5481", "454", "7083", "345", "2951", "149", "334", "4525", "4001", "9400", "3610", "753", "5963", "9204", "9827", "5161", "8398", "1666", "4514", "2227", "3378", "1025", "2607", "1744", "2816", "849", "3619", "5465", "3690", "363", "37", "6554", "9365", "4096", "8738", "4937", "8244", "2794", "9005", "862", "5719", "8552", "7901", "4071", "6604", "5931", "2042", "9105", "9243", "8667", "9374", "5243", "4413", "1386", "6816", "3535", "369", "7114", "5202", "9322", "2016", "8204", "536", "1223", "7452", "5855", "2757", "6514", "4021", "2327", "1776", "2899", "8488", "5440", "6323", "6728", "2131", "2015", "5466", "1687", "8876", "2067", "2636", "4546", "1978", "4368", "813", "1144", "5115", "6588", "2435", "9247", "9708", "7787", "4589", "4491", "4152", "320", "5138", "3318", "7284", "857", "7079", "8014", "2389", "1240", "1159", "3053", "941", "5784", "1310", "1258", "5241", "6601", "1322", "1295", "917", "3089", "5508", "9533", "1801", "3066", "9860", "8001", "5104", "9566", "7283", "4614", "7760", "8375", "2336", "2195", "6498", "2828", "202", "2025", "2298", "4081", "1623", "2624", "8595", "7857", "9260", "9102", "5358", "9756", "8868", "129", "4482", "9221", "9369", "1752", "1710", "5194", "9015", "5516", "4279", "6645", "6024", "610", "723", "5404", "8721", "6181", "8251", "4680", "788", "6749", "2165", "342", "98", "9766", "7054", "7512", "7779", "9035", "5063", "8379", "8294", "8182", "1173", "5084", "6797", "6461", "5430", "8527", "7700", "1922", "533", "3139", "2534", "6364", "3102", "8478", "871", "9986", "6099", "9145", "4284", "1610", "3163", "3502", "9110", "3070", "9003", "6990", "2762", "8832", "1110", "9635", "2105", "9176", "1686", "1567", "6957", "6608", "2536", "9293", "7418", "5248", "7272", "5284", "9702", "3814", "1072", "6736", "7036", "4176", "9761", "1294", "5567", "7964", "2643", "6018", "8464", "9762", "4874", "7864", "5391", "6995", "3800", "9663", "1180", "1676", "2712", "2754", "9642", "9068", "9442", "6929", "5055", "4855", "3004", "868", "1572", "800", "2000", "3497", "7729", "3571", "2572", "4371", "6757", "7600", "4729", "3279", "8619", "6423", "7384", "5938", "1656", "9634", "8786", "3883", "8960", "9905", "192", "8791", "1974", "3951", "5164", "1548", "8718", "293", "5935", "7171", "4723", "2859", "2", "1063", "5533", "4839", "2525", "1643", "6552", "8099", "7728", "6987", "1742", "452", "6525", "8396", "2498", "5656", "7730", "2095", "4167", "9438", "877", "9093", "6909", "434", "7220", "8585", "2196", "1912", "4408", "4631", "7320", "9769", "7055", "6960", "6550", "2908", "1313", "7530", "3034", "4865", "2387", "5427", "7041", "6822", "4028", "186", "9793", "1630", "8532", "9372", "4095", "142", "4403", "3948", "3520", "4866", "8985", "1190", "3374", "3430", "3921", "982", "2358", "6805", "1444", "9922", "1261", "9299", "8537", "3578", "4515", "4334", "333", "1171", "4044", "2941", "4042", "2101", "2006", "5364", "225", "9834", "9477", "9799", "1321", "7056", "9384", "4202", "9499", "6052", "4127", "5691", "3773", "5777", "2229", "7107", "4426", "6229", "2203", "3117", "2317", "1739", "338", "9808", "3445", "1625", "8732", "8451", "4782", "9903", "4005", "9185", "1986", "2428", "1587", "3678", "6768", "9479", "7733", "2981", "8857", "3909", "726", "3914", "1008", "2537", "5200", "1841", "9937", "5355", "3629", "8279", "6056", "949", "7348", "1920", "4762", "7243", "3582", "4849", "9775", "586", "4357", "5732", "7097", "4942", "3354", "8915", "8679", "8276", "3791", "3443", "6158", "7754", "4041", "2800", "1152", "375", "1050", "9043", "9375", "4370", "5917", "7816", "1473", "2662", "2083", "8102", "823", "2505", "6445", "8400", "6172", "9451", "3855", "6088", "126", "6150", "5314", "72", "496", "5775", "872", "1474", "1064", "6734", "6907", "1848", "2818", "5008", "1665", "4063", "2065", "9950", "2729", "6100", "7489", "3642", "7797", "1248", "453", "5188", "3626", "9306", "9004", "4774", "5375", "8509", "8859", "7244", "6823", "1578", "7514", "8562", "1251", "3727", "3510", "1806", "5763", "6927", "3328", "4418", "8860", "221", "1357", "7885", "4558", "8592", "8205", "6897", "9437", "62", "542", "7298", "916", "1942", "5408", "9906", "5152", "8598", "9087", "4989", "6724", "8292", "1403", "1557", "1290", "3523", "2615", "2825", "5163", "7925", "1820", "4977", "5630", "2467", "8443", "8513", "4566", "3979", "2240", "125", "4342", "1094", "6978", "9488", "7962", "4000", "9957", "8728", "5663", "7924", "5526", "4145", "7444", "694", "9246", "2253", "4507", "8833", "8042", "8827", "471", "3680", "9196", "8256", "7442", "3997", "864", "2170", "8074", "3631", "9955", "4435", "7277", "6185", "5624", "5128", "8634", "5776", "3376", "5451", "6755", "3220", "2829", "7482", "111", "2584", "6432", "6813", "9266", "3400", "4902", "8564", "7088", "1719", "1332", "4203", "242", "4981", "997", "2425", "6499", "4642", "9885", "304", "3930", "9730", "121", "1534", "5157", "1463", "3912", "8148", "9862", "4221", "8606", "4241", "9712", "2705", "9640", "150", "6812", "7035", "8704", "3347", "2834", "1550", "4601", "7649", "8687", "6959", "9033", "9942", "9929", "2247", "5626", "5762", "5094", "24", "6233", "6829", "1866", "7953", "5495", "523", "268", "1906", "8020", "6837", "8291", "2395", "7459", "8808", "6074", "5812", "100", "3593", "6469", "8816", "6542", "6092", "731", "2250", "989", "1349", "9770", "5819", "5201", "1760", "6710", "8433", "7659", "3766", "4828", "3649", "7334", "6304", "5884", "7792", "8774", "9877", "4805", "30", "1522", "6503", "9362", "4440", "1429", "5196", "7312", "2122", "4484", "216", "8241", "5277", "3274", "4845", "5788", "2892", "2439", "3045", "1892", "7120", "3434", "1204", "5615", "2945", "443", "7541", "8053", "8079", "8037", "305", "7783", "5834", "3173", "3255", "287", "2547", "5936", "6095", "6258", "8756", "5318", "6397", "9115", "1840", "1756", "2147", "6975", "2530", "8404", "9366", "2136", "5229", "7001", "560", "7716", "1546", "3866", "7125", "7472", "7947", "5210", "6979", "6046", "4862", "6682", "6086", "1677", "9143", "2673", "2235", "514", "8363", "6320", "5259", "8107", "1331", "6444", "8852", "1777", "6266", "2040", "1401", "8290", "290", "7523", "9969", "4073", "3281", "7082", "7202", "3592", "2036", "8803", "2238", "4738", "3465", "2097", "5929", "7769", "3035", "1859", "240", "170", "3734", "7161", "5789", "4012", "4423", "2629", "6854", "7374", "4470", "1680", "8930", "5392", "2814", "5455", "626", "5602", "9624", "9327", "8553", "7377", "6223", "271", "7198", "213", "3261", "6810", "1432", "4236", "4908", "5312", "6920", "9518", "5397", "3118", "7484", "7048", "1013", "7238", "3729", "1684", "1227", "7942", "5370", "8658", "7638", "661", "8423", "6300", "8853", "4465", "9783", "370", "4619", "9273", "5979", "1103", "8382", "1118", "8455", "3033", "8735", "7768", "4955", "9062", "207", "2133", "3594", "5261", "9412", "5951", "6414", "9658", "2943", "4088", "3153", "3010", "8234", "8906", "509", "3864", "249", "5036", "7604", "4909", "6752", "2559", "6390", "325", "2519", "6179", "7990", "5406", "5271", "772", "2443", "7075", "9722", "313", "3361", "7064", "6595", "9465", "8711", "4765", "8492", "3214", "6353", "9245", "9536", "7401", "7906", "646", "8467", "9875", "1913", "84", "2047", "4584", "5586", "231", "2600", "6510", "9551", "8601", "1462", "557", "4090", "1547", "2388", "308", "406", "8628", "4244", "8267", "837", "8084", "4204", "2372", "1611", "235", "9201", "4054", "6151", "6454", "6862", "1862", "365", "234", "3491", "1340", "2447", "8540", "3996", "8957", "6404", "4915", "4489", "8908", "531", "1454", "4708", "3794", "270", "1277", "7311", "4387", "378", "7316", "5228", "6438", "5372", "3492", "5824", "2548", "6132", "6621", "7921", "5731", "9317", "1229", "822", "2745", "5943", "6564", "6110", "2827", "7141", "6630", "485", "7808", "5849", "2847", "2668", "4741", "3270", "283", "7750", "7435", "9752", "3412", "1603", "4467", "1689", "4200", "1111", "5791", "6162", "3481", "1568", "4367", "1424", "9529", "6776", "7337", "8670", "3806", "4879", "4616", "5798", "6893", "3026", "9261", "900", "2407", "5208", "7341", "3031", "6926", "4424", "3176", "7772", "1121", "3812", "4665", "6597", "171", "2502", "3787", "4293", "9665", "9589", "5831", "3123", "2073", "4210", "1728", "7204", "3973", "1627", "6118", "3726", "2652", "9186", "3999", "5204", "2338", "738", "6681", "5021", "4190", "8171", "7232", "5901", "5439", "9254", "839", "8077", "803", "8817", "5550", "1343", "5548", "6530", "4852", "5985", "175", "572", "687", "9946", "4266", "5712", "9751", "6023", "7933", "6999", "8568", "9228", "7310", "5396", "1914", "1004", "4928", "1398", "1537", "6496", "3968", "3198", "8778", "9683", "8800", "8407", "2104", "4791", "5251", "1079", "9343", "8989", "9382", "5955", "6479", "6122", "6548", "2645", "3799", "4347", "7488", "7275", "4045", "6188", "1751", "9514", "747", "1792", "6311", "7803", "9543", "4115", "9020", "1530", "2921", "9995", "1962", "1721", "3271", "8904", "7749", "48", "3409", "8920", "3065", "7023", "3893", "2117", "8782", "9650", "248", "312", "7930", "2683", "3998", "9170", "2784", "376", "5738", "2564", "9376", "8744", "9801", "1324", "6080", "3689", "1443", "2942", "5804", "938", "5679", "3903", "4128", "6579", "5463", "819", "8922", "9638", "4234", "559", "6629", "7397", "5434", "289", "3112", "5401", "915", "4185", "5240", "8380", "417", "9049", "1773", "7127", "239", "8384", "705", "8881", "9671", "4844", "9161", "8243", "177", "7061", "9553", "1772", "7584", "709", "1329", "7030", "4053", "42", "5795", "5539", "5119", "278", "3073", "8741", "6678", "3988", "9866", "1705", "7813", "5175", "3485", "5330", "2044", "2973", "9279", "7826", "8180", "3828", "4186", "8940", "5076", "2937", "6591", "3496", "9031", "7157", "5560", "8700", "2597", "9171", "2750", "8793", "3816", "3436", "1589", "4166", "1468", "6", "7662", "998", "5990", "6637", "6145", "3665", "675", "9726", "5069", "3433", "1798", "7380", "4330", "6766", "2890", "978", "3756", "1255", "8646", "2842", "4666", "1800", "9449", "5595", "4577", "9963", "262", "4510", "2562", "6125", "2208", "3310", "6411", "5293", "392", "2001", "676", "591", "2651", "6537", "7305", "4233", "492", "8304", "4887", "9516", "4350", "678", "3133", "2862", "3604", "6057", "3304", "8587", "502", "8328", "9528", "3796", "7074", "3963", "8947", "6836", "1835", "8394", "5371", "4629", "1746", "4340", "8282", "6260", "3793", "8162", "5733", "2268", "6325", "3185", "6844", "3217", "4876", "8231", "6129", "8371", "8346", "1194", "2228", "1515", "1544", "4581", "3919", "4318", "4793", "3733", "4521", "6635", "1392", "5274", "3332", "6318", "7619", "3955", "3264", "5888", "9713", "9844", "2382", "3339", "7852", "2546", "3556", "5830", "8245", "3461", "7121", "7404", "8609", "2231", "8235", "7031", "3534", "5007", "8828", "5338", "1531", "3167", "7794", "4109", "63", "5071", "7095", "2416", "8141", "7614", "4780", "8550", "6194", "1141", "7527", "2279", "9520", "9600", "8278", "5222", "712", "9534", "7583", "8734", "7139", "6619", "5922", "8883", "4375", "8834", "663", "3092", "907", "5328", "3238", "1830", "8456", "1647", "6211", "924", "35", "9674", "9495", "9146", "2791", "5166", "3158", "1302", "5048", "3789", "8333", "133", "3895", "8434", "3251", "5464", "9908", "6484", "1663", "9498", "3686", "47", "5425", "3761", "4806", "3392", "7993", "5351", "3764", "1753", "3", "272", "2455", "2927", "3751", "8365", "2098", "8351", "4777", "9017", "1952", "973", "1212", "7196", "8610", "5728", "6349", "8004", "1284", "6005", "9274", "1203", "1864", "8298", "9482", "441", "6665", "8392", "5800", "8642", "466", "3878", "6329", "1805", "7739", "8900", "5928", "9160", "2484", "8482", "281", "1667", "5409", "4622", "4769", "780", "7824", "2108", "9414", "7714", "5752", "2311", "5279", "5226", "5444", "6832", "7491", "211", "8751", "4732", "9597", "1089", "4124", "4084", "8854", "4626", "2707", "8743", "1387", "9697", "612", "7874", "9379", "8436", "1910", "727", "2152", "4392", "7635", "7462", "1230", "6175", "9109", "2246", "3368", "292", "8437", "4598", "1077", "7261", "8992", "6670", "6476", "2138", "1655", "5994", "6308", "4645", "7065", "4321", "9935", "6538", "1735", "9121", "2464", "5244", "285", "3441", "7300", "9235", "649", "4914", "6446", "8185", "1670", "2768", "5040", "2658", "1495", "87", "6178", "493", "7005", "6115", "9123", "7038", "748", "9575", "5880", "1052", "5433", "92", "7494", "2418", "6807", "7479", "9128", "7625", "3931", "2958", "4394", "389", "804", "9167", "2551", "5137", "3529", "7270", "8425", "9742", "5949", "3551", "9111", "7276", "5441", "3970", "2703", "2891", "6694", "7693", "6000", "9040", "6171", "2826", "2078", "9468", "1218", "1428", "3817", "8092", "9312", "6350", "5991", "9544", "6239", "5043", "4764", "8062", "2406", "5126", "8919", "3819", "5918", "5783", "5153", "6783", "6022", "7646", "7003", "7378", "11", "1871", "1241", "7795", "9698", "1918", "5415", "4216", "810", "3621", "6133", "7182", "6522", "8655", "5790", "6330", "4587", "8458", "5319", "5272", "6991", "2984", "4787", "9945", "4747", "2681", "3233", "6756", "6231", "261", "7431", "7160", "7207", "5388", "6458", "8321", "845", "9241", "5494", "2129", "9868", "4790", "3561", "4881", "409", "933", "604", "798", "9047", "9843", "1695", "5840", "7346", "1451", "3842", "4707", "4061", "8621", "4716", "8146", "189", "9018", "3232", "7497", "8138", "4582", "4296", "2004", "2478", "9549", "9546", "8387", "1703", "6523", "4730", "3788", "664", "8772", "4635", "7321", "1837", "9308", "2925", "6602", "4971", "2844", "3236", "1712", "512", "8639", "9321", "4929", "8124", "8962", "6955", "1987", "8703", "9836", "6684", "9975", "2274", "5998", "566", "3813", "5227", "3422", "4591", "1499", "8893", "6041", "6875", "5568", "8136", "4877", "1466", "7631", "6234", "3509", "8067", "6764", "4724", "6113", "8701", "5744", "222", "686", "2345", "3429", "1404", "9257", "5809", "7686", "6746", "9364", "5385", "1164", "1247", "6765", "7454", "1481", "1813", "991", "8337", "3352", "3858", "6369", "7726", "7413", "2237", "2896", "8343", "4273", "5613", "6470", "3570", "194", "8846", "3459", "8645", "2963", "208", "7611", "2849", "6720", "6935", "3584", "1917", "8884", "3719", "7722", "401", "6340", "7513", "3843", "2139", "5536", "9354", "8998", "9654", "7279", "4077", "1283", "2987", "7950", "9450", "5220", "9118", "8579", "3218", "530", "5326", "9711", "7650", "5407", "1694", "9265", "457", "8647", "2948", "3588", "5774", "1598", "6412", "7357", "2517", "6763", "1519", "5247", "5324", "7886", "5365", "8558", "5295", "9690", "5353", "25", "7669", "3783", "3853", "8391", "582", "5854", "6950", "6298", "8203", "4814", "9736", "3126", "1697", "1390", "9117", "4875", "6703", "1901", "9662", "7326", "440", "1948", "3691", "5294", "8386", "1297", "6712", "2185", "4223", "3343", "2838", "746", "3300", "939", "9778", "1932", "2578", "735", "7227", "8982", "1233", "3235", "6779", "571", "193", "6521", "5214", "8045", "2664", "7086", "3623", "4712", "8891", "9263", "1060", "3918", "6361", "8954", "1099", "4808", "8504", "172", "6971", "9531", "9816", "6153", "2217", "6725", "7020", "6845", "2012", "8750", "2863", "5359", "1", "4486", "2976", "267", "683", "9252", "8112", "555", "921", "5056", "1899", "174", "6972", "4157", "7949", "1136", "6968", "8980", "691", "5340", "5418", "9791", "5714", "2676", "3275", "7476", "4536", "5513", "7639", "3333", "4029", "7800", "8418", "4235", "7049", "1267", "2423", "2453", "5814", "6625", "265", "6634", "9072", "60", "7966", "3315", "9063", "9398", "5930", "1883", "5256", "8050", "2219", "3964", "9071", "5102", "6455", "9889", "6085", "5690", "7474", "5748", "9392", "8484", "460", "7732", "3447", "9458", "3946", "1359", "6296", "9051", "191", "885", "3703", "3018", "4532", "4941", "6427", "7717", "677", "8874", "8829", "4501", "2632", "4809", "4496", "6449", "3916", "8415", "2728", "2956", "7173", "2271", "5290", "5167", "1965", "3055", "9370", "3060", "3600", "4033", "1069", "4211", "7319", "5650", "9203", "4843", "4559", "93", "2161", "739", "4585", "5211", "616", "1631", "7299", "4363", "8368", "6819", "1021", "9991", "6947", "2585", "8600", "5734", "6534", "539", "2411", "5907", "360", "4995", "2449", "6524", "7032", "4542", "449", "3661", "7293", "6031", "8445", "5413", "467", "5165", "8843", "5282", "8052", "415", "3410", "3517", "2374", "9401", "9224", "9314", "4575", "2153", "3712", "2716", "5518", "6295", "8144", "6409", "4615", "1459", "352", "9692", "9052", "6771", "7546", "6117", "3508", "6942", "4966", "3989", "7636", "1988", "9788", "1160", "5621", "5773", "3435", "9993", "8976", "3684", "3772", "9194", "4480", "8011", "6443", "6743", "2516", "2199", "8666", "5964", "9326", "8296", "6655", "2992", "4801", "1799", "298", "6803", "7736", "2618", "2154", "4311", "6398", "6462", "5380", "1767", "7788", "7853", "3860", "8097", "2058", "8864", "6456", "4085", "5766", "838", "4951", "3982", "7986", "4209", "2128", "7673", "1511", "9853", "909", "8309", "6281", "4950", "6599", "9012", "828", "2719", "103", "7761", "594", "1461", "9594", "3383", "436", "3536", "2808", "8643", "5156", "923", "4518", "6111", "3625", "6721", "4884", "5934", "4948", "8569", "814", "4834", "621", "3738", "8345", "3501", "7445", "714", "8419", "5708", "8623", "135", "4278", "5278", "1732", "7538", "7353", "8017", "8174", "7450", "6596", "9887", "564", "5327", "9664", "3688", "5206", "2003", "7612", "6679", "3905", "8689", "6064", "1133", "5176", "2877", "8577", "2222", "9388", "5886", "9363", "9439", "7421", "2739", "5473", "4133", "4240", "3865", "7798", "1539", "4824", "6198", "3170", "5940", "2583", "3290", "9485", "1513", "8939", "5249", "2880", "2690", "3723", "6371", "8850", "1086", "6249", "4983", "9032", "6800", "5945", "2386", "9082", "15", "7136", "5502", "6657", "199", "1147", "2085", "6447", "8007", "927", "7575", "2383", "6058", "756", "9958", "5265", "9645", "3856", "8486", "9914", "7135", "3402", "477", "7129", "9855", "2429", "6436", "2789", "1878", "3795", "8460", "119", "5520", "26", "1076", "1717", "4009", "7381", "4506", "8293", "9027", "8932", "7774", "3824", "7217", "8977", "9931", "2115", "8886", "2868", "2721", "6668", "139", "2100", "4106", "9316", "4973", "3889", "504", "9236", "8352", "1431", "2660", "2367", "7336", "7382", "2840", "3763", "5459", "1215", "9602", "6963", "1877", "532", "4901", "5287", "8659", "8401", "4922", "2120", "8322", "3836", "2064", "556", "6483", "5416", "9763", "9037", "356", "3564", "5758", "3755", "5780", "6751", "7117", "9440", "6553", "7632", "8472", "1447", "1402", "1464", "1335", "7656", "2278", "9041", "1528", "4492", "8855", "7296", "3698", "783", "1867", "8764", "6299", "7167", "2674", "9694", "1980", "8048", "4197", "5307", "6410", "7307", "6059", "8625", "861", "7345", "947", "1184", "6616", "8341", "507", "771", "4959", "8959", "2735", "3140", "3596", "797", "8965", "2596", "4893", "8942", "9399", "3522", "5336", "5083", "547", "1139", "9309", "7465", "3743", "8719", "5286", "9681", "8644", "7186", "4776", "3468", "4606", "6116", "7325", "8285", "3023", "9391", "1187", "8839", "4107", "8055", "9973", "3431", "1373", "1507", "1214", "5006", "2431", "2567", "9886", "1923", "4638", "1615", "3246", "1843", "28", "446", "4545", "2404", "9297", "4445", "384", "9175", "8541", "4927", "3648", "201", "8752", "8882", "7992", "5333", "8172", "4493", "481", "5926", "8727", "579", "1039", "4652", "7363", "7899", "9237", "6877", "4064", "8984", "2602", "4550", "608", "6310", "8421", "1714", "1661", "4520", "7720", "4263", "5900", "6384", "1692", "5751", "7801", "2323", "4934", "4103", "9164", "2790", "6705", "167", "782", "4431", "7089", "1783", "529", "3046", "3206", "8654", "9979", "6452", "4823", "8547", "1367", "7396", "1691", "8708", "865", "5782", "1571", "2644", "8524", "7536", "4034", "527", "252", "9083", "4736", "5700", "1997", "4982", "9932", "4399", "3877", "6780", "5906", "387", "1929", "2096", "130", "2094", "2430", "4453", "3974", "5343", "4218", "1336", "4419", "8332", "5653", "3746", "8402", "6649", "2483", "9856", "4298", "6727", "7100", "7423", "3765", "8896", "5323", "6674", "7355", "32", "2182", "2167", "8866", "4476", "1904", "657", "2889", "1036", "8535", "3984", "7029", "3103", "3466", "5130", "6146", "1984", "3411", "5402", "2365", "5820", "4891", "3627", "3922", "4600", "2581", "2380", "8720", "3423", "6622", "1795", "5001", "2488", "2433", "2846", "846", "6878", "2127", "6191", "1032", "6677", "1465", "926", "1605", "5127", "9715", "6030", "9744", "7520", "3897", "3809", "2628", "4637", "5996", "405", "3928", "7855", "2521", "7742", "578", "1882", "2282", "2592", "6701", "3780", "1266", "4345", "1091", "3326", "7757", "3276", "8179", "3653", "5600", "7419", "5859", "7410", "7152", "1289", "9329", "9924", "6003", "331", "4604", "8089", "5350", "5902", "8905", "1142", "7674", "2872", "8155", "9430", "3781", "5662", "3724", "8522", "8217", "284", "7508", "6932", "2414", "8151", "4498", "3871", "8428", "6899", "8548", "2730", "8888", "7471", "5025", "6791", "2143", "2670", "8590", "5611", "2508", "7810", "1062", "4954", "227", "8263", "8964", "8044", "563", "5012", "4795", "3854", "625", "7230", "6488", "9585", "8664", "3181", "2903", "5098", "5237", "85", "724", "925", "5664", "4119", "7420", "5190", "1018", "3705", "2357", "3774", "4556", "4735", "3015", "9220", "9938", "3612", "3487", "617", "8813", "335", "7954", "685", "7105", "3798", "4260", "3741", "6072", "4621", "9394", "9952", "2881", "1991", "8604", "1902", "952", "4636", "8120", "102", "4643", "2333", "3558", "9572", "412", "4156", "7935", "4325", "7390", "6480", "9987", "151", "9733", "9446", "4633", "1327", "5145", "3007", "8357", "8788", "3880", "3504", "4811", "2843", "4313", "2744", "5169", "7689", "273", "729", "8663", "2285", "8422", "7268", "4672", "6375", "7398", "8236", "1075", "7847", "2277", "6094", "8783", "6407", "381", "4641", "8819", "9006", "600", "2848", "6834", "2965", "3695", "4407", "3682", "2678", "9567", "7084", "1903", "472", "1833", "7904", "8442", "3090", "920", "61", "9489", "1880", "43", "486", "4733", "824", "1911", "622", "7037", "7490", "5450", "4889", "8403", "8913", "5655", "7743", "4254", "1608", "7340", "3505", "835", "1188", "2806", "1350", "1580", "33", "5235", "5389", "6302", "816", "2725", "1416", "9443", "3641", "38", "1066", "7040", "6224", "6379", "8473", "1529", "7668", "8143", "3498", "8723", "2968", "2684", "1155", "6841", "8771", "6376", "3599", "6928", "1521", "6327", "4690", "180", "2369", "8046", "9162", "7559", "8480", "5821", "4414", "8781", "4089", "4391", "4297", "3470", "2695", "5852", "4215", "1925", "6838", "1378", "640", "6859", "4586", "9367", "6084", "8016", "3888", "6731", "4208", "5212", "4057", "1736", "4048", "1288", "4899", "8474", "1166", "2837", "6704", "9748", "9441", "3760", "4316", "1037", "1759", "6212", "6355", "9320", "9337", "4302", "7718", "2145", "6748", "2264", "7328", "2018", "4307", "7002", "4300", "197", "7821", "182", "7596", "7197", "336", "7863", "1941", "2815", "6831", "2723", "7265", "7344", "7784", "8218", "8238", "1708", "383", "6958", "673", "5506", "569", "4308", "5310", "2366", "2014", "5537", "7973", "1723", "2918", "8355", "1868", "3700", "3204", "7468", "4369", "8648", "2378", "2793", "8187", "3166", "2563", "3467", "1888", "1045", "6343", "5492", "8471", "4561", "3748", "6045", "4322", "3572", "2232", "2614", "6793", "1003", "7715", "4168", "5607", "8117", "118", "7409", "8934", "3805", "6730", "1053", "632", "1826", "8763", "6759", "6011", "4286", "1441", "6641", "9962", "6451", "413", "8973", "7882", "6969", "4567", "362", "9149", "1312", "6818", "5285", "3541", "893", "2554", "6283", "6860", "7245", "5454", "3330", "6504", "9998", "2041", "350", "1394", "8160", "2509", "4192", "1636", "1232", "9275", "6785", "9821", "8510", "8409", "343", "7851", "543", "3097", "6093", "5694", "1509", "2226", "8865", "7124", "2982", "5424", "6490", "6970", "3187", "101", "8108", "7737", "732", "2724", "6527", "9508", "8712", "3114", "5947", "4460", "6754", "2088", "9191", "4417", "3803", "8518", "6840", "1015", "6338", "6586", "3016", "4446", "8192", "3387", "2438", "9397", "8128", "1731", "9899", "4986", "9504", "3581", "2967", "1512", "6240", "9972", "4850", "2399", "5168", "7629", "2295", "6502", "9685", "8672", "7091", "9283", "3668", "1300", "9433", "1270", "3424", "1016", "8208", "5093", "505", "7564", "5606", "2865", "8350", "6217", "5989", "9618", "6642", "5641", "1438", "6247", "6474", "9879", "3844", "7744", "2363", "4715", "2218", "6060", "8184", "725", "3069", "4237", "3519", "847", "4455", "8925", "9501", "5034", "524", "9070", "5129", "4065", "7680", "1186", "3215", "5808", "2565", "2773", "4389", "6594", "8277", "7331", "4932", "2734", "7478", "8498", "5760", "757", "8364", "282", "5337", "3935", "6386", "482", "1395", "3611", "4125", "7393", "2450", "3663", "658", "2920", "1384", "1002", "5275", "6335", "9607", "1453", "6737", "7850", "3241", "1282", "1170", "5489", "4858", "1958", "2758", "2224", "1724", "3151", "8247", "5181", "5470", "4596", "662", "9095", "7193", "5677", "8806", "8544", "2194", "8950", "4523", "6272", "1576", "4290", "6464", "2151", "5666", "8206", "2970", "9659", "6686", "3205", "3863", "96", "9519", "3580", "1707", "3091", "479", "5180", "7011", "5867", "3406", "890", "3484", "1042", "690", "2046", "2650", "9197", "7302", "8259", "826", "1857", "3861", "3681", "3063", "5726", "5299", "5073", "3037", "5705", "5589", "3601", "3042", "266", "851", "8858", "2341", "2535", "228", "246", "6196", "3539", "9039", "4219", "6663", "5769", "3252", "8974", "7343", "8220", "4438", "9486", "7165", "3839", "4264", "1279", "8019", "3006", "9480", "779", "5845", "2623", "8536", "5209", "2929", "9521", "219", "8101", "2851", "1206", "7969", "9967", "4320", "3792", "8127", "6689", "6400", "1982", "2370", "4789", "5750", "5035", "1968", "6321", "648", "9129", "9420", "769", "4373", "4182", "5292", "5863", "2080", "7072", "6733", "349", "3752", "5681", "4194", "7249", "8818", "859", "7473", "8567", "321", "5817", "5112", "9077", "8785", "5966", "6213", "2575", "7892", "5873", "5999", "6418", "7045", "741", "4505", "3448", "8397", "6089", "4910", "1525", "9044", "9182", "4027", "5870", "430", "341", "29", "5975", "408", "6758", "5123", "7493", "3081", "7552", "310", "7873", "9677", "674", "8490", "2144", "1852", "385", "8372", "6357", "1207", "9058", "9506", "7678", "1876", "9427", "200", "4349", "7967", "2339", "1637", "2959", "4252", "1328", "8892", "2923", "643", "1659", "1019", "3677", "7259", "9066", "7235", "1778", "1407", "1832", "2994", "2299", "2995", "8988", "5186", "9686", "5342", "9144", "629", "2352", "897", "5028", "1540", "3196", "5419", "6739", "3875", "8326", "6988", "6377", "8199", "1351", "8948", "6997", "9614", "6532", "5219", "6770", "7620", "3029", "3940", "9735", "8890", "1326", "9286", "1117", "9878", "3345", "3418", "4432", "5735", "1881", "6054", "3381", "6081", "5816", "6660", "3838", "4118", "3851", "8512", "4668", "2084", "5543", "6274", "1396", "4682", "9966", "9152", "7309", "8258", "6186", "3841", "8898", "8261", "7905", "8799", "9500", "277", "4415", "2971", "4565", "9560", "8280", "4019", "3040", "4821", "3488", "8545", "7939", "6549", "9076", "8662", "9148", "9460", "4262", "6517", "7968", "2079", "2954", "7358", "9612", "5037", "9073", "1239", "1675", "873", "5925", "9510", "2953", "8583", "1161", "6658", "6232", "7324", "3711", "9678", "8385", "7327", "6306", "2283", "799", "3421", "8825", "4339", "5889", "73", "65", "7633", "9390", "4996", "5185", "9850", "6173", "7103", "7573", "3208", "4155", "4355", "6717", "4650", "7113", "6804", "902", "7977", "5142", "3506", "3706", "8760", "7597", "7183", "4416", "3670", "3708", "4539", "3211", "5399", "5868", "9470", "3071", "6903", "4225"]], "xtest": ["int", ["4627", "4669", "7487", "5811", "2749", "5770", "2910", "6263", "4306", "9934", "8921", "1049", "3186", "585", "3518", "9819", "9126", "9358", "6070", "937", "6127", "3388", "4921", "4924", "8571", "7470", "2413", "596", "8173", "9212", "9007", "5223", "6275", "7027", "4946", "223", "9383", "1452", "3933", "2620", "1604", "7070", "9423", "549", "5132", "854", "1861", "609", "3807", "5311", "2221", "3609", "4450", "399", "3808", "9992", "2326", "1809", "2351", "7376", "7111", "4245", "4644", "6078", "5026", "668", "1570", "5070", "3282", "5111", "4980", "892", "704", "3512", "5484", "6326", "4038", "1559", "9745", "2328", "8073", "6406", "3457", "490", "2319", "4555", "5234", "4003", "3827", "5449", "6729", "6354", "431", "7367", "6004", "2334", "6440", "4149", "297", "8320", "3549", "3350", "9434", "6090", "9838", "9693", "9199", "3316", "2626", "7096", "9192", "9912", "2962", "5842", "3514", "7979", "6675", "4925", "5787", "4939", "7908", "6718", "6193", "2574", "1949", "7820", "2021", "7362", "6886", "3952", "8836", "6646", "8686", "6636", "9593", "2511", "6656", "2051", "8561", "3638", "9603", "5692", "2999", "5042", "6044", "4174", "7975", "3312", "2434", "3249", "3115", "5483", "4409", "6372", "1510", "8412", "9342", "5266", "4134", "5339", "4175", "3882", "7213", "716", "7071", "8302", "9138", "5090", "4613", "8823", "8953", "8871", "7974", "5914", "3868", "7822", "7190", "5971", "7763", "3628", "7062", "7918", "7295", "3538", "6644", "2672", "4361", "8699", "1849", "4727", "7498", "3416", "6328", "9613", "641", "2698", "5939", "2649", "7610", "5972", "7860", "1397", "9757", "5616", "1873", "5642", "7185", "4766", "874", "9124", "6210", "3785", "552", "655", "4945", "9385", "9524", "9892", "3707", "109", "5173", "3801", "6160", "1977", "1827", "5283", "5105", "3005", "9239", "2329", "7153", "1526", "7211", "4820", "7483", "5357", "4868", "9977", "7366", "1496", "7556", "4999", "3132", "6235", "4285", "7547", "7424", "1727", "1887", "9737", "9679", "733", "8681", "2286", "2263", "4798", "955", "3385", "4867", "942", "9271", "1844", "3480", "1112", "8709", "7568", "4010", "5511", "5730", "3379", "2545", "8080", "2737", "3093", "6536", "8031", "9576", "9721", "2691", "136", "403", "4043", "7373", "1940", "7271", "7134", "3286", "6413", "138", "6531", "367", "4659", "6745", "7594", "2312", "7060", "3452", "2270", "2538", "8317", "5313", "2346", "850", "1414", "4963", "7642", "6520", "7052", "6939", "5864", "1001", "8845", "3463", "9984", "6585", "1821", "1635", "6901", "6322", "6348", "9067", "8875", "1556", "3971", "6515", "6790", "7297", "1894", "1092", "8329", "5721", "8733", "5004", "5704", "2879", "8675", "9968", "5215", "7443", "7951", "881", "2213", "8967", "7987", "152", "2677", "3336", "4242", "3348", "6453", "5467", "9431", "6937", "8022", "75", "1383", "9818", "9921", "5131", "8225", "8366", "9643", "8613", "2233", "4346", "1412", "1597", "2304", "7356", "836", "9120", "5878", "4826", "706", "5709", "9291", "263", "4869", "256", "7603", "9625", "9174", "7907", "3848", "7057", "3351", "1769", "6587", "4122", "4772", "8935", "2193", "1477", "9755", "6473", "3323", "6289", "4123", "3937", "8305", "9455", "946", "2503", "1939", "5827", "6102", "4105", "701", "1562", "7475", "4049", "1828", "7066", "6839", "2156", "9644", "7665", "7076", "1969", "1456", "6383", "9472", "2783", "2030", "4047", "3072", "9981", "834", "2882", "1051", "4385", "9933", "2473", "4628", "956", "5673", "8459", "7887", "5877", "8416", "7287", "107", "7856", "9792", "4767", "4213", "1701", "6592", "6952", "3602", "5074", "6809", "2722", "4778", "3164", "2940", "5191", "204", "9621", "4700", "5722", "2360", "5617", "3847", "5853", "9923", "4055", "1154", "5923", "8015", "7922", "3366", "1353", "3942", "6481", "7926", "2180", "1654", "9302", "6713", "6563", "5920", "8796", "9269", "2401", "3021", "8088", "8370", "9002", "7372", "9069", "4181", "353", "4323", "1956", "9714", "4804", "7841", "1847", "4859", "4434", "9782", "3822", "2616", "541", "6309", "1582", "8879", "9371", "3396", "2181", "5431", "9556", "3790", "9256", "9211", "5807", "7191", "6459", "6424", "2230", "2499", "7655", "6559", "5573", "883", "6465", "4312", "7876", "3925", "2755", "5637", "9719", "7004", "5016", "1543", "575", "1733", "7558", "4571", "3263", "4483", "5291", "4454", "3438", "789", "8847", "8877", "3260", "1810", "317", "4965", "6177", "6887", "4753", "5724", "7164", "3598", "8186", "6555", "1269", "6715", "4231", "1970", "3311", "9649", "5170", "1971", "6267", "8801", "3099", "7403", "4494", "6293", "6825", "5575", "4032", "2126", "5510", "5912", "7963", "4137", "3258", "117", "6998", "3829", "4651", "7615", "6535", "8616", "2198", "394", "5619", "9418", "1641", "2303", "4366", "1699", "6561", "9459", "1706", "5538", "4994", "1168", "5095", "5417", "736", "4276", "327", "20", "8899", "1101", "2347", "4004", "1743", "4294", "1807", "671", "2909", "7587", "3395", "4871", "8740", "7361", "2487", "3074", "7209", "6385", "9001", "7080", "2586", "7168", "790", "4734", "1151", "2914", "3358", "5405", "1737", "275", "9728", "3872", "4896", "7591", "717", "2544", "2646", "8887", "6301", "7189", "1533", "6356", "9841", "6141", "2902", "5544", "6228", "5574", "1179", "3243", "2580", "1380", "9249", "7846", "8495", "3736", "607", "3085", "6993", "6405", "7391", "5629", "3495", "1606", "34", "3953", "2654", "4750", "4466", "1846", "435", "3147", "2123", "7427", "8389", "5482", "7984", "870", "5823", "8201", "8944", "2576", "7206", "9960", "8110", "2986", "762", "5899", "8323", "4851", "7888", "8272", "5590", "6154", "919", "6156", "6285", "9815", "841", "3259", "794", "6711", "7260", "9615", "6123", "4524", "7529", "767", "4142", "1646", "7859", "630", "9765", "7411", "3417", "4471", "3739", "7879", "4356", "3273", "1845", "5230", "5693", "2938", "4553", "5671", "4305", "2996", "9699", "7205", "329", "2427", "934", "5171", "5317", "2566", "8912", "1484", "8470", "8661", "8271", "5645", "7812", "114", "6617", "1275", "3135", "9555", "5887", "9680", "9943", "796", "3145", "2570", "3962", "2717", "5747", "6512", "6869", "1417", "2619", "9024", "5059", "6367", "3926", "6556", "5596", "9738", "6558", "4224", "6362", "832", "2269", "9784", "6911", "8410", "713", "3607", "4354", "8076", "1780", "7441", "1425", "5456", "1379", "5468", "5803", "7616", "2392", "8450", "4694", "1616", "5003", "5625", "8914", "6543", "4686", "5643", "5699", "805", "3701", "3399", "3397", "4058", "7677", "8228", "9300", "3178", "624", "6161", "4017", "7118", "1645", "1506", "9709", "2733", "9795", "5682", "3577", "7510", "7016", "5742", "237", "5411", "5652", "110", "7464", "8511", "9646", "1668", "1113", "2667", "8025", "9881", "4900", "930", "1109", "4430", "6270", "7806", "7081", "7154", "2301", "6342", "5452", "7688", "7578", "5490", "3671", "8211", "198", "8266", "1995", "9832", "7565", "6408", "2555", "2993", "5382", "5869", "7828", "3030", "250", "4646", "8692", "8449", "4944", "3095", "3043", "681", "6652", "7405", "6468", "1501", "6571", "9475", "1747", "8842", "2497", "2742", "5139", "2179", "1202", "7290", "9116", "5676", "395", "8018", "7654", "969", "5570", "9633", "1225", "6700", "9893", "8427", "2960", "5905", "6485", "2342", "7422", "6861", "5599", "4", "5146", "8198", "5344", "7694", "1377", "911", "9897", "4792", "6798", "1555", "4513", "2552", "1033", "9848", "1436", "4100", "3537", "4919", "9901", "9131", "411", "1818", "4697", "9581", "1140", "4543", "428", "9219", "5054", "8191", "9045", "9926", "3924", "5114", "659", "2778", "6416", "5765", "1726", "5717", "5727", "5141", "1472", "4534", "2917", "4169", "4503", "9970", "1041", "7747", "971", "6333", "3960", "3576", "515", "7685", "8777", "1612", "6921", "4093", "5347", "2059", "7592", "6268", "7053", "2621", "9244", "4265", "9631", "4180", "2045", "7166", "6430", "3657", "9038", "210", "1575", "6128", "2876", "4807", "238", "4384", "1274", "2913", "4892", "8176", "9463", "3369", "5872", "3603", "5786", "9034", "8963", "833", "898", "4918", "7179", "1928", "3934", "615", "4608", "3184", "7109", "4711", "6863", "9617", "5429", "3193", "7258", "8032", "4607", "4639", "817", "4663", "4799", "7697", "1047", "2421", "8990", "9749", "5263", "2192", "8671", "483", "2933", "7602", "2595", "2832", "7705", "8657", "7106", "6773", "9285", "7280", "5445", "3477", "7695", "1177", "6238", "1963", "6794", "5475", "5577", "983", "577", "3867", "5818", "5184", "2300", "8909", "8457", "576", "2489", "1217", "4897", "1657", "104", "8131", "8159", "4771", "5802", "2869", "1363", "1817", "8010", "2500", "7162", "1791", "9867", "5491", "6347", "6038", "2613", "5977", "5524", "5813", "976", "5871", "1593", "3256", "5068", "9805", "5113", "9772", "3078", "1065", "2190", "7455", "3364", "6245", "8113", "9351", "6192", "1145", "1020", "6169", "7577", "3908", "7819", "1356", "4683", "8901", "4353", "2685", "1165", "6796", "7518", "3972", "1628", "4758", "2013", "1205", "4890", "7369", "1652", "8452", "9590", "8971", "7012", "414", "6344", "6287", "9142", "4066", "534", "4863", "6393", "2276", "9296", "8325", "698", "3039", "9290", "8936", "1134", "8574", "2446", "339", "3080", "8134", "4280", "5581", "76", "7159", "3687", "7433", "3961", "3113", "158", "7291", "7868", "2692", "4421", "8135", "4214", "8889", "9858", "2266", "8620", "4199", "8505", "7658", "476", "9822", "4488", "5159", "6828", "5633", "5386", "7123", "718", "6884", "4335", "8116", "1674", "5437", "2688", "6769", "9092", "7543", "627", "5555", "3107", "1673", "4880", "6871", "3543", "2817", "4552", "9307", "3331", "1280", "7250", "6359", "8057", "5665", "9994", "3936", "9448", "4802", "5172", "4574", "36", "424", "1609", "3840", "7548", "778", "8880", "1000", "1762", "8669", "141", "5892", "6165", "6492", "2640", "618", "4878", "8129", "7223", "8961", "5826", "2461", "39", "4654", "8123", "3540", "6222", "69", "314", "218", "3478", "279", "4287", "3000", "3390", "105", "4390", "7567", "5507", "8591", "8414", "8060", "9357", "4229", "1891", "1639", "1886", "8145", "5883", "7440", "958", "1085", "3450", "5968", "4135", "7507", "9961", "4832", "6691", "793", "2930", "4131", "9965", "6544", "8210", "3676", "4713", "166", "5081", "1590", "2137", "3229", "5301", "8189", "2767", "9831", "1098", "1551", "8222", "2141", "74", "9666", "3047", "8383", "6043", "526", "6500", "3277", "4039", "9313", "4953", "3992", "3486", "7146", "3583", "5393", "5260", "7894", "5014", "3778", "9112", "7834", "784", "6865", "4329", "912", "1410", "7195", "3852", "2093", "8820", "9408", "8586", "8064", "9250", "6392", "510", "6227", "7221", "3142", "7059", "8081", "9754", "8315", "2322", "4447", "638", "232", "7415", "1106", "3802", "23", "4508", "3902", "9205", "432", "4568", "4025", "8626", "9890", "8706", "4519", "1716", "5032", "1722", "4191", "7796", "7637", "6547", "5702", "4069", "7085", "3106", "9880", "6900", "3201", "8995", "6946", "544", "2184", "3415", "6426", "4481", "554", "5436", "4183", "4187", "2110", "2432", "5623", "8028", "8542", "5635", "6291", "7365", "299", "8554", "1626", "568", "7920", "6472", "8104", "6782", "749", "821", "9207", "5038", "7605", "2495", "9813", "703", "6021", "6849", "6425", "5308", "3634", "7108", "6063", "3692", "8125", "5387", "6786", "4441", "1318", "1905", "81", "4490", "6603", "1819", "3168", "6403", "5487", "9177", "7039", "7511", "9378", "3786", "4094", "6014", "4611", "1176", "9091", "5118", "5011", "8798", "5255", "6098", "230", "583", "9845", "4634", "3247", "9346", "253", "5276", "858", "9232", "465", "7746", "5632", "3876", "2124", "9902", "968", "5023", "656", "6593", "8895", "2111", "7690", "9538", "3083", "7187", "4220", "3110", "2687", "5158", "1022", "4079", "2599", "3941", "7007", "1745", "5458", "5501", "2364", "7469", "904", "763", "5956", "3542", "1779", "217", "8041", "1959", "8118", "427", "8911", "8252", "9710", "112", "750", "1618", "8058", "979", "4709", "5246", "6632", "9684", "3131", "7408", "3475", "393", "224", "3474", "1693", "4315", "3267", "2029", "9101", "397", "8137", "3393", "7144", "241", "1272", "8219", "9596", "2905", "1839", "962", "6026", "7292", "830", "184", "2008", "9639", "1498", "1265", "6250", "4926", "3209", "6201", "525", "2445", "3012", "2171", "7496", "9537", "9494", "2727", "4448", "5075", "8448", "9338", "9410", "4617", "9059", "7582", "9874", "8929", "9632", "9920", "5958", "5039", "1440", "7315", "450", "1285", "5398", "5485", "5529", "3898", "5050", "9980", "2781", "6855", "7943", "3818", "6108", "4833", "1713", "4786", "422", "1516", "3709", "9974", "2245", "9133", "5233", "6491", "3149", "7849", "8713", "5447", "1193", "1007", "1957", "9305", "4281", "1688", "5053", "8981", "4458", "6251", "4163", "6008", "9097", "5080", "1259", "5236", "1146", "7147", "4796", "44", "1573", "6130", "4464", "3544", "8439", "6002", "4958", "6189", "7257", "173", "7169", "1304", "9564", "5013", "9904", "3560", "147", "6442", "7976", "743", "6867", "5160", "8072", "4599", "977", "4830", "1293", "8676", "9569", "2830", "3244", "91", "1989", "458", "7453", "6519", "6513", "3367", "1301", "2609", "4485", "9718", "9190", "6159", "4326", "2011", "2415", "7289", "3885", "3656", "7531", "7350", "2454", "5197", "8566", "3291", "6943", "844", "5718", "7877", "5843", "8441", "6944", "1757", "8082", "5332", "7889", "5193", "7042", "9268", "8684", "4386", "6019", "2878", "6204", "3161", "3320", "1698", "9591", "8521", "3745", "6802", "7090", "6610", "9330", "3530", "3144", "5847", "2819", "4731", "590", "6613", "8773", "5432", "9424", "3349", "6417", "7224", "6732", "9281", "884", "4923", "7026", "2032", "1210", "3087", "9332", "8359", "1718", "354", "5031", "4597", "4070", "6546", "1372", "1491", "8119", "7848", "159", "8026", "6864", "7670", "8826", "7214", "3565", "6699", "5015", "8678", "7867", "478", "7628", "3056", "1854", "6661", "3357", "8581", "4013", "9502", "5739", "6062", "2740", "8424", "97", "7802", "5269", "6821", "8168", "5224", "2747", "9421", "88", "6170", "6956", "3675", "7988", "2412", "9610", "1029", "734", "68", "2381", "2821", "538", "1408", "9328", "1503", "3346", "5078", "8749", "4272", "7429", "1149", "2543", "22", "8085", "5542", "8970", "7172", "5124", "2916", "7458", "1483", "8677", "8190", "8463", "7151", "5549", "1370", "2440", "7910", "5631", "869", "3335", "7240", "1924", "8039", "3911", "9022", "3407", "7338", "1936", "7018", "7827", "6590", "8656", "8233", "8794", "8122", "7793", "9173", "2857", "251", "3049", "294", "9568", "6612", "994", "7017", "8034", "9272", "6824", "6692", "4304", "4275", "9857", "423", "307", "2310", "2249", "3977", "6633", "4717", "3338", "6883", "2647", "4230", "3287", "2355", "1339", "7517", "1788", "4987", "3245", "5182", "7386", "5065", "3124", "8308", "4967", "9583", "1890", "5753", "7934", "5390", "5024", "3900", "1990", "1476", "4478", "9157", "589", "4677", "3120", "1662", "4195", "1601", "3640", "6974", "4970", "6336", "3566", "1700", "3870", "6346", "3831", "0", "2974", "8762", "7911", "2309", "6365", "5678", "1183", "4074", "2039", "3683", "4883", "8705", "1242", "7630", "2485", "9999", "1967", "1182", "9876", "8091", "3148", "5841", "840", "8665", "1943", "4888", "2209", "1664", "6741", "2807", "3770", "8170", "195", "8378", "4102", "2403", "4895", "7255", "672", "5010", "6166", "6795", "4364", "9997", "3182", "6985", "1613", "9586", "7323", "4705", "8367", "9936", "1200", "6135", "9928", "3373", "6077", "9651", "755", "7068", "6598", "7770", "9155", "3472", "6948", "1048", "1584", "1527", "9179", "6282", "1564", "4456", "9042", "4907", "3231", "9660", "9135", "6120", "3713", "4898", "8856", "2076", "3008", "3907", "7710", "9483", "143", "9842", "3469", "4270", "291", "1253", "1884", "6868", "1256", "5974", "315", "7861", "247", "1027", "4755", "1916", "1619", "1163", "3169", "4949", "6264", "959", "2991", "8361", "1709", "4188", "7449", "4706", "9445", "4016", "4360", "6497", "4380", "9870", "8685", "2763", "9578", "6996", "5661", "4917", "9601", "3059", "1009", "90", "1764", "1196", "5614", "8565", "3288", "1549", "1405", "3647", "1311", "8622", "9258", "1123", "2481", "6368", "9796", "6826", "5585", "7269", "1054", "54", "2669", "8812", "8680", "7709", "4452", "8916", "558", "5688", "6989", "2779", "7923", "8519", "6889", "1231", "4870", "2291", "2297", "9764", "6506", "2907", "5479", "5672", "3552", "7067", "1825", "7104", "2007", "3219", "6389", "7228", "484", "5321", "9225", "1271", "3363", "6431", "3128", "4975", "1263", "8978", "2081", "8555", "3309", "1341", "5066", "9122", "2490", "1081", "2550", "3957", "4794", "2776", "8533", "620", "3969", "2259", "7210", "1856", "4120", "2988", "4517", "2325", "1083", "4947", "7102", "988", "4920", "4885", "1262", "128", "4404", "7301", "3947", "2529", "5806", "8682", "5122", "503", "6872", "5320", "19", "4788", "3094", "7200", "6716", "5833", "1488", "4737", "5060", "5381", "3295", "9287", "4477", "3702", "8056", "6214", "5825", "7970", "5741", "2861", "2019", "6332", "8702", "5740", "8726", "3068", "9315", "2422", "1115", "742", "5051", "9753", "1681", "8573", "6888", "7927", "1973", "7385", "3945", "7126", "1116", "719", "9088", "7024", "8242", "6314", "9098", "1167", "2043", "404", "2898", "8563", "4840", "9277", "5660", "5366", "7807", "2265", "3845", "451", "7044", "8167", "1960", "3119", "9233", "3662", "1399", "4310", "1317", "4014", "3614", "8996", "7500", "7609", "1376", "2459", "9675", "2069", "7148", "2017", "1532", "2166", "5965", "8972", "8943", "5044", "4972", "8215", "551", "967", "2915", "7766", "5187", "5903", "7099", "5133", "7313", "3660", "7858", "439", "337", "3391", "3899", "45", "894", "9785", "3849", "3714", "2715", "2158", "684", "364", "8096", "5136", "5848", "8784", "5860", "4670", "6252", "4873", "4671", "1448", "6898", "7842", "8166", "4991", "5862", "5683", "1508", "67", "7940", "3405", "8688", "1366", "2214", "2697", "2294", "2420", "1434", "605", "9053", "5952", "4656", "9435", "9941", "2091", "9609", "9786", "4238", "6787", "8849", "606", "4086", "4549", "8766", "2696", "4383", "1812", "1287", "6381", "8631", "5941", "4468", "7909", "8114", "5297", "986", "3155", "4472", "8038", "2989", "6284", "4756", "6415", "9707", "6581", "1361", "1766", "9669", "3248", "3162", "3440", "8717", "9814", "6152", "7132", "7745", "6833", "3837", "1838", "4974", "2528", "2786", "2805", "4397", "1765", "9214", "8098", "2494", "7789", "8531", "2997", "27", "944", "795", "9565", "6434", "8983", "7481", "9242", "5092", "6273", "447", "5356", "7000", "8814", "8376", "7093", "8596", "7267", "480", "8528", "2769", "4935", "4775", "3011", "9406", "8226", "2541", "8941", "8594", "9571", "4810", "1126", "3920", "8501", "901", "6976", "2513", "3428", "3575", "8683", "2396", "6174", "1334", "402", "5179", "9806", "940", "4800", "8061", "1320", "257", "1487", "4905", "7303", "3453", "1268", "9304", "7446", "6905", "1927", "6121", "6331", "875", "4658", "1286", "2504", "6286", "8348", "2568", "2558", "2436", "3225", "4620", "6526", "511", "8353", "52", "7439", "7643", "3019", "7919", "7563", "7046", "3890", "3939", "7463", "4500", "9159", "5177", "4691", "811", "7379", "1497", "5099", "3207", "3569", "6147", "1955", "4087", "9270", "5649", "6697", "654", "9036", "2189", "8546", "8468", "2159", "1034", "5238", "8516", "7387", "9368", "3978", "6460", "4479", "829", "3545", "5242", "6106", "474", "6482", "4916", "6448", "1983", "5978", "3986", "3002", "7282", "2860", "2486", "5448", "7854", "4104", "4428", "3820", "4359", "9513", "3643", "4178", "4718", "2738", "3525", "6042", "5442", "1306", "905", "1213", "3632", "9193", "3109", "7534", "7406", "3483", "2197", "2173", "2148", "540", "1450", "1024", "2760", "1653", "6207", "7872", "1975", "1273", "7818", "161", "2964", "9278", "6719", "4745", "2201", "9687", "3329", "164", "470", "3307", "6441", "3152", "2321", "3757", "4685", "1169", "3324", "8183", "3319", "6589", "3197", "5096", "5488", "2075", "9474", "8043", "5894", "2901", "5897", "6091", "1442", "4759", "3932", "8582", "2052", "2598", "6297", "7006", "7896", "4443", "3098", "1105", "2362", "6363", "4703", "3769", "2911", "9825", "120", "7712", "9292", "2256", "1055", "3616", "9075", "9604", "2811", "1711", "7580", "2759", "8748", "8986", "3834", "6528", "9919", "3672", "1071", "340", "3550", "1648", "286", "6144", "3586", "2699", "8444", "2255", "7566", "6750", "6638", "7485", "886", "4861", "2631", "9188", "9108", "6890", "7368", "9503", "7506", "5559", "4402", "2792", "5836", "4993", "5057", "2493", "2665", "9454", "3159", "9253", "1393", "7932", "2612", "3227", "4911", "5058", "4162", "1951", "8264", "438", "1236", "7014", "2106", "2160", "6028", "1158", "1162", "1622", "966", "1067", "3427", "3651", "1038", "9011", "8602", "8212", "7069", "7281", "7641", "3636", "1478", "2579", "2666", "5082", "1553", "2060", "3659", "4997", "7898", "4687", "6155", "358", "7645", "2480", "2912", "4838", "8863", "5103", "5743", "613", "7684", "9107", "1851", "302", "7329", "8405", "9381", "6319", "7460", "9125", "9837", "8164", "2591", "812", "3666", "2200", "3375", "7388", "5109", "9851", "3317", "258", "4674", "4171", "9457", "17", "7306", "876", "7140", "8730", "3732", "9676", "751", "2946", "5117", "2633", "8830", "5101", "5086", "473", "4412", "3398", "416", "601", "5997", "7434", "7704", "5832", "4551", "316", "8229", "5144", "5207", "4226", "8", "2121", "6237", "7958", "5865", "2874", "9134", "7579", "7395", "8273", "5879", "6294", "49", "4547", "7174", "3327", "4609", "8946", "8316", "9982", "4227", "2254", "9835", "7844", "8356", "535", "4114", "2337", "4037", "9086", "931", "1748", "5796", "3617", "187", "8997", "2871", "2885", "5362", "5545", "8493", "4657", "4396", "3884", "3605", "4684", "913", "2140", "9008", "1181", "7549", "8254", "7691", "2063", "3650", "8319", "2056", "4721", "6671", "2005", "1219", "8652", "9119", "9335", "1228", "9794", "3088", "4541", "6735", "500", "8506", "9267", "9579", "2335", "2028", "78", "1012", "6973", "7416", "7438", "1869", "4222", "9541", "5594", "3613", "9481", "5523", "2417", "4461", "2657", "2225", "6396", "1898", "770", "1749", "8453", "8033", "8805", "9701", "2726", "2714", "1197", "7576", "9230", "5685", "5258", "8991", "5583", "1189", "8770", "7767", "7570", "6874", "4704", "3202", "5239", "9467", "1011", "8393", "9670", "1599", "5472", "4660", "5268", "528", "5329", "4459", "5512", "2700", "8335", "5289", "6651", "7354", "5757", "7294", "4161", "660", "4154", "4603", "9409", "7528", "5378", "9025", "8994", "1010", "7936", "3239", "5576", "6131", "5497", "2262", "5140", "8362", "2972", "708", "7870", "5893", "5715", "5005", "5988", "1994", "7128", "359", "3305", "6806", "1211", "9696", "5346", "7652", "9491", "2330", "3943", "4538", "2466", "8761", "2708", "79", "7590", "1235", "7158", "5713", "5270", "6097", "692", "3975", "2375", "9628", "9319", "8299", "3499", "127", "190", "7871", "7176", "7033", "5697", "773", "1435", "6801", "4548", "4827", "7817", "6615", "4537", "6566", "3082", "1243", "4783", "5515", "3189", "215", "4803", "3111", "6307", "259", "1853", "8736", "5218", "1238", "765", "2469", "896", "6073", "5587", "3652", "1198", "508", "867", "3637", "8466", "8126", "8924", "7437", "9864", "351", "2894", "2479", "8188", "296", "2936", "7247", "8556", "4630", "9016", "1542", "1389", "4931", "7087", "7869", "8494", "1895", "4853", "2086", "1475", "3944", "1638", "2590", "8111", "7755", "9913", "7698", "5609", "5593", "6620", "2402", "8534", "3382", "5514", "4148", "7753", "8885", "4746", "1119", "801", "4228", "2472", "4757", "5835", "7008", "5348", "9183", "2272", "5921", "163", "3747", "3125", "4376", "9404", "553", "2344", "2520", "8360", "9724", "5421", "3553", "996", "6662", "1651", "1122", "950", "7978", "5875", "2491", "6640", "2023", "3341", "7212", "8966", "2457", "4836", "9218", "689", "7809", "7672", "7644", "5648", "2424", "7937", "965", "2867", "169", "1030", "7428", "2107", "8013", "3716", "4933", "2824", "3850", "2804", "6852", "1870", "8103", "3886", "7723", "6529", "1423", "12", "6205", "3003", "2766", "8608", "1104", "7585", "3414", "918", "680", "2027", "168", "8714", "3585", "9240", "699", "7607", "9523", "4533", "9940", "6195", "9803", "7707", "429", "5771", "6029", "3694", "2492", "8589", "9561", "8489", "6312", "4936", "3014", "4269", "7502", "132", "2150", "3062", "8933", "2318", "3524", "5469", "2561", "1095", "4530", "9580", "2204", "4117", "5810", "5960", "3257", "4497", "8739", "3873", "8078", "7569", "6055", "2035", "4288", "936", "2522", "2594", "2002", "3654", "4570", "9731", "5027", "420", "6866", "1421", "7447", "6136", "1319", "9759", "8196", "1226", "2350", "6984", "4829", "8615", "7895", "9262", "4726", "1427", "1108", "6429", "852", "9688", "1734", "3891", "1486", "9776", "4593", "5029", "1879", "6457", "5498", "887", "7015", "6600", "7994", "808", "3022", "66", "7682", "8109", "1199", "3362", "3077", "1078", "4205", "8314", "1221", "7364", "5553", "1536", "8897", "8872", "4382", "6163", "7627", "4248", "7622", "462", "4856", "6236", "7542", "9426", "7660", "2682", "243", "9373", "3222", "1014", "2452", "8311", "144", "2215", "7996", "2957", "8993", "9141", "9798", "8605", "6885", "7233", "4664", "8707", "5993", "3730", "4129", "2507", "1823", "9505", "7389", "6540", "1909", "4401", "3995", "9894", "7177", "1178", "8790", "4661", "4940", "7623", "7790", "6337", "8230", "8903", "6977", "6103", "1624", "4817", "5077", "4943", "7756", "2257", "6964", "8342", "4406", "4410", "3777", "1457", "4457", "4504", "5221", "7370", "1090", "8649", "7253", "9355", "2169", "744", "3630", "9195", "1074", "1308", "3976", "6609", "1961", "1040", "9172", "4662", "4487", "2926", "4179", "7972", "8618", "7764", "6265", "4429", "6918", "5505", "2175", "5106", "4097", "4495", "9352", "2639", "5178", "9915", "7515", "8968", "1325", "760", "9916", "5689", "1649", "9608", "5915", "4388", "3394", "8804", "2142", "6915", "8597", "3413", "2732", "309", "4610", "6808", "7965", "2371", "2858", "7426", "5521", "5183", "2130", "8130", "7112", "18", "8485", "6951", "6351", "9055", "4969", "3237", "8560", "2752", "5252", "6075", "9909", "57", "650", "1299", "8848", "9704", "1815", "7122", "8071", "4648", "9829", "7903", "5150", "6139", "8303", "700", "8475", "3253", "4675", "6965", "8870", "9535", "7731", "4978", "2839", "9056", "165", "6276", "1816", "9469", "4040", "4160", "1893", "4101", "3296", "8066", "707", "7862", "7734", "8040", "602", "8426", "7699", "4411", "7571", "9734", "9882", "7957", "3038", "3568", "3557", "4813", "7180", "2701", "5686", "2400", "3548", "5591", "711", "5216", "6896", "6744", "2637", "1514", "3051", "116", "5262", "9592", "6183", "3342", "330", "6693", "9026", "301", "4212", "5950", "3195", "3127", "4427", "8221", "4499", "3188", "4068", "3425", "3993", "3228", "124", "9548", "506", "9029", "1415", "3278", "2068", "4261", "3299", "8021", "2313", "2031", "4259", "6577", "2239", "6789", "7394", "2605", "4374", "5457", "5422", "5566", "3906", "6933", "1947", "9661", "2823", "6230", "5428", "669", "7050", "2900", "3203", "13", "1855", "2315", "860", "7229", "1345", "8795", "95", "2736", "5100", "1690", "6013", "6847", "5361", "3639", "4580", "274", "5264", "5331", "14", "1972", "4147", "374", "8768", "3020", "8177", "4333", "3180", "6858", "4781", "2774", "1585", "2092", "6421", "4749", "9846", "2102", "5091", "4649", "7175", "6466", "5322", "3365", "6104", "8765", "2710", "3859", "8525", "8336", "4779", "188", "2062", "3462", "9202", "2731", "5443", "5639", "4076", "6626", "2990", "7601", "4139", "1934", "4569", "448", "3775", "9215", "6618", "4763", "2512", "945", "4314", "8075", "6495", "9852", "1900", "9019", "455", "8133", "2146", "6050", "1671", "3563", "8578", "7254", "7560", "2308", "9779", "5503", "7741", "388", "9626", "3693", "4110", "9828", "2741", "6180", "2178", "1558", "9081", "8029", "214", "5944", "644", "8269", "6280", "2875", "8792", "9911", "6986", "5561", "9323", "9341", "7550", "8388", "9264", "7466", "4317", "777", "9084", "4998", "2952", "1382", "9540", "9656", "6507", "3810", "9030", "4930", "4886", "5785", "2074", "4358", "6341", "2606", "7218", "4051", "3767", "9668", "9411", "6478", "1741", "8627", "3515", "6846", "5610", "464", "9238", "2077", "7225", "2026", "3658", "2248", "8313", "5061", "2477", "1993", "961", "6244", "8006", "633", "4822", "328", "8030", "181", "5937", "3857", "437", "6687", "9691", "1129", "4052", "82", "8617", "7178", "7913", "863", "3183", "3646", "8869", "8465", "5772", "1938", "6167", "8710", "3460", "8694", "220", "6628", "1479", "3136", "8318", "9990", "2267", "318", "9976", "4563", "7574", "4136", "8630", "4138", "4595", "9964", "5995", "8668", "9636", "2164", "1400", "8529", "1814", "6606", "3901", "6627", "377", "1246", "3489", "9653", "1992", "3446", "3451", "2506", "9301", "593", "4060", "3389", "3784", "4268", "8411", "7181", "4689", "2172", "8759", "319", "2343", "4173", "5592", "1517", "8767", "3574", "929", "498", "517", "2468", "9895", "759", "6259", "4831", "4526", "71", "623", "3121", "1061", "6032", "6471", "737", "4433", "8270", "7916", "501", "1586", "3476", "9623", "7647", "9767", "516", "5839", "8214", "7359", "3160", "1642", "3240", "9861", "7351", "5759", "8758", "1771", "9140", "9021", "4172", "7778", "8979", "2470", "5373", "7981", "8660", "3302", "7480", "1981", "4378", "9476", "8307", "2168", "5352", "4140", "7145", "2944", "5723", "7314", "7285", "3130", "592", "9865", "2049", "3067", "6168", "1093", "7237", "7664", "8526", "2787", "7521", "670", "1811", "3597", "5646", "9983", "6114", "5856", "5987", "1249", "6930", "3546", "5799", "8927", "9360", "1338", "3076", "5703", "2174", "7400", "587", "1260", "7278", "1640", "7702", "2870", "3455", "5509", "7215", "6702", "7960", "6277", "5519", "6557", "5049", "1761", "5504", "7840", "1291", "6339", "347", "3527", "5232", "4239", "444", "3137", "9380", "9725", "1046", "154", "2641", "2608", "4092", "4022", "2296", "910", "9616", "1100", "9900", "8324", "8049", "115", "7738", "9345", "8653", "5217", "5674", "8584", "6501", "9647", "2713", "8951", "3959", "580", "2630", "323", "5959", "8063", "2866", "9280", "2542", "7156", "4904", "6373", "9402", "7991", "7815", "8197", "7917", "3322", "6034", "2114", "4842", "8154", "2070", "7823", "8207", "5638", "4590", "6541", "419", "6243", "5579", "9871", "7883", "603", "4059", "9350", "1502", "4952", "2702", "628", "715", "6669", "9361", "8275", "4475", "140", "3154", "9732", "8406", "6902", "5203", "4688", "9530", "7961", "8918", "6257", "2408", "1439", "1634", "6760", "9746", "9100", "4020", "6096", "7461", "809", "9780", "888", "6938", "8745", "6435", "6505", "8807", "7184", "4067", "4035", "1043", "3200", "4710", "8673", "7522", "2361", "3507", "185", "7624", "4251", "6079", "5192", "981", "7477", "5707", "5030", "6182", "8068", "8420", "8446", "6820", "9064", "2463", "1954", "2048", "77", "7516", "1782", "2795", "7626", "2711", "682", "7533", "9595", "9939", "9673", "8861", "6261", "9169", "442", "7676", "31", "9223", "6962", "3804", "954", "9325", "4740", "7663", "3892", "6486", "9869", "8193", "7725", "5345", "2788", "637", "3915", "1252", "7929", "6516", "16", "8469", "3297", "2748", "4818", "9951", "9013", "4605", "3531", "1056", "4024", "7262", "9527", "8815", "6493", "7598", "8576", "7231", "7912", "9790", "9294", "3927", "8377", "254", "2113", "6624", "7519", "8157", "9773", "9797", "9781", "6817", "1566", "4773", "5423", "1629", "64", "1430", "6775", "6572", "2496", "5403", "4111", "4398", "9750", "5651", "6673", "6509", "8557", "6220", "4217", "8413", "7999", "3742", "6891", "5768", "3340", "5493", "4099", "8083", "3618", "7814", "2397", "7251", "7727", "866", "6647", "3685", "5541", "6067", "906", "7252", "7034", "9947", "8715", "3633", "4422", "1541", "1620", "1035", "7486", "1482", "8150", "8002", "2207", "459", "7719", "8810", "1872", "2743", "9854", "8330", "679", "6292", "5793", "4036", "8240", "7759", "5933", "2753", "3655", "522", "4960", "5335", "878", "4046", "4121", "8147", "597", "1937", "4319", "7952", "3308", "5476", "4588", "7666", "2373", "4752", "9333", "3337", "3740", "6197", "7881", "206", "1298", "3731", "1185", "8607", "2050", "3710", "6047", "3894", "5779", "5046", "3419", "255", "1750", "391", "8695", "6992", "6910", "7771", "5325", "6695", "6853", "205", "3268", "4201", "5628", "6278", "8824", "9178", "5477", "3134", "8334", "7417", "521", "6157", "843", "3674", "9891", "4679", "6039", "4602", "4469", "6562", "9050", "8003", "2939", "7430", "9667", "581", "2157", "3464", "269", "1804", "5363", "7077", "8507", "1355", "4395", "9347", "1125", "4400", "8737", "6648", "7782", "9168", "8202", "721", "2797", "574", "5898", "1131", "2831", "9573", "5866", "8952", "1858", "5017", "2149", "5334", "4141", "2515", "5851", "696", "3100", "7047", "8023", "4894", "355", "1471", "226", "4405", "2306", "1808", "786", "9629", "8268", "5298", "1309", "1362", "3064", "2897", "468", "6569", "2409", "4681", "5426", "7342", "4165", "2356", "4207", "2533", "9222", "8838", "1730", "1931", "9611", "5257", "2935", "3762", "6076", "1485", "1896", "4912", "4988", "5698", "2573", "1388", "2087", "9552", "9652", "1907", "1314", "4624", "7266", "6255", "5154", "5253", "1220", "5982", "2969", "7248", "3904", "2332", "7833", "4594", "9000", "2331", "2236", "2634", "9288", "6851", "2919", "2922", "3306", "4743", "7897", "1715", "4673", "9582", "2072", "4623", "2112", "787", "2864", "903", "6623", "6912", "518", "9339", "7194", "8223", "9165", "2931", "8331", "8570", "5684", "9910", "550", "5754", "6767", "8181", "4592", "4562", "985", "1996", "8462", "3403", "825", "8780", "537", "6511", "2340", "53", "8358", "5597", "5895", "7013", "8149", "1754", "5711", "2314", "5992", "2813", "6953", "5453", "3966", "4860", "9809", "7982", "9104", "9103", "5953", "9918", "9720", "6040", "4257", "1305", "8297", "6931", "3950", "2932", "6187", "8105", "3821", "6583", "1489", "1964", "5885", "3811", "7274", "3725", "9587", "4695", "8722", "5532", "820", "9622", "2751", "9153", "1017", "5608", "588", "3490", "2241", "9200", "8822", "4576", "4554", "5946", "1678", "8878", "3479", "6664", "4720", "7308", "7687", "9896", "6904", "7078", "2980", "3768", "4968", "2465", "6053", "3325", "9340", "8344", "1365", "9484", "1469", "6685", "7661", "9473", "2656", "5957", "879", "5394", "6290", "9054", "4992", "8481", "2442", "6253", "5973", "3105", "5148", "9331", "348", "9048", "5792", "1915", "1374", "999", "3669", "4098", "5861", "6908", "9132", "5701", "6762", "5460", "1237", "9542", "6242", "5245", "5213", "567", "4957", "7101", "5927", "4377", "8289", "2451", "4015", "935", "5522", "5435", "8300", "3032", "1120", "1644", "4159", "7028", "5618", "2510", "9396", "5097", "9466", "9605", "4091", "1908", "6539", "7618", "1617", "8614", "8629", "1729", "8697", "5155", "2186", "8262", "4255", "1545", "7540", "1802", "9657", "9217", "9061", "6202", "3449", "3334", "6119", "9948", "4449", "2458", "647", "7073", "2822", "4696", "5557", "1493", "695", "463", "5162", "9147", "8094", "2855", "7019", "6148", "9584", "889", "6422", "8115", "3096", "6200", "5280", "4632", "1375", "5341", "3503", "8514", "5367", "1860", "2116", "3116", "7043", "9760", "9181", "5908", "4761", "5041", "3606", "5986", "6324", "4337", "8742", "9166", "4784", "3289", "2302", "8095", "8239", "1316", "631", "2251", "932", "4760", "598", "7143", "6433", "3754", "1740", "3025", "4653", "2055", "4444", "1650", "3456", "322", "9550", "6707", "6140", "2444", "4158", "1834", "9487", "697", "4072", "4267", "1245", "6401", "8005", "1082", "9403", "5412", "6980", "4023", "928", "9009", "7838", "3833", "9422", "2761", "9234", "8757", "9276", "4348", "5369", "9988", "7063", "8027", "895", "3298", "9386", "899", "5828", "1790", "1574", "2526", "5002", "1224", "4258", "2556", "5755", "5306", "2531", "1368", "8949", "3471", "5360", "1614", "3360", "5962", "2405", "6391", "7865", "8065", "9547", "842", "2706", "3138", "960", "6882", "3513", "5565", "7805", "3444", "8862", "1467", "6087", "1143", "5890", "2419", "1919", "5654", "8651", "4008", "9074", "2720", "5932", "9507", "3454", "5303", "1897", "3473", "9377", "407", "4351", "9705", "3141", "3869", "494", "4170", "2242", "4379", "9344", "6643", "3013", "1658", "5500", "7554", "7692", "768", "9210", "1347", "1865", "7944", "2482", "7758", "8395", "5636", "987", "5530", "9", "5471", "4857", "5640", "7383", "2177", "5151", "2845", "1127", "7199", "8213", "5486", "3696", "9028", "666", "2663", "7532", "8216", "3265", "4676", "9559", "1944", "8937", "4011", "7683", "5571", "5410", "2359", "2275", "1998", "4474", "94", "1950", "5634", "3058", "2135", "8811", "1803", "2260", "4976", "8087", "3036", "5088", "1446", "5764", "5474", "1720", "7503", "7022", "2109", "4655", "2066", "306", "4198", "9545", "8431", "2090", "5572", "573", "1292", "8479", "781", "9872", "639", "7375", "2671", "848", "8195", "8559", "2103", "7192", "8894", "7360", "3272", "6683", "3562", "8674", "2694", "5288", "3715", "8000", "1418", "5535", "2349", "4640", "5349", "1057", "6772", "5064", "1330", "9804", "7025", "5250", "5695", "8575", "8593", "373", "3749", "5658", "3737", "3179", "943", "752", "9807", "8438", "4742", "9106", "1672", "4283", "5970", "5089", "1026", "7945", "8867", "5710", "5737", "5598", "957", "7304", "8152", "1460", "5231", "7241", "2549", "4962", "6799", "5305", "2211", "984", "4573", "3516", "7706", "4544", "5881", "5198", "4108", "6680", "2293", "963", "2625", "7138", "212", "9648", "9456", "9415", "9208", "5696", "5552", "5379", "4002", "9497", "6747", "914", "7432", "7875", "8301", "1535", "9216", "2850", "7959", "176", "7349", "398", "4075", "8520", "9163", "5582", "8640", "2020", "6611", "1150", "2589", "2410", "9154", "6382", "7208", "5756", "9839", "7995", "8086", "2771", "2895", "8312", "1786", "4511", "1569", "162", "766", "7613", "3494", "8156", "1591", "3157", "4256", "1600", "5147", "7115", "8100", "7137", "2655", "4509", "5019", "6913", "3620", "1518", "9295", "5461", "1831", "3432", "2348", "2746", "3664", "3949", "1391", "1504", "9949", "802", "774", "1524", "5797", "9511", "1583", "2884", "3108", "6714", "4825", "8637", "9416", "7900", "7713", "9478", "9512", "3967", "40", "3832", "4126", "1874", "693", "513", "4979", "6876", "178", "7414", "7775", "7149", "5983", "6017", "2258", "4184", "7971", "5143", "7946", "8747", "3283", "7671", "519", "7843", "1758", "5047", "974", "9562", "7914", "2801", "9989", "2601", "1976", "8132", "5984", "1784", "1455", "7425", "5384", "2775", "5896", "3194", "8956", "7891", "8910", "6994", "5838", "245", "9226", "311", "5020", "6387", "7501", "396", "6134", "8432", "6378", "3254", "4324", "9884", "1563", "9248", "469", "2398", "3985", "7130", "2384", "108", "6967", "3753", "1770", "2617", "2125", "7985", "4206", "9471", "3589", "4362", "7352", "792", "2648", "4560", "2476", "882", "5687", "5578", "8635", "3826", "6777", "6856", "2284", "7983", "1660", "1682", "3027", "2034", "9184", "7133", "8835", "9689", "4365", "3313", "9282", "7621", "9978", "5980", "1130", "1088", "9907", "2244", "2288", "5273", "9014", "9156", "1966", "5225", "6919", "1702", "2061", "5135", "8349", "6218", "7330", "3165", "9811", "55", "3673", "5205", "2518", "7752", "2853", "4151", "6690", "1875", "7163", "880", "8754", "980", "9522", "1276", "2934", "5882", "8106", "728", "3929", "6071", "5767", "9768", "2501", "1794", "9090", "6742", "665", "1437", "2210", "1058", "6388", "1385", "4516", "1787", "5072", "371", "8926", "3226", "2679", "5659", "9574", "7110", "6209", "776", "4572", "3420", "4336", "3129", "9577", "6892", "4699", "1602", "3990", "9789", "7436", "3079"]]} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist.config b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist.config new file mode 100644 index 0000000..44c29d8 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/mnist.config @@ -0,0 +1,13 @@ +{ + "scheduler": ["str", "cos"], + "eta_min" : ["float", "0.0"], + "epochs" : ["int", "50"], + "warmup" : ["int", "0"], + "optim" : ["str", "SGD"], + "LR" : ["float", "0.1"], + "decay" : ["float", "0.0005"], + "momentum" : ["float", "0.9"], + "nesterov" : ["bool", "1"], + "criterion": ["str", "Softmax"], + "batch_size": ["int", "256"] +} diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets-split.txt b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets-split.txt new file mode 100644 index 0000000..9fe861c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets-split.txt @@ -0,0 +1 @@ +{"train": ["int", ["5470", "3295", "4905", "5837", "4853", "3074", "2927", "2081", "4554", "4171", "4888", "4802", "3257", "4932", "558", "468", "2944", "1332", "97", "5759", "2293", "2086", "3518", "2031", "5505", "4948", "1182", "4418", "111", "2149", "4166", "1451", "1673", "2646", "5307", "705", "3374", "2090", "3914", "1413", "1350", "5066", "5212", "3148", "4543", "1970", "943", "1511", "5125", "1203", "2822", "937", "2529", "5970", "3879", "4231", "2104", "2175", "2954", "5919", "4658", "3601", "5774", "418", "2515", "4338", "6227", "5805", "488", "2857", "1600", "3758", "420", "3762", "761", "5880", "1864", "3248", "1520", "796", "2403", "3784", "1009", "635", "2148", "4743", "2263", "299", "3189", "2511", "1870", "1467", "2087", "5118", "4728", "923", "127", "696", "5659", "3949", "6007", "5180", "5817", "4060", "2065", "1190", "5101", "4391", "1780", "2919", "5463", "2766", "3785", "5756", "36", "5860", "344", "868", "3674", "3508", "4984", "3739", "314", "2539", "3145", "5249", "720", "5055", "2648", "676", "3805", "1119", "5967", "814", "599", "4145", "5751", "5521", "3542", "5926", "144", "1898", "4925", "4075", "2962", "2238", "4487", "1766", "5017", "3534", "3401", "463", "3579", "663", "1471", "5796", "2302", "5883", "4329", "5778", "1821", "5742", "6", "1393", "5343", "4406", "3957", "860", "703", "999", "3041", "4400", "1476", "3721", "1307", "4813", "4445", "5409", "674", "5794", "1352", "2902", "1259", "4427", "5995", "5615", "3494", "2685", "2466", "5811", "3848", "916", "4548", "4141", "1293", "2963", "3070", "5625", "5223", "4011", "2726", "3444", "1681", "4700", "5858", "2122", "4398", "2817", "4987", "636", "4721", "5818", "685", "409", "1168", "4557", "3884", "3013", "4268", "293", "2045", "4525", "5730", "92", "839", "3440", "5985", "5681", "4408", "4250", "1670", "2953", "1528", "5303", "1254", "25", "2340", "3889", "4881", "3268", "4111", "3025", "3027", "3997", "2725", "2564", "3701", "1233", "1058", "2152", "6179", "659", "540", "5110", "592", "1154", "3525", "266", "6024", "638", "4381", "1811", "2814", "4529", "2199", "2794", "4249", "3535", "3635", "1191", "6234", "1491", "5677", "5384", "1441", "3612", "1268", "4127", "124", "5953", "4958", "2774", "4819", "821", "5386", "3326", "3180", "4148", "597", "4863", "4074", "1850", "573", "3970", "3794", "3611", "3962", "4794", "5142", "70", "54", "3920", "4118", "3092", "3215", "3091", "4014", "5575", "2169", "4892", "2974", "1628", "3331", "835", "1577", "5389", "1440", "773", "1575", "688", "5666", "2051", "5120", "44", "1704", "3956", "4530", "2231", "4770", "5457", "472", "901", "5261", "1822", "3507", "390", "5515", "5297", "5918", "629", "5693", "3850", "1019", "4327", "4120", "5026", "1938", "4895", "4506", "4624", "979", "3621", "2635", "1668", "2694", "5565", "6213", "115", "2578", "2061", "5595", "990", "1700", "2483", "4002", "5525", "4220", "3050", "5993", "1430", "6272", "5930", "3124", "2889", "1193", "2076", "933", "1599", "3244", "3280", "2385", "5958", "1640", "3436", "3652", "3839", "2206", "1614", "4562", "2958", "1513", "5912", "4670", "2760", "3355", "5861", "1849", "388", "2147", "738", "590", "4045", "3207", "5671", "2421", "227", "5450", "5903", "5029", "1942", "4135", "6202", "283", "5510", "3551", "526", "2830", "3430", "1897", "4540", "4980", "4489", "2159", "1998", "1030", "1038", "3143", "5831", "5700", "3691", "2443", "5834", "6261", "1349", "1696", "4221", "4224", "1286", "5074", "3161", "4542", "2003", "5310", "2353", "4370", "3054", "2304", "4512", "1627", "4366", "3834", "1275", "5552", "1691", "4776", "4649", "737", "5405", "3277", "137", "116", "5488", "3220", "4481", "2769", "464", "3582", "1241", "1358", "3172", "4200", "433", "2956", "5884", "2134", "2803", "3622", "1016", "1195", "1370", "1063", "2729", "5863", "5495", "4595", "4827", "1956", "1664", "6057", "4281", "2575", "1025", "4125", "5037", "5763", "4996", "5452", "3577", "1380", "4898", "1946", "4488", "1969", "4063", "2856", "1622", "3782", "29", "1889", "1887", "1319", "3638", "2341", "2358", "3445", "5094", "6004", "4638", "2790", "1013", "4300", "2038", "989", "5197", "3442", "1610", "1420", "2239", "3883", "159", "914", "5813", "4509", "3361", "2546", "166", "98", "3188", "202", "1947", "415", "1591", "907", "4834", "2367", "2210", "4968", "2492", "5185", "3989", "5999", "5787", "661", "3866", "2359", "2936", "533", "4399", "3425", "3122", "82", "3709", "1959", "6136", "1164", "2226", "6131", "5028", "2557", "3645", "3669", "5155", "2113", "785", "809", "5873", "717", "2510", "5103", "147", "3446", "5732", "442", "1741", "4998", "2029", "2062", "2246", "1076", "690", "6062", "5088", "1853", "2544", "1201", "3323", "306", "2708", "2609", "459", "5679", "985", "4211", "2819", "5702", "527", "3740", "3514", "3297", "2217", "3099", "755", "4692", "4151", "1206", "3648", "5870", "1472", "1554", "683", "4566", "1812", "757", "6170", "1302", "3922", "1156", "6098", "1940", "2850", "3998", "2335", "5444", "1958", "3196", "5451", "5506", "1161", "864", "677", "4337", "2472", "5825", "4493", "4228", "1461", "1125", "4769", "698", "3702", "190", "4126", "4096", "5601", "6183", "4628", "4349", "5327", "3386", "151", "1361", "5215", "4038", "4685", "2964", "3811", "4029", "3906", "4033", "5964", "4947", "5331", "1772", "3472", "3166", "5240", "5098", "992", "2350", "1751", "1605", "300", "5181", "2495", "2015", "5676", "4515", "3861", "5040", "1065", "3135", "4150", "6224", "2080", "5925", "1540", "5388", "2861", "274", "1483", "2185", "2396", "1062", "1978", "2100", "4313", "601", "4590", "1296", "5724", "2596", "72", "4199", "5224", "45", "325", "5899", "4901", "1953", "1903", "2739", "3214", "6109", "1545", "3787", "3926", "2487", "3130", "5042", "1086", "1235", "3917", "6138", "3409", "4927", "243", "5001", "831", "892", "998", "6011", "2221", "2227", "4739", "1636", "5067", "1239", "2281", "3427", "2990", "5537", "5626", "5061", "2489", "1138", "3637", "3617", "26", "6235", "3675", "3324", "2129", "550", "5954", "832", "2566", "2541", "4967", "253", "5853", "225", "1725", "3790", "4640", "2410", "3126", "2257", "3845", "5684", "3349", "3538", "2617", "3605", "2179", "2338", "3217", "3817", "4724", "74", "5871", "3100", "2662", "1059", "2981", "5348", "5847", "375", "357", "621", "3060", "2296", "4706", "6243", "2615", "42", "4876", "4009", "21", "4433", "898", "2016", "4461", "3627", "585", "187", "4142", "4326", "4379", "4401", "4444", "3363", "3987", "4783", "980", "2204", "2606", "4865", "1612", "3424", "5797", "4389", "4716", "3833", "4133", "5056", "2298", "6071", "5781", "780", "3984", "5342", "4080", "4431", "3947", "4662", "3644", "5141", "1899", "4950", "3985", "982", "4851", "4762", "80", "4130", "317", "904", "4866", "647", "3213", "5167", "4653", "3197", "1922", "802", "4701", "3628", "1720", "2327", "2446", "1316", "2659", "2137", "4752", "877", "499", "6049", "2042", "1951", "4791", "4287", "2207", "3405", "5358", "31", "2317", "201", "3813", "5578", "2616", "6002", "73", "4615", "3403", "2422", "632", "354", "4363", "4815", "1265", "939", "2674", "3823", "4880", "410", "4889", "2167", "1641", "4714", "4465", "5572", "3808", "4321", "4594", "362", "3362", "5300", "3774", "1738", "2740", "3314", "3294", "4351", "826", "4255", "4633", "4081", "3066", "5465", "2424", "897", "2570", "4490", "3202", "2952", "3804", "5828", "874", "5403", "5849", "2286", "55", "3413", "4912", "5504", "6043", "691", "3036", "1036", "3102", "3452", "5000", "1072", "3140", "5960", "2225", "657", "942", "4157", "103", "4235", "6029", "1862", "27", "434", "565", "5063", "1633", "6190", "5593", "5865", "4687", "1100", "936", "2827", "2554", "4959", "867", "4340", "2121", "1624", "2130", "1192", "4962", "2940", "126", "2468", "5283", "4858", "4972", "30", "4191", "2885", "3059", "2143", "4521", "234", "4040", "173", "113", "3098", "4718", "5216", "4215", "1395", "1331", "2097", "5974", "5966", "2505", "4003", "2145", "3227", "3742", "2501", "3654", "5165", "2146", "3653", "1536", "2111", "2888", "4513", "1151", "1043", "2567", "6099", "2890", "5337", "2585", "4568", "539", "5630", "1697", "3569", "1021", "5673", "2680", "5211", "5009", "6155", "371", "381", "4829", "5462", "2916", "1745", "2117", "5226", "3418", "3733", "2579", "405", "6026", "4571", "5377", "1051", "1915", "2069", "5560", "5900", "3952", "2628", "4707", "156", "1436", "2862", "4729", "2268", "5425", "3506", "2131", "5603", "3640", "1855", "6199", "370", "5989", "799", "146", "5178", "2759", "2552", "3504", "5720", "754", "947", "6271", "2083", "5100", "4732", "2258", "425", "6005", "3337", "1368", "2054", "1755", "6186", "5020", "1066", "5421", "2088", "6121", "2675", "4143", "4119", "5158", "2319", "218", "1158", "4808", "6111", "4066", "4476", "5675", "2005", "3937", "4223", "5780", "4989", "1635", "200", "2096", "5116", "941", "5400", "1541", "5963", "3152", "4688", "2959", "2677", "1188", "1481", "3936", "3888", "2451", "578", "340", "3259", "1389", "4335", "5201", "191", "3581", "3307", "2821", "4887", "948", "338", "4471", "4360", "1568", "1465", "1570", "3792", "3222", "2299", "1684", "1717", "1364", "580", "5168", "4092", "3626", "6197", "4862", "5516", "5247", "5982", "5317", "700", "5013", "944", "6178", "3744", "2048", "4414", "5186", "1199", "5915", "1762", "2698", "648", "862", "1468", "2623", "4930", "2743", "4267", "3340", "5394", "3613", "3513", "3296", "100", "2997", "5309", "4916", "1163", "1764", "2871", "719", "6228", "4593", "5422", "1226", "1234", "2736", "3316", "6000", "2840", "339", "1310", "3435", "4396", "2809", "5585", "1868", "4833", "2587", "321", "4759", "1522", "4680", "1455", "392", "5461", "5743", "1651", "4173", "5150", "3106", "1715", "4008", "2154", "5766", "4956", "1113", "5866", "1480", "4364", "63", "5391", "3779", "3343", "1620", "477", "3667", "3049", "2351", "4270", "59", "1843", "1505", "4879", "559", "2530", "5447", "3870", "1798", "1001", "724", "5779", "4085", "4971", "4735", "5210", "6267", "739", "3821", "5217", "4484", "1228", "4945", "1706", "3288", "205", "1828", "4831", "1960", "104", "1464", "5412", "1689", "402", "858", "4785", "4755", "6162", "5005", "2918", "2670", "2205", "3094", "5687", "4944", "1576", "625", "1558", "4605", "2756", "3862", "236", "6077", "1816", "3865", "1795", "1049", "1584", "1311", "1068", "5613", "4302", "2481", "2690", "3651", "949", "1997", "2613", "1117", "4323", "4606", "6221", "6252", "4645", "3778", "5407", "4761", "2722", "4782", "153", "2881", "3875", "675", "2259", "3073", "4434", "257", "5367", "6053", "624", "5864", "5646", "1746", "1957", "3919", "2793", "2277", "3897", "1323", "2714", "5608", "2333", "4276", "6113", "3006", "5807", "5845", "2562", "5890", "546", "78", "4306", "6176", "2542", "3372", "813", "910", "2270", "766", "3142", "270", "1503", "1433", "260", "3081", "3284", "1734", "3370", "1968", "259", "3305", "1242", "5735", "623", "95", "1507", "5725", "2520", "885", "4810", "4144", "4138", "326", "1243", "1469", "3749", "714", "48", "5177", "1775", "4188", "2044", "6058", "6079", "4778", "178", "1060", "3672", "562", "3965", "1655", "1514", "4884", "2397", "5236", "541", "3015", "5584", "5667", "4600", "896", "2110", "870", "3429", "2671", "3771", "5653", "16", "2658", "2309", "5614", "64", "1758", "2604", "5710", "5769", "5726", "563", "66", "4678", "4", "84", "2297", "404", "6204", "2892", "5419", "3151", "1305", "5219", "1427", "3988", "2930", "743", "2678", "5547", "3910", "4024", "3961", "2208", "5788", "4665", "2688", "3887", "4061", "1609", "5571", "2203", "437", "2176", "1621", "732", "5932", "1753", "3539", "455", "38", "1925", "1962", "3002", "2951", "4072", "6245", "1179", "1824", "3206", "2398", "3707", "5889", "4576", "1866", "3131", "4404", "1835", "2314", "5301", "5152", "3457", "2354", "861", "3747", "5888", "3321", "3146", "2526", "4502", "3365", "2265", "162", "4565", "4870", "5904", "5640", "3589", "3315", "1396", "3505", "2034", "367", "4129", "1220", "903", "5938", "1174", "4098", "1934", "5800", "3088", "745", "4570", "542", "1478", "3512", "6237", "5765", "5109", "993", "4044", "2727", "2713", "4286", "4271", "5475", "4043", "1589", "6276", "4918", "2279", "905", "681", "5190", "1539", "4459", "5852", "2377", "5174", "1909", "5382", "3097", "4312", "4754", "2710", "4395", "4671", "5558", "3058", "1761", "2877", "5044", "43", "3468", "4993", "422", "3996", "2164", "5652", "3407", "3274", "4637", "452", "5840", "5976", "3312", "2284", "4382", "4377", "2195", "2506", "1421", "5582", "3878", "3874", "238", "2521", "1693", "4629", "246", "1728", "5869", "298", "1543", "4773", "628", "3369", "5198", "139", "279", "5434", "6142", "2538", "804", "2875", "5140", "5551", "1726", "5321", "4832", "3385", "1238", "3591", "4410", "1927", "4602", "591", "1806", "4334", "1719", "5791", "1041", "1475", "6144", "4897", "765", "1883", "520", "3855", "1407", "125", "4609", "4786", "3441", "871", "3940", "2128", "2209", "2342", "2339", "5375", "2448", "795", "2574", "3944", "4064", "5996", "2189", "3428", "1410", "5529", "649", "4644", "311", "3698", "4805", "5986", "1653", "3553", "3335", "5526", "4052", "5723", "3633", "102", "5576", "5620", "5191", "6019", "612", "1532", "775", "1649", "2150", "3354", "250", "6177", "4443", "6120", "4019", "5822", "1157", "1419", "846", "4669", "5489", "4147", "2716", "704", "5146", "3459", "5833", "4291", "2414", "5749", "2020", "3192", "447", "5549", "4068", "6038", "430", "3796", "4868", "2555", "529", "2565", "5025", "4903", "3000", "954", "4763", "2757", "1081", "1074", "379", "693", "5359", "1022", "917", "1162", "6021", "3209", "4793", "3991", "1439", "3225", "5263", "4451", "5076", "951", "6193", "3825", "768", "5494", "4713", "6257", "6246", "1053", "851", "4620", "3903", "4768", "4070", "1851", "2158", "3011", "3931", "5654", "4028", "295", "2960", "2928", "4386", "5034", "1381", "1124", "1040", "4664", "2455", "3695", "2824", "1677", "6198", "5973", "188", "1132", "3798", "3990", "2135", "240", "1385", "2047", "2366", "1848", "5115", "5468", "6226", "2440", "2106", "5347", "1121", "2050", "4203", "679", "6274", "1109", "667", "1340", "5349", "3971", "6080", "5032", "593", "2282", "2305", "1524", "1184", "1087", "1460", "3267", "1549", "840", "5123", "2893", "220", "3298", "2464", "6241", "1047", "4486", "605", "3880", "5862", "181", "350", "4727", "511", "3696", "2166", "382", "5099", "2545", "4842", "2994", "3241", "3461", "3147", "5192", "2412", "3176", "2222", "3356", "2126", "1000", "2493", "3641", "4904", "1874", "4042", "400", "5939", "957", "1200", "1329", "5808", "4877", "5823", "4243", "930", "510", "6059", "5438", "2836", "5568", "1644", "5517", "3397", "3520", "5624", "3711", "4198", "2408", "2860", "5387", "3270", "2039", "1886", "2837", "6174", "622", "4303", "3484", "5581", "1637", "5597", "6157", "1091", "3023", "1784", "931", "4923", "3680", "504", "805", "4523", "853", "5775", "1878", "5278", "5207", "2858", "3421", "2815", "3847", "1453", "1373", "1312", "5637", "3051", "2838", "2883", "3710", "4574", "5225", "2975", "1985", "2665", "5162", "4090", "5314", "2549", "668", "2343", "3201", "454", "4561", "2219", "2630", "3114", "4405", "4146", "5143", "2816", "5476", "3198", "5277", "4635", "1744", "3448", "849", "6030", "3153", "6060", "5698", "5981", "5812", "4558", "2977", "5994", "3760", "3482", "2859", "3493", "2491", "4964", "5166", "4841", "2389", "2656", "1993", "4608", "4981", "5784", "4647", "3921", "4387", "1271", "4314", "5455", "5798", "1819", "5752", "5496", "2779", "342", "1710", "4599", "3458", "6075", "3986", "3044", "1935", "1473", "5621", "5122", "5738", "453", "4578", "5906", "5439", "4202", "2676", "549", "216", "6047", "5232", "1071", "3563", "1177", "2519", "2469", "1587", "4290", "1403", "5334", "2571", "733", "4137", "3925", "2095", "3830", "3224", "1564", "2324", "2654", "5202", "3432", "3832", "1891", "1562", "3464", "1426", "2672", "5737", "5728", "2750", "5754", "5956", "2832", "4941", "2693", "5031", "3752", "725", "4711", "2056", "7", "4910", "589", "2660", "996", "4544", "5472", "983", "2948", "4965", "3105", "4859", "5896", "2568", "470", "3026", "2504", "5977", "2190", "329", "2496", "4591", "3797", "4006", "1366", "150", "1977", "5703", "2770", "2387", "3586", "5762", "1176", "303", "1723", "3930", "4260", "2313", "478", "1551", "4994", "1255", "3113", "806", "310", "1017", "5440", "1921", "4547", "4726", "3226", "2349", "3584", "182", "3831", "4766", "2882", "1052", "2433", "4757", "662", "749", "4677", "1140", "2807", "3069", "2650", "2943", "2618", "4811", "5821", "5479", "4113", "2910", "4828", "493", "4261", "1793", "4517", "4882", "93", "3046", "2734", "204", "1836", "4421", "3327", "4592", "5338", "1024", "1388", "1248", "5411", "1122", "2023", "5062", "2488", "6096", "4083", "5801", "3496", "3101", "673", "1008", "3485", "3089", "5544", "1929", "5642", "1044", "927", "2434", "2639", "1077", "3527", "1894", "3184", "3045", "5627", "643", "5473", "1042", "2778", "3235", "2183", "2261", "1263", "1357", "5508", "5147", "3892", "3588", "3713", "2467", "1965", "5090", "518", "1692", "1033", "2089", "5616", "5454", "1508", "4588", "1906", "4455", "2507", "2573", "1134", "587", "3923", "2782", "4656", "521", "5096", "2887", "4942", "3818", "3247", "716", "4843", "6106", "2292", "1688", "2776", "4974", "1773", "4504", "6118", "1774", "2283", "2115", "3660", "1205", "6129", "3982", "2328", "1674", "1542", "5672", "1814", "3076", "3755", "1786", "751", "4510", "3239", "4738", "3406", "99", "4214", "4020", "561", "5628", "3974", "3604", "2561", "2055", "2010", "3994", "2915", "2805", "4426", "921", "1770", "364", "2684", "5091", "3783", "2524", "1295", "5481", "224", "4309", "607", "6045", "5015", "2153", "4385", "1447", "4573", "4394", "3107", "710", "532", "4478", "6217", "4031", "1569", "976", "913", "5302", "5325", "4368", "633", "3171", "2360", "2825", "3249", "2437", "3858", "233", "2717", "5707", "3631", "1933", "239", "5951", "3104", "3705", "3756", "5287", "5097", "4035", "5573", "5187", "5329", "801", "604", "1274", "4409", "294", "6009", "4960", "1262", "1550", "5874", "2355", "4238", "4084", "5638", "4496", "3999", "1114", "1863", "1180", "4279", "2933", "1705", "4623", "5990", "4099", "5718", "189", "3566", "2718", "5649", "2172", "414", "4598", "5433", "2663", "3541", "3859", "1318", "263", "5771", "1297", "3683", "3829", "4280", "5945", "5520", "2586", "4100", "3467", "1988", "6187", "3688", "746", "5456", "5816", "6249", "2160", "5071", "4328", "1703", "1172", "2066", "2267", "5156", "929", "5077", "3978", "4940", "1917", "2834", "4750", "5949", "1315", "978", "6211", "348", "4392", "4369", "5376", "2651", "5295", "5235", "2699", "4149", "5802", "5179", "2407", "5172", "3992", "1046", "3193", "462", "2748", "4450", "5786", "2976", "1525", "1752", "5380", "6145", "1930", "4041", "2450", "5139", "5969", "5480", "2771", "2486", "2368", "709", "3064", "918", "4466", "2732", "2937", "1185", "4077", "5806", "2196", "1144", "4436", "3918", "2804", "1639", "5312", "114", "3420", "2929", "2234", "5514", "2187", "2027", "2133", "5708", "3911", "6023", "940", "4089", "651", "2285", "3716", "269", "1787", "1208", "6256", "1827", "3835", "1990", "3938", "5316", "4372", "1749", "3689", "615", "6192", "333", "5161", "3826", "2046", "1377", "1792", "3057", "2595", "3560", "4936", "2597", "3765", "2631", "5144", "3530", "646", "830", "1098", "963", "2441", "5285", "6260", "6242", "1061", "3080", "4425", "2085", "479", "2686", "4473", "1485", "5057", "1210", "782", "1881", "1777", "1678", "1722", "3292", "4204", "1135", "2823", "1450", "6159", "4422", "1067", "6126", "5136", "484", "5133", "3399", "5795", "5021", "6090", "5844", "2608", "1209", "2428", "419", "4241", "1724", "2006", "767", "1495", "304", "971", "1865", "1351", "1701", "5004", "5819", "1779", "5662", "4966", "4983", "2073", "1632", "3009", "4115", "3384", "1981", "5273", "4325", "1084", "5135", "1645", "4177", "1443", "4541", "5522", "3692", "4549", "5978", "3253", "3028", "1045", "413", "3769", "1129", "5559", "3264", "818", "2711", "4693", "4536", "2909", "4470", "3687", "1007", "1056", "1593", "3676", "3650", "5016", "2839", "4684", "869", "4109", "133", "3052", "231", "5931", "6231", "3396", "3574", "4723", "4262", "4269", "3136", "3328", "320", "3170", "6225", "1607", "120", "5997", "1404", "9", "644", "3789", "3546", "2093", "3913", "3629", "1841", "4816", "701", "794", "4176", "2071", "1488", "1781", "4322", "3730", "6072", "106", "3898", "2109", "2318", "3250", "1687", "1794", "6212", "1813", "4482", "2640", "6116", "1136", "2738", "332", "5041", "4630", "5757", "1246", "3477", "1867", "47", "5891", "1538", "2444", "5965", "974", "984", "5602", "5129", "3393", "256", "431", "1698", "4694", "3634", "3993", "5230", "1070", "4467", "439", "1531", "3075", "5711", "1735", "6191", "4209", "3873", "3609", "3981", "4824", "1535", "5936", "617", "1252", "5318", "2118", "5089", "5199", "1731", "1994", "1604", "3275", "1418", "4850", "1290", "4437", "1750", "1003", "5014", "4617", "5979", "4767", "5848", "4538", "4737", "5789", "226", "3178", "0", "1283", "678", "3243", "2569", "783", "5138", "6041", "524", "967", "1996", "894", "2768", "3665", "571", "3272", "2478", "6018", "4373", "3281", "616", "2155", "2213", "5591", "6196", "5617", "4336", "6201", "1660", "1606", "841", "1085", "5634", "1225", "2924", "6151", "4758", "1890", "2745", "6083", "3599", "79", "5785", "2528", "5750", "3561", "2236", "3332", "28", "880", "5937", "1737", "4055", "730", "2141", "2657", "4218", "1877", "5747", "2865", "1187", "4000", "316", "4639", "5543", "1181", "2465", "1401", "950", "3886", "1207", "3500", "2525", "3483", "3671", "4095", "1115", "5491", "4627", "938", "2322", "3585", "6054", "845", "5048", "2593", "1739", "4417", "2848", "2028", "838", "6175", "6110", "4854", "4519", "3545", "2499", "772", "4339", "83", "2142", "2957", "2383", "4062", "5922", "2162", "1064", "53", "568", "3748", "6188", "5609", "3775", "3110", "5396", "5362", "600", "1309", "2242", "3437", "5308", "6219", "5368", "491", "6040", "4208", "6001", "3411", "5184", "4520", "5332", "5692", "1414", "3183", "4681", "3763", "3003", "3182", "1264", "6025", "952", "1884", "174", "1376", "4781", "890", "3720", "2078", "4826", "134", "1278", "4265", "2178", "760", "4806", "1398", "4163", "4812", "1031", "4526", "5562", "1326", "4603", "3149", "5035", "5539", "1299", "318", "2655", "1995", "6124", "4435", "2632", "4058", "2423", "791", "2704", "2905", "1699", "2621", "1386", "1829", "5856", "2513", "873", "2523", "4934", "3916", "4156", "2123", "2011", "4991", "5282", "3899", "1328", "6143", "428", "6012", "3600", "2019", "5050", "3233", "4359", "572", "2320", "6215", "5484", "4247", "1617", "3024", "5941", "180", "3410", "197", "5669", "4285", "1285", "4625", "1516", "3200", "1466", "4500", "3806", "934", "52", "1367", "4909", "4039", "1444", "1005", "4182", "4288", "3639", "280", "105", "3211", "972", "3111", "5008", "2900", "4246", "779", "1911", "842", "6259", "2746", "1756", "432", "2715", "4419", "2652", "2993", "970", "4777", "4821", "1150", "3377", "5553", "4442", "4537", "3603", "2393", "4251", "2091", "4304", "1321", "6240", "3746", "346", "1435", "852", "4153", "1667", "5354", "450", "3186", "376", "820", "5835", "2668", "508", "2581", "995", "172", "5790", "1902", "2795", "2592", "3572", "17", "237", "2649", "1791", "1148", "5306", "1654", "2763", "217", "1082", "473", "2633", "1955", "4921", "4691", "2425", "3838", "3358", "2473", "932", "2370", "4951", "1534", "154", "141", "3877", "6051", "3236", "1602", "1344", "3881", "778", "117", "2233", "4552", "3262", "2784", "1080", "876", "4282", "6063", "3223", "512", "4240", "4295", "2751", "6173", "6161", "3404", "2463", "4634", "5471", "2818", "581", "1229", "3843", "1768", "5373", "706", "5371", "3492", "731", "1313", "2874", "4581", "2438", "2527", "2806", "507", "5920", "169", "6069", "6013", "4734", "3271", "5293", "2057", "5719", "2260", "6133", "2938", "5002", "543", "2749", "5586", "4420", "429", "4741", "2012", "4626", "4054", "1131", "3449", "3544", "5204", "2913", "640", "3456", "3969", "2787", "3103", "3154", "2835", "1709", "961", "5988", "3950", "1626", "2878", "4311", "1415", "2707", "2418", "3851", "1236", "262", "5345", "2503", "3313", "5286", "68", "3820", "3273", "5929", "2534", "3341", "1371", "1277", "5427", "1411", "531", "2194", "878", "2851", "136", "5842", "1648", "1506", "6050", "6087", "5502", "3400", "1647", "4086", "5694", "4988", "2374", "3034", "2330", "5428", "2703", "502", "4717", "4992", "3480", "1871", "4155", "2119", "866", "37", "3466", "650", "5690", "1876", "4277", "4799", "728", "6056"]], "valid": ["int", ["4342", "2416", "3712", "179", "5770", "4636", "2000", "2289", "4411", "4528", "3706", "3179", "4123", "212", "3256", "3043", "2955", "2171", "1817", "6230", "3231", "35", "1266", "3743", "3503", "345", "1826", "1547", "4239", "3625", "3786", "1658", "5252", "4178", "5946", "2084", "167", "2904", "2449", "2201", "4893", "5346", "5518", "436", "8", "4784", "2897", "2308", "2558", "5566", "4736", "1583", "5406", "412", "3348", "4237", "5881", "3014", "752", "474", "5655", "4935", "1048", "3673", "2880", "3012", "5767", "2181", "5699", "5940", "5151", "322", "2518", "1101", "576", "1652", "2474", "5850", "196", "4139", "1800", "3344", "2191", "609", "138", "5435", "1529", "1597", "3373", "1490", "6117", "1167", "2831", "5234", "843", "3592", "4015", "5803", "1104", "195", "909", "5992", "2230", "5727", "1619", "1740", "5052", "2371", "5492", "5464", "4610", "5947", "3718", "3391", "5372", "3708", "2855", "23", "2583", "5229", "4104", "4661", "1500", "5924", "5497", "2594", "1", "4619", "2485", "5739", "3564", "5933", "2744", "5436", "5121", "3402", "2879", "4076", "5298", "2719", "3286", "6065", "1020", "4192", "61", "6147", "2413", "3234", "177", "4253", "4452", "1322", "5390", "1142", "4704", "3767", "2537", "5987", "660", "2509", "516", "545", "1231", "3681", "4495", "4453", "3526", "6156", "1742", "5364", "626", "3824", "327", "235", "3662", "1356", "5656", "4978", "3388", "3983", "2577", "2533", "4913", "4017", "2363", "5404", "1273", "6022", "5106", "2973", "96", "4575", "5426", "4162", "3469", "2869", "3346", "4682", "3129", "4673", "2969", "5905", "5154", "3731", "5357", "547", "1446", "2796", "2291", "4651", "1919", "3836", "711", "2170", "4353", "330", "1983", "5910", "6266", "4036", "3345", "5741", "5051", "194", "2301", "5570", "12", "2245", "6262", "2547", "5352", "3392", "34", "5453", "3590", "3726", "5242", "5258", "4136", "110", "2761", "3610", "4472", "3062", "3431", "1544", "3383", "5493", "1665", "170", "3800", "1108", "2456", "1918", "1694", "438", "2563", "2721", "5588", "694", "1279", "6123", "2645", "2679", "2174", "3814", "5478", "3160", "4790", "2375", "2682", "5193", "4449", "2730", "2551", "926", "1029", "5556", "5127", "5429", "3929", "5196", "1854", "4937", "1055", "5395", "2002", "40", "1799", "5130", "1288", "4986", "689", "3230", "902", "5753", "3684", "5350", "2508", "5500", "665", "4845", "2272", "3237", "5319", "4131", "3375", "85", "2689", "5921", "5437", "3729", "3736", "645", "5901", "4438", "451", "2780", "5717", "2791", "1669", "1718", "6247", "847", "1797", "4087", "1629", "1546", "637", "2683", "5173", "5712", "5189", "1989", "4023", "1601", "5119", "977", "158", "1372", "5632", "5341", "3322", "5950", "4584", "5607", "2480", "1474", "5948", "1790", "3334", "4837", "4021", "3266", "4108", "365", "5291", "3565", "3735", "2917", "3159", "3815", "421", "286", "494", "4165", "4491", "3019", "2898", "3540", "1057", "4775", "1284", "3519", "3619", "4878", "4367", "2758", "2978", "41", "544", "4689", "3722", "3090", "1489", "387", "4197", "4193", "4774", "6064", "2427", "5340", "3593", "3750", "4582", "50", "2626", "1416", "6039", "5715", "2945", "1143", "3141", "4347", "1347", "800", "4977", "5894", "3567", "4205", "4900", "2967", "5761", "1281", "5639", "2802", "4855", "1258", "3453", "6128", "656", "3802", "4733", "1498", "1857", "3907", "458", "5446", "2894", "1823", "4885", "2921", "2576", "3854", "2378", "4946", "2156", "863", "3018", "2731", "895", "1936", "1018", "4307", "740", "2276", "351", "2591", "210", "4753", "3181", "824", "551", "148", "4675", "121", "2409", "1803", "3382", "3083", "6263", "315", "4698", "5648", "654", "1967", "3728", "6164", "3995", "5682", "1830", "911", "1145", "4621", "5289", "3523", "1224", "1494", "385", "5772", "4708", "1566", "6066", "2400", "5023", "4121", "19", "335", "3912", "1680", "199", "1327", "1232", "786", "3927", "3138", "3287", "4567", "214", "4091", "3434", "2021", "1280", "296", "973", "3125", "3465", "2058", "556", "1837", "1409", "4871", "1931", "2107", "4699", "3727", "4933", "3486", "5689", "2980", "669", "3719", "2454", "2661", "2332", "1502", "4660", "1556", "4674", "3902", "2808", "3417", "2901", "6107", "5378", "1219", "3856", "2040", "215", "1872", "4185", "4297", "3139", "5171", "5644", "119", "3556", "5683", "4505", "4424", "2157", "817", "2077", "2116", "5416", "2139", "3770", "1611", "4583", "5011", "2312", "2828", "815", "1261", "2334", "3364", "4920", "5112", "2849", "4838", "3955", "5113", "2696", "6027", "2232", "1980", "5829", "486", "5485", "2004", "2627", "567", "736", "3490", "4839", "4213", "1341", "5206", "1166", "4440", "5736", "1314", "3522", "3768", "1729", "2112", "492", "1170", "900", "6006", "3173", "1560", "2599", "3156", "859", "4210", "3636", "2867", "884", "2024", "3632", "1523", "6093", "3351", "3300", "2252", "3087", "386", "756", "264", "4158", "3317", "5214", "3670", "4631", "3416", "3803", "4999", "5943", "4292", "4107", "457", "2345", "3042", "5875", "5975", "3118", "4278", "888", "4601", "443", "4801", "3700", "3408", "4348", "6035", "3963", "5339", "1027", "4535", "6248", "3795", "5622", "5507", "4836", "6122", "2841", "2382", "5563", "1497", "4051", "3973", "4236", "517", "5872", "109", "456", "3187", "6073", "4883", "5998", "630", "5163", "3004", "2911", "1438", "1105", "3837", "6232", "2395", "4463", "2775", "4047", "2666", "2798", "2987", "3309", "2461", "4720", "1630", "1580", "1765", "727", "2", "10", "836", "5183", "4264", "4683", "2611", "3204", "4643", "2614", "5501", "2636", "3379", "75", "276", "481", "1405", "1169", "1173", "991", "6108", "5670", "2724", "251", "3071", "2531", "4646", "4516", "671", "5668", "3212", "1116", "4807", "4551", "2995", "2642", "3319", "2447", "1325", "2811", "5868", "4346", "5245", "4747", "3132", "3077", "4354", "2402", "1618", "282", "1171", "5688", "6082", "3791", "1240", "5897", "5012", "577", "4012", "569", "355", "5511", "4201", "5641", "5984", "2843", "6149", "1093", "4222", "2872", "4501", "3455", "1949", "1459", "108", "816", "3531", "5019", "3067", "4556", "4333", "5335", "3134", "953", "4533", "2931", "2310", "5745", "5744", "5231", "328", "2500", "2197", "5942", "2966", "3846", "5523", "3751", "3167", "3117", "712", "208", "2453", "3953", "5083", "2102", "3261", "5678", "435", "3242", "268", "2459", "899", "4057", "2290", "3016", "4078", "5820", "3079", "5176", "4856", "1833", "449", "4352", "3251", "1623", "4597", "5841", "3557", "5721", "3123", "5205", "3578", "4825", "2979", "1308", "3433", "3020", "4464", "964", "3398", "3647", "4846", "1345", "5542", "608", "5483", "4652", "2783", "764", "2347", "5366", "4492", "1932", "2323", "163", "3228", "4970", "1146", "4161", "5401", "4082", "5916", "90", "1634", "857", "3895", "5085", "3219", "4355", "3933", "5276", "4365", "2307", "5467", "4046", "2972", "446", "5583", "2381", "5729", "292", "4686", "1815", "602", "252", "4497", "3524", "3649", "2754", "4167", "3939", "5557", "2025", "614", "5532", "1552", "1548", "744", "1561", "6270", "4817", "3550", "5643", "1428", "3548", "5895", "2092", "4390", "2240", "5385", "1186", "3909", "4485", "3573", "1785", "5149", "185", "5782", "267", "5047", "4795", "5054", "4454", "2180", "4460", "4804", "2220", "4446", "1861", "337", "1118", "2479", "5108", "1343", "4350", "406", "6273", "3084", "3620", "1530", "6209", "1776", "3664", "4324", "4183", "5827", "2346", "6251", "828", "2244", "2866", "1073", "789", "1028", "211", "1979", "4244", "5251", "5374", "2582", "1943", "5706", "4456", "4416", "2271", "4803", "5661", "5250", "596", "2186", "401", "4580", "3928", "1971", "5908", "6014", "228", "2598", "1253", "1574", "4132", "4749", "3968", "5548", "4744", "6222", "6036", "4030", "2311", "1183", "5629", "4531", "6172", "5379", "4001", "5336", "4296", "3568", "5280", "6046", "3844", "5663", "4230", "6115", "6279", "2925", "6184", "925", "3290", "1189", "2471", "1482", "2022", "1245", "4462", "1335", "721", "219", "2105", "3657", "856", "2647", "6067", "2653", "1078", "670", "3529", "3450", "4441", "3157", "313", "297", "4284", "353", "4097", "2165", "2294", "5968", "2009", "3175", "5", "2767", "1097", "2777", "4907", "4511", "1106", "3764", "2030", "1257", "3946", "3366", "5311", "4310", "1527", "3357", "3935", "4049", "1004", "5182", "872", "3499", "5253", "5417", "247", "1458", "3972", "5612", "1982", "2912", "4931", "469", "5080", "3304", "5660", "3127", "1457", "309", "3891", "1838", "3030", "4764", "4225", "1382", "39", "619", "1204", "278", "6238", "535", "2215", "2988", "2607", "4587", "2773", "363", "3419", "5269", "5111", "5243", "222", "5246", "4374", "1904", "2845", "4315", "2372", "3005", "2177", "1014", "3781", "2249", "1324", "4499", "3533", "1754", "3325", "2376", "3353", "4949", "5631", "3598", "750", "3827", "4116", "1858", "6275", "2742", "3371", "692", "2842", "2287", "2348", "3115", "5431", "5503", "2445", "2914", "3964", "687", "3478", "4194", "6189", "4679", "5647", "3828", "4283", "4760", "2269", "2522", "290", "1214", "3924", "908", "360", "3336", "4114", "5351", "5878", "1598", "1010", "3801", "20", "2026", "255", "5777", "6092", "2280", "2218", "3376", "1384", "373", "3128", "3699", "734", "4432", "5686", "3532", "3571", "1801", "1079", "1424", "915", "594", "5957", "5024", "5399", "4257", "3864", "5228", "1510", "4393", "579", "4975", "792", "2477", "4896", "128", "4112", "1818", "2214", "4650", "254", "1337", "6094", "3311", "1160", "1582", "397", "5972", "2331", "1137", "3853", "4056", "5068", "574", "2634", "4973", "1778", "1708", "2998", "444", "2550", "331", "1675", "5259", "3460", "1808", "5882", "2394", "827", "4383", "4730", "3246", "4254", "1987", "5606", "1747", "2765", "4345", "1348", "1306", "4969", "968", "4069", "5420", "3085", "2644", "1012", "6097", "2223", "4985", "4667", "1155", "32", "3438", "5078", "4559", "5731", "4641", "2873", "5046", "723", "4771", "4522", "1434", "5294", "2935", "5087", "1213", "4886", "89", "2369", "788", "498", "1573", "4403", "2876", "886", "2907", "5914", "4869", "4860", "1730", "4319", "1567", "1642", "2295", "4010", "1807", "928", "4289", "2991", "2306", "2253", "5902", "14", "33", "5685", "2060", "1683", "1895", "3715", "3112", "5043", "5962", "4206", "2895", "1032", "3318", "6255", "4402", "2810", "770", "2068", "3738", "5746", "2664", "2494", "5363", "5322", "5221", "2248", "776", "5132", "682", "480", "5007", "3238", "69", "6135", "2989", "5674", "347", "762", "2864", "2391", "5836", "424", "5033", "308", "3065", "4299", "697", "3714", "5045", "1111", "4318", "3761", "4124", "6146", "1565", "2502", "2101", "5596", "1127", "2484", "1999", "383", "555", "3656", "1625", "5260", "2224", "4016", "3150", "2692", "6010", "3807", "4226", "5598", "4007", "2723", "4564", "922", "1178", "4553", "5574", "3203", "2344", "1406", "4384", "2516", "5360", "1112", "1094", "3602", "1397", "554", "171", "865", "3061", "5733", "2996", "3488", "5153", "5851", "906", "1287", "1227", "5474", "2540", "2643", "1452", "3554", "1893", "2482", "4169", "4294", "2436", "242", "2188", "3229", "2108", "213", "4320", "5713", "2625", "6134", "4867", "3977", "1736", "1966", "2352", "122", "265", "1250", "3479", "960", "1445", "6089", "4929", "5983", "3278", "1354", "5799", "4745", "3737", "4375", "4572", "4899", "4524", "6020", "5534", "5843", "5911", "3724", "4266", "5369", "5268", "4788", "4273", "672", "3788", "4666", "6104", "4356", "13", "2961", "3208", "4154", "2325", "4207", "1581", "6152", "699", "2922", "5680", "1809", "176", "4005", "4376", "2846", "2379", "2288", "1392", "3471", "482", "1110", "460", "1907", "5927", "920", "5705", "2326", "5095", "1218", "2462", "2854", "3867", "2211", "5546", "525", "2373", "6141", "1832", "2124", "2059", "4498", "4789", "966", "2262", "1842", "4740", "2965", "5134", "962", "4184", "2985", "378", "3516", "5442", "2720", "2490", "1336", "2949", "1088", "3852", "277", "2886", "658", "2457", "5541", "3368", "5370", "5114", "1339", "1695", "6037", "3447", "5892", "2316", "5619", "3776", "1272", "3606", "5830", "132", "1006", "2728", "4820", "6074", "5018", "3381", "4429", "812", "6268", "2590", "1390", "735", "3310", "1924", "1879", "5657", "2852", "5036", "3646", "2229", "5272", "1330", "5611", "3690", "4212", "1300", "1291", "4672", "3439", "3412", "86", "3038", "4616", "1521", "4397", "2799", "4695", "1294", "1712", "3095", "5160", "5758", "3894", "959", "261", "3849", "4995", "1973", "1721", "5633", "1840", "6269", "1912", "403", "3871", "3240", "3021", "639", "4690", "4875", "1557", "497", "1035", "164", "4219", "1015", "6044", "6033", "4065", "4844", "741", "5913", "3517", "6148", "2772", "1492", "2737", "3842", "6220", "3945", "5072", "2163", "1470", "6103", "4256", "2629", "3265", "3659", "416", "3", "5768", "759", "4216", "2681", "1992", "1782", "4915", "4105", "87", "4798", "981", "5082", "855", "1412", "4245", "879", "3308", "49", "3329", "2386", "5284", "1223", "2329", "4607", "248", "1615", "1844", "1555", "4676", "112", "1733", "4818", "5793", "4330", "988", "3967", "1845", "161", "4479", "6081", "165", "742", "4894", "3116", "3489", "4545", "5188", "819", "2536", "4179", "5944", "5748", "245", "1579", "4358", "4106", "440", "4088", "4654", "6127", "6088", "1896", "5701", "6171", "2041", "829", "919", "6130", "2812", "3809", "5304", "1885", "1748", "528", "3245", "924", "3032", "1484", "5299", "5320", "2735", "3347", "5893", "586", "3096", "642", "5288", "2572", "3677", "3597", "2099", "3010", "3595", "5448", "4469", "5696", "1431", "5244", "1197", "5734", "4026", "3462", "2098", "2250", "4928", "2439", "1149", "1846", "3029", "2144", "3137", "6052", "1685", "5137", "2941", "183", "5579", "875", "323", "1537", "123", "514", "5605", "4152", "135", "3491", "3301", "2254", "5664", "854", "1875", "707", "5128", "4697", "4048", "3812", "4423", "3068", "758", "5934", "3608", "1346", "2891", "5971", "5361", "6114", "1034", "5886", "4589", "1592", "1039", "1269", "2667", "3451", "4976", "5755", "2968", "4613", "273", "2700", "5117", "249", "891", "1141", "570", "5487", "4702", "1270", "6200", "2673", "1448", "553", "394", "1769", "557", "4719", "2923", "3547", "2764", "302", "4569", "2430", "3623", "5898", "5519", "2833", "5079", "1120", "4217", "6253", "3255", "5814", "1362", "2800", "5535", "1603", "4160", "2950", "1860", "2548", "5213", "5126", "4259", "4447", "2942", "3885", "2624", "787", "3958", "284", "4772", "2600", "1657", "6216", "5482", "476", "275", "2781", "287", "6158", "2067", "5423", "5776", "5867", "2701", "5928", "3120", "3008", "4722", "4917", "4073", "5069", "3254", "784", "380", "3047", "1359", "3177", "5760", "515", "1463", "5513", "889", "3616", "5564", "1212", "2603", "5209", "5854", "4233", "5248", "1486", "22", "6180", "1276", "1517", "2401", "5961", "4079", "4614", "1759", "2241", "2200", "3723", "3549", "3666", "1963", "627", "3502", "844", "5804", "1856", "2362", "4906", "3741", "1437", "1810", "6207", "4252", "5645", "2136", "6031", "5740", "2792", "1477", "2017", "6205", "2417", "1442", "3643", "6068", "1939", "6140", "729", "94", "2813", "4361", "2475", "6101", "4648", "4873", "3509", "4103", "2605", "6015", "3169", "4874", "203", "3031", "2986", "718", "5459", "3570", "2826", "4025", "1479", "5935", "4891", "6055", "130", "4982", "4388", "1571", "748", "6181", "2476", "1317", "2497", "2193", "3302", "1588", "6150", "3882", "341", "358", "4622", "140", "1679", "312", "3476", "4961", "3048", "3615", "3773", "5315", "969", "5527", "5279", "2384", "4508", "4710", "2052", "1360", "4957", "1839", "445", "448", "5073", "530", "389", "4027", "4532", "5885", "4655", "4474", "209", "6032", "5569", "797", "5107", "3799", "523", "3162", "3443", "6182", "4293", "3306", "5239", "2120", "1984", "3498", "3473", "2741", "6034", "811", "1859", "5783", "1090", "4742", "1952", "986", "2140", "5275", "6028", "2709", "3876", "1002", "1586", "5402", "994", "1888", "1666", "67", "143", "4938", "5550", "1075", "584", "3966", "5907", "131", "3511", "5610", "3387", "258", "603", "4990", "3624", "5540", "1986", "2691", "5499", "2202", "1249", "4004", "4779", "4696", "4196", "4032", "1202", "4248", "1914", "3276", "4514", "5195", "223", "1267", "1882", "2899", "3232", "4272", "71", "3001", "5486", "2797", "369", "2556", "1244", "2420", "4926", "1194", "893", "2337", "4908", "2074", "1901", "4187", "3303", "1083", "5587", "4413", "4093", "24", "3655", "5238", "5533", "3668", "1320", "1711", "3703", "3562", "1825", "595", "1175", "2755", "6016", "4181", "4731", "1650", "1519", "4071", "5324", "4195", "1222", "3868", "408", "618", "3537", "564", "4748", "1948", "2336", "232", "307", "4604", "2458", "471", "6154", "2712", "4924", "2637", "3753", "2553", "1714", "107", "441", "4430", "6280", "4560", "2971", "4756", "5344", "4013", "4765", "3943", "3463", "423", "5084", "1920", "4357", "4953", "1408", "2053", "5955", "912", "583", "2695", "1834", "702", "3536", "5618", "3263", "1103", "6208", "1037", "5665", "3390", "753", "5274", "1713", "1892", "769", "5604", "3022", "2788", "519", "4507", "3289", "3840", "500", "1512", "411", "5691", "5857", "5220", "3039", "1089", "467", "4274", "2934", "4067", "5392", "2435", "4180", "2884", "1928", "3596", "5716", "391", "5356", "4159", "4954", "5567", "5636", "4848", "3521", "1743", "4902", "2419", "3195", "5418", "3819", "496", "5887", "5695", "4134", "157", "1423", "407", "5305", "3777", "2399", "3293", "5561", "2075", "2266", "2033", "2114", "3904", "487", "2785", "3772", "2752", "241", "5952", "3618", "3282", "5290", "336", "1251", "5271", "5081", "1916", "4914", "4800", "4022", "3164", "5233", "4797", "1152", "3380", "955", "5266", "1387", "3896", "2356", "653", "1417", "1820", "2063", "3320", "5839", "3979", "834", "1590", "6265", "3663", "1585", "4703", "229", "3732", "2470", "395", "485", "3395", "6105", "5075", "3863", "2906", "377", "1926", "1107", "652", "5365", "398", "5445", "684", "3260", "3869", "722", "1374", "2406", "4415", "2517", "5524", "1230", "1595", "3975", "2273", "5064", "145", "6229", "230", "2070", "6078", "1023", "4263", "4168", "324", "2127", "1159", "2452", "6167", "1133", "6008", "1682", "6017", "1672", "1646", "2357", "5599", "206", "5536", "1771", "77", "1643", "4371", "2932", "798", "1941", "6250", "560", "3515", "2669", "3793", "5531", "244", "3035", "2702", "4709", "2013", "4457", "833", "3475", "2132", "4830", "2999", "1831", "4943", "1954", "288", "3108", "4494", "11", "1298", "1304", "5577", "4186", "2278", "3056", "1732", "1518", "5175", "4258", "781", "4546", "3144", "3158", "3908", "426", "774", "5398", "2275", "793", "4378", "2896", "5267", "4725", "1594", "3960", "5432", "4117", "3495", "3580", "5635", "1578", "5441", "2018", "6277", "3414", "3422", "1880", "1391", "1757", "6163", "2300", "4174", "1217", "6132", "118", "1365", "2390", "81", "3780", "582", "1702", "5093", "3258", "823", "1092", "2705", "4642", "2411", "3426", "6070", "2243", "3121", "6214", "3694", "18", "2984", "3510", "5980", "5397", "3040", "3155", "46", "505", "2442", "5059", "6042", "1690", "2251", "1900", "352", "5449", "155", "5102", "4963", "3915", "3299", "1661", "5292", "3954", "825", "1501", "427", "506", "987", "1572", "2037", "207", "5859", "2868", "1616", "4955", "3053", "5006", "1760", "2908", "5554", "5555", "2237", "946", "1596", "1659", "4659", "3594", "2560", "3210", "4611", "4534", "2161", "3007", "3252", "6239", "3205", "1456", "2947", "2747", "3474", "4864", "2580", "4037", "3037", "4344", "5580", "5460", "4555", "6203", "5714", "1763", "3658", "1944", "3704", "4172", "4122", "536", "1333", "3959", "5722", "1196", "1096", "5545", "4751", "4668", "4939", "2610", "4997", "3678", "15", "883", "6102", "2460", "4911", "2601", "4563", "4332", "6168", "1054", "2638", "359", "285", "3934", "3661", "3163", "1663", "3063", "3218", "4094", "1462", "6278", "4480", "4663", "6048", "3185", "6169", "935", "3697", "1139", "5469", "2970", "5203", "1383", "5477", "3717", "1153", "4275", "2255", "193", "975", "5208", "2687", "4170", "3269", "1937", "3976", "3679", "3948", "837", "2043", "5509", "5709", "726", "1422", "881", "6137", "5270", "2404", "2946", "4316", "3614", "149", "2801", "3078", "5131", "4164", "4317", "6076", "272", "4823", "3367", "5038", "2138", "1363", "186", "4852", "1099", "3165", "5773", "1256", "2920", "5383", "3168", "5623", "4847", "6003", "5105", "3394", "4175", "631", "537", "366", "4059", "5222", "4861", "2264", "1608", "850", "4439", "1454", "2216", "3905", "2532", "4596", "2151", "1559", "3559", "5846", "2983", "334", "3423", "1011", "3941", "466", "3872", "2589", "1496", "4840", "4227", "1128", "2939", "3359", "5959", "1303", "168", "4849", "958", "4034", "3119", "1638", "4809", "641", "289", "1950", "175", "5027", "4612", "1504", "810", "3330", "2926", "4746", "4586", "490", "1847", "2762", "1289", "1945", "4979", "3017", "4458", "5810", "5792", "356", "5658", "5594", "1976", "51", "2429", "715", "2064", "1282", "1334", "822", "1796", "384", "1338", "91", "680", "3342", "5650", "5512", "501", "2619", "4657", "5227", "4919", "2543", "5815", "3810", "1379", "1292", "3481", "2535", "5086", "3333", "4922", "1260", "777", "4101", "465", "5410", "221", "5124", "848", "1216", "1449", "5809", "129", "4102", "6061", "1429", "4380", "6236", "6185", "1147", "1788", "192", "5424", "305", "5333", "4475", "4792", "4110", "3194", "6100", "2380", "552", "695", "2008", "1487", "4242", "3191", "790", "1515", "5826", "2198", "5876", "1908", "4518", "1613", "5323", "4234", "548", "3285", "503", "5353", "6194", "1069", "2820", "5328", "5498", "1991", "2559", "1425", "1237", "4527", "5170", "5589", "771", "65", "1165", "4190", "6244", "598", "184", "6086", "3497", "2364", "3378", "4412", "4705", "372", "611", "5600", "3630", "4579", "620", "5824", "2584", "5393", "4362", "5257", "1526", "5254", "483", "101", "6264", "4050", "1355", "4301", "2361", "3558", "5832", "4585", "6091", "2032", "2182", "5194", "396", "3822", "6218", "4468", "5764", "1974", "152", "6139", "3575", "2082", "522", "2844", "5330", "566", "4428", "3082", "5264", "393", "6153", "2847", "5281", "1783", "3552", "5592", "2228", "1671", "2014", "2903", "6160", "5909", "5355", "4477", "4712", "2426", "2405", "4503", "708", "5838", "4539", "2125", "3086", "2036", "1873", "6125", "3725", "664", "5148", "3555", "1375", "5022", "3528", "489", "4618", "319", "1130", "4550", "3055", "5877", "3816", "1402", "62", "4483", "1400", "5408", "1215", "3501", "2035", "2247", "1369", "5159", "2173", "3980", "56", "3389", "76", "5414", "1923", "4814", "2870", "4343", "5145", "513", "2103", "5530", "291", "634", "945", "3576", "3890", "5917", "997", "6210", "1026", "2588", "5458", "4341", "2641", "3901", "5991", "4835", "1102", "538", "2612", "1961", "2786", "343", "5490", "2001", "3932", "2512", "2706", "3682", "60", "2697", "3279", "1910", "1342", "6195", "3543", "6206", "3686", "713", "3454", "3841", "3951", "1707", "1247", "58", "965", "803", "2303", "6085", "5528", "3642", "3757", "5313", "1563", "5157", "475", "3360", "5538", "2602", "4229", "4189", "1905", "3338", "4822", "2388", "2315", "3587", "5241", "5415", "399", "882", "4787", "417", "2514", "1727", "2365", "6095", "3221", "5923", "5443", "2431", "5381", "534", "2415", "1221", "5003", "3734", "3415", "4053", "5060", "1533", "1399", "2072", "4232", "3607", "3942", "2982", "281", "1869", "4140", "666", "6166", "2753", "2829", "3291", "5058", "5590", "1353", "613", "1394", "5265", "588", "5070", "3759", "1805", "5651", "2622", "655", "6165", "3470", "6119", "3350", "3283", "887", "3072", "2184", "1964", "686", "575", "2256", "3745", "5466", "2212", "4448", "1050", "461", "5049", "3093", "2392", "1095", "2992", "1716", "1767", "4857", "3190", "1631", "2733", "5413", "6112", "3693", "5010", "368", "2853", "5296", "1913", "301", "4715", "3339", "5704", "5030", "1972", "3487", "4407", "3857", "1509", "3109", "142", "5065", "2620", "3583", "3033", "5092", "1662", "1852", "1123", "3174", "5255", "5855", "1975", "3133", "2235", "495", "1676", "1789", "4796", "4331", "3685", "807", "2168", "3900", "1432", "2192", "88", "1301", "3893", "361", "1804", "4305", "3199", "2432", "5879", "606", "3216", "1656", "57", "1493", "956", "6258", "4780", "5200", "2863", "2007", "1553", "747", "4128", "5039", "5262", "4872", "6233", "1211", "160", "4298", "5218", "5169", "763", "271", "374", "1802", "2321", "2498", "6084", "509", "2049", "5326", "4018", "349", "1198", "6254", "1378", "4632", "3860", "4308", "5430", "2274", "5104", "610", "5164", "1686", "4577", "3766", "4890", "5697", "5053", "6223", "4952", "3352", "1126", "2789", "198", "808", "1499", "5256", "5237", "3754", "2094", "2079"]]} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets-test-split.txt b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets-test-split.txt new file mode 100644 index 0000000..3792a10 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets-test-split.txt @@ -0,0 +1 @@ +{"xvalid": ["int", ["253", "834", "737", "45", "373", "390", "961", "592", "41", "217", "245", "1039", "67", "3", "121", "71", "407", "786", "511", "504", "514", "659", "521", "524", "173", "611", "1023", "636", "113", "148", "283", "174", "1074", "176", "387", "285", "915", "213", "699", "916", "382", "66", "550", "496", "692", "732", "709", "633", "538", "911", "735", "1058", "980", "923", "795", "238", "769", "75", "225", "107", "200", "763", "1046", "264", "315", "838", "109", "259", "427", "74", "221", "275", "1022", "436", "596", "586", "456", "805", "568", "989", "949", "139", "231", "557", "237", "352", "499", "994", "877", "268", "531", "269", "672", "476", "68", "2", "1052", "330", "828", "56", "188", "371", "487", "548", "497", "710", "501", "1040", "912", "314", "651", "728", "25", "1069", "969", "17", "284", "554", "816", "272", "809", "1078", "904", "997", "578", "662", "1037", "270", "664", "933", "1071", "823", "761", "945", "723", "631", "687", "306", "1000", "60", "953", "30", "190", "693", "725", "1094", "972", "372", "775", "469", "897", "785", "119", "178", "122", "526", "442", "304", "396", "566", "438", "963", "444", "581", "1017", "235", "905", "1106", "33", "132", "463", "398", "564", "787", "1056", "547", "579", "655", "942", "136", "558", "485", "772", "451", "717", "1065", "567", "830", "6", "676", "607", "776", "12", "991", "508", "807", "529", "599", "832", "1009", "509", "517", "484", "116", "756", "334", "936", "695", "661", "827", "288", "891", "652", "301", "629", "992", "311", "171", "203", "44", "919", "433", "464", "189", "958", "343", "197", "495", "180", "884", "500", "1042", "353", "612", "572", "748", "298", "588", "903", "215", "713", "1007", "808", "560", "694", "921", "266", "902", "929", "532", "159", "1079", "533", "715", "783", "752", "906", "701", "1", "1103", "917", "94", "482", "289", "38", "543", "1081", "404", "434", "309", "129", "854", "341", "418", "474", "999", "962", "342", "815", "143", "262", "313", "47", "424", "62", "803", "546", "1031", "97", "751", "489", "893", "328", "105", "307", "677", "123", "733", "552", "523", "369", "324", "1108", "1041", "479", "1033", "439", "773", "872", "46", "971", "978", "103", "257", "843", "1002", "1076", "700", "615", "868", "55", "1092", "817", "849", "458", "106", "767", "571", "594", "755", "683", "593", "93", "605", "19", "951", "117", "134", "426", "169", "300", "928", "437", "535", "987", "29", "401", "448", "399", "973", "707", "147", "595", "126", "839", "825", "862", "1098", "758", "336", "684", "539", "757", "124", "52", "580", "177", "1088", "377", "616", "696", "689", "505", "858", "940", "455", "234", "214", "81", "865", "1021", "880", "818", "551", "966", "39", "1096", "1003", "506", "42", "201", "127", "412", "591", "393", "1029", "114", "996", "671", "410", "467", "645", "965", "765", "943", "470", "831", "254", "250", "453", "762", "840", "429", "914", "462", "950", "859", "770", "913", "674", "583", "1101", "974", "69", "286", "861", "883", "525", "445", "1030", "175", "70", "1044", "43", "967", "211", "386", "556", "450", "954", "779", "811", "416", "1035", "98", "549", "425", "187", "61", "754", "486", "918", "447", "461", "368", "908", "603", "364", "440", "959", "381", "920", "598", "204", "582", "22", "1015", "394", "120", "305", "419", "553", "54", "310", "277", "281", "16", "627", "415", "1026", "1085", "488", "88", "856", "888", "635", "857", "657", "243", "15", "1047", "1012", "988", "1063", "513", "379", "13", "273", "1018", "79", "507", "584", "829", "166", "92", "423", "1005", "1010", "14", "870", "620", "845", "57", "741", "976", "291", "146", "796", "673", "680", "609", "851", "252", "1075", "494", "293", "822", "545", "714", "360", "80", "937", "647", "51", "774", "167", "384", "800", "338", "1008", "1011", "320", "938", "738", "990", "90", "388", "925", "844", "472", "49", "1064", "590", "722"]], "xtest": ["int", ["934", "577", "926", "760", "297", "383", "1004", "931", "939", "1067", "229", "1019", "984", "628", "1072", "84", "361", "73", "332", "824", "316", "909", "292", "736", "894", "216", "794", "716", "375", "866", "184", "983", "179", "255", "348", "335", "744", "194", "901", "191", "280", "65", "730", "417", "791", "536", "871", "512", "806", "948", "59", "363", "492", "7", "156", "402", "327", "624", "18", "4", "378", "302", "960", "638", "640", "955", "750", "565", "370", "483", "970", "354", "747", "76", "345", "530", "1050", "667", "907", "435", "21", "1054", "688", "630", "478", "267", "898", "1049", "812", "118", "1084", "1001", "261", "814", "230", "321", "648", "705", "819", "1104", "702", "36", "63", "128", "347", "274", "561", "782", "781", "232", "703", "452", "1014", "248", "359", "104", "138", "739", "947", "125", "172", "1034", "137", "473", "986", "932", "276", "83", "642", "205", "385", "679", "678", "130", "520", "643", "704", "96", "1086", "28", "323", "797", "27", "220", "35", "869", "623", "58", "663", "23", "77", "731", "344", "544", "1077", "185", "764", "226", "326", "742", "977", "685", "813", "165", "621", "101", "115", "793", "162", "892", "576", "802", "924", "218", "639", "365", "244", "601", "542", "585", "192", "196", "613", "600", "100", "493", "534", "350", "930", "1093", "879", "975", "376", "656", "183", "1027", "675", "874", "711", "863", "31", "153", "89", "968", "279", "271", "430", "608", "618", "522", "510", "540", "465", "789", "537", "131", "778", "208", "133", "669", "686", "158", "570", "527", "362", "1013", "964", "299", "563", "956", "331", "222", "1016", "227", "706", "698", "468", "199", "206", "163", "1053", "867", "836", "263", "294", "575", "10", "1089", "397", "935", "847", "374", "233", "574", "569", "150", "768", "161", "885", "351", "666", "168", "108", "422", "641", "413", "941", "9", "198", "24", "1102", "207", "26", "247", "708", "518", "278", "290", "406", "164", "349", "841", "366", "637", "287", "910", "339", "0", "622", "889", "295", "195", "649", "876", "48", "236", "927", "626", "792", "993", "258", "1061", "820", "1095", "878", "357", "431", "1059", "799", "753", "855", "810", "900", "219", "614", "256", "1100", "20", "860", "712", "503", "333", "587", "668", "589", "597", "1105", "346", "317", "155", "606", "864", "40", "471", "170", "466", "1073", "160", "985", "788", "421", "392", "632", "875", "1028", "541", "562", "318", "850", "790", "239", "995", "1082", "653", "826", "240", "682", "78", "1060", "142", "457", "112", "981", "395", "833", "718", "658", "743", "265", "319", "95", "409", "1038", "690", "91", "477", "459", "691", "1032", "355", "670", "719", "246", "228", "1025", "853", "5", "490", "420", "340", "881", "1024", "610", "145", "1107", "480", "428", "835", "475", "86", "882", "650", "952", "282", "181", "1080", "260", "411", "209", "296", "798", "1090", "391", "37", "400", "982", "777", "727", "251", "99", "1070", "102", "491", "241", "979", "720", "922", "766", "443", "367", "654", "848", "34", "182", "202", "8", "852", "481", "1097", "224", "729", "998", "780", "502", "734", "625", "784", "515", "141", "644", "210", "1087", "740", "1051", "746", "414", "890", "660", "446", "555", "837", "87", "681", "110", "842", "380", "449", "724", "1091", "157", "50", "193", "405", "498", "887", "619", "886", "873", "358", "135", "403", "617", "242", "846", "329", "140", "821", "759", "249", "1006", "559", "1055", "441", "899", "697", "957", "745", "454", "1066", "1043", "646", "64", "528", "1045", "85", "1068", "32", "111", "634", "771", "749", "322", "82", "946", "944", "896", "149", "212", "721", "516", "804", "72", "602", "432", "308", "356", "186", "11", "152", "1020", "1083", "1099", "389", "312", "801", "325", "144", "460", "726", "1062", "53", "1048", "303", "1036", "1057", "604", "337", "151", "154", "223", "573", "665", "519", "895", "408"]]} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets.config b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets.config new file mode 100644 index 0000000..b681784 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/pets.config @@ -0,0 +1,13 @@ +{ + "scheduler": ["str", "cos"], + "eta_min" : ["float", "0.0"], + "epochs" : ["int", "200"], + "warmup" : ["int", "0"], + "optim" : ["str", "SGD"], + "LR" : ["float", "0.1"], + "decay" : ["float", "0.0005"], + "momentum" : ["float", "0.9"], + "nesterov" : ["bool", "1"], + "criterion": ["str", "Softmax"], + "batch_size": ["int", "256"] +} diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn-split.txt b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn-split.txt new file mode 100644 index 0000000..900f9ca --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn-split.txt @@ -0,0 +1 @@ +{"train": ["int", ["46692", "63322", "22278", "34546", "51016", "65663", "32396", "61118", "67588", "61927", "41653", "34445", "45341", "34474", "30044", "65543", "64296", "24247", "28223", "54951", "7086", "3344", "65544", "57959", "2793", "13348", "38852", "38973", "61390", "38433", "42827", "37751", "21130", "29843", "33870", "38905", "1112", "53200", "69372", "6617", "50871", "9639", "5537", "36575", "39983", "31938", "35264", "30473", "16072", "39486", "27091", "61447", "68934", "51813", "73159", "2990", "33773", "67028", "41033", "71838", "38365", "2531", "23573", "65200", "42751", "46358", "35692", "51932", "36771", "63597", "32707", "70888", "27849", "39975", "46571", "52978", "26162", "12439", "43351", "45507", "15878", "12087", "41316", "25375", "42809", "5520", "38007", "32421", "37915", "44742", "843", "27355", "32407", "66276", "46841", "49864", "45413", "41687", "45777", "23920", "56825", "53115", "23124", "57713", "22823", "27785", "5995", "39110", "42860", "26561", "68430", "54362", "36096", "48619", "11777", "52806", "72878", "12459", "36695", "7924", "787", "29375", "49860", "27857", "67157", "21354", "29585", "50875", "52253", "67390", "53346", "2832", "31120", "45994", "18487", "48012", "61885", "72106", "42253", "71651", "44334", "45376", "68682", "2957", "35104", "45681", "49753", "4232", "6022", "16280", "22740", "30986", "63643", "13517", "23660", "60462", "16047", "27819", "19865", "6681", "16714", "59934", "63756", "42871", "32026", "69854", "26154", "41131", "49196", "48877", "30720", "69456", "14456", "30178", "59507", "40378", "47023", "52844", "19954", "67633", "5515", "41974", "42306", "13192", "25377", "37653", "18058", "61268", "14164", "4572", "20789", "39163", "23488", "33995", "48417", "40818", "40752", "37699", "54186", "13715", "51493", "50888", "72498", "17134", "59776", "12306", "63031", "13373", "42085", "65492", "47141", "16953", "27799", "47433", "44285", "44278", "15009", "65007", "15460", "12727", "63060", "16730", "400", "53580", "57297", "55130", "68971", "40375", "13748", "60862", "4178", "17720", "41268", "40732", "31306", "71488", "29469", "17157", "11947", "30877", "47302", "68945", "16202", "34821", "18598", "33670", "12078", "58131", "73082", "10476", "46435", "67653", "49187", "46883", "53096", "23351", "1239", "63132", "43214", "53662", "47915", "3233", "4811", "37110", "69588", "25782", "6999", "30270", "61073", "62485", "17421", "9600", "67476", "53051", "17305", "4553", "26833", "6640", "22686", "19541", "26735", "70051", "6505", "26777", "11398", "16199", "52216", "71280", "62717", "37532", "71774", "48359", "47289", "30402", "39123", "5309", "19676", "8058", "29044", "61148", "38281", "38398", "39962", "42124", "30805", "51482", "51568", "9349", "51960", "56409", "31278", "7139", "44671", "43058", "60330", "48902", "39047", "55359", "15480", "72553", "64583", "28073", "3395", "34918", "42355", "35902", "24567", "53936", "30528", "66491", "8755", "3011", "70723", "9012", "25549", "19661", "28925", "72608", "59749", "38961", "53287", "24344", "56977", "475", "42137", "56089", "30797", "54692", "1114", "14004", "73184", "7699", "30759", "70519", "34020", "57654", "67210", "25244", "16568", "23276", "64680", "56906", "57032", "45530", "19231", "22647", "40398", "56176", "37253", "2477", "17790", "62674", "1603", "16956", "5940", "16788", "34473", "16892", "60684", "34884", "25555", "7044", "60912", "32591", "49178", "45064", "33335", "72536", "51979", "34100", "29301", "60363", "1875", "49206", "11938", "13261", "27875", "14561", "28646", "60488", "34706", "9655", "55050", "37403", "24523", "25627", "13194", "21780", "12298", "41591", "15935", "21542", "22458", "56380", "11981", "73185", "46537", "49730", "56848", "61933", "2860", "71951", "56467", "15161", "55607", "67239", "55624", "70384", "54736", "32823", "63082", "64249", "47652", "8075", "66714", "50224", "63411", "48042", "49209", "50338", "45066", "18064", "67745", "4373", "32819", "16054", "815", "18558", "37590", "59169", "64078", "2813", "28701", "29293", "20608", "10915", "66265", "72607", "47168", "35143", "41560", "22275", "29778", "38621", "56201", "1230", "71432", "54660", "55544", "59608", "12584", "43293", "51807", "11160", "70845", "6233", "5631", "27924", "16014", "11839", "15711", "66711", "2995", "35911", "62344", "6461", "42256", "19194", "57748", "39021", "69531", "21725", "7281", "10337", "53825", "66743", "35718", "50288", "41796", "7527", "39318", "47438", "37506", "22813", "40249", "5420", "42703", "37100", "13330", "7757", "7106", "28895", "21105", "5756", "62963", "6097", "22051", "7503", "6420", "22341", "28889", "31655", "24824", "11314", "73144", "7903", "16299", "35931", "28920", "53502", "8095", "50384", "2953", "23000", "67815", "31504", "12848", "7764", "18272", "68295", "841", "69277", "40198", "55650", "44581", "59466", "56607", "39815", "26182", "8438", "30816", "23599", "68692", "30201", "26856", "59709", "43506", "72237", "31927", "32127", "15355", "56416", "55283", "59016", "39953", "23054", "61568", "31818", "7767", "64136", "14728", "47300", "15975", "30951", "31688", "25684", "72881", "54094", "55178", "67265", "32838", "2500", "34913", "67575", "41516", "23589", "31182", "59621", "66025", "71462", "40134", "17366", "10805", "45836", "61264", "28116", "2008", "31437", "47376", "35543", "20317", "36146", "6037", "29021", "21667", "39317", "36964", "44377", "870", "56205", "37334", "28446", "15897", "7229", "19728", "12589", "50730", "55157", "4945", "63710", "12933", "24715", "40662", "24613", "8855", "62896", "13805", "54869", "66290", "24108", "24501", "7249", "50075", "39190", "37872", "27534", "34902", "65791", "19366", "4149", "44201", "31522", "21233", "63629", "70204", "51516", "71768", "53429", "27151", "20260", "3162", "4323", "38192", "13035", "47133", "67080", "61808", "72746", "26440", "7128", "59945", "50706", "55214", "72542", "39314", "66039", "58867", "67736", "13913", "10124", "72156", "35436", "56471", "68478", "10455", "53727", "4783", "51741", "34233", "20268", "64300", "48451", "30087", "34216", "66960", "12605", "1590", "32375", "9115", "44882", "72258", "3509", "54328", "50113", "51642", "60924", "34058", "49108", "35945", "18363", "54426", "37667", "26483", "48938", "17107", "4668", "41574", "49939", "20022", "22889", "69823", "31659", "50615", "62045", "7726", "47213", "30753", "3007", "60713", "31061", "13620", "60300", "36140", "15444", "57849", "28374", "28955", "21704", "22918", "41303", "39003", "46437", "32496", "65904", "38393", "72471", "26275", "62215", "57516", "62104", "4147", "47976", "11333", "16672", "61381", "2004", "67810", "51156", "42628", "23267", "15320", "1716", "64566", "36133", "35932", "42445", "50879", "16978", "68815", "55369", "247", "35496", "35409", "7272", "54273", "12323", "21103", "10892", "69593", "27589", "19868", "1379", "29161", "16857", "10959", "17586", "3771", "16277", "44168", "898", "73117", "12054", "65866", "41192", "48480", "53622", "57644", "1705", "72370", "18063", "26444", "5828", "23268", "66494", "41022", "61649", "32595", "2563", "36753", "43140", "47732", "29170", "71205", "54304", "2298", "22104", "18548", "2234", "22074", "22213", "53990", "4917", "3608", "71037", "30575", "39557", "55151", "42655", "19164", "32928", "10784", "35460", "45135", "32244", "7508", "70596", "33395", "57128", "923", "52699", "64774", "19131", "64008", "61805", "34248", "45849", "60571", "18361", "56797", "11735", "2423", "37966", "2774", "34885", "14988", "17272", "61799", "63692", "9589", "3617", "63919", "5920", "54875", "21504", "40508", "19549", "72899", "66140", "12600", "37839", "44018", "8178", "33095", "43830", "16609", "64967", "18239", "45325", "62617", "49738", "63704", "9433", "12374", "71152", "26015", "38513", "58323", "50053", "26372", "11270", "57565", "1868", "18876", "25239", "71881", "50367", "41878", "35236", "43927", "30807", "547", "66765", "26988", "25560", "2472", "2485", "63507", "8837", "63989", "24431", "4872", "21969", "60097", "59355", "12703", "70226", "47402", "2652", "32093", "41925", "64988", "11656", "71629", "57320", "44663", "34476", "67420", "28174", "37148", "23354", "72134", "54400", "32269", "47445", "49616", "65487", "21453", "61270", "11833", "3701", "24641", "54254", "15719", "27281", "3045", "66399", "64277", "38467", "2273", "24268", "53003", "7214", "44383", "46372", "16538", "5492", "8676", "1935", "41160", "29114", "57187", "50548", "54615", "50312", "44567", "5917", "17548", "10821", "55346", "24494", "40625", "64067", "11281", "49139", "13468", "67467", "69066", "22017", "13610", "15489", "46747", "23398", "56696", "20117", "47690", "49710", "64563", "48382", "52225", "27992", "20372", "16192", "2969", "23018", "62251", "19276", "673", "1021", "32411", "7162", "56275", "1780", "58321", "49412", "25703", "24650", "29401", "43155", "33634", "60110", "44069", "33895", "67812", "52441", "71538", "67933", "1127", "3392", "30704", "66302", "20831", "30345", "41787", "7372", "53706", "28352", "64023", "18654", "2708", "37387", "58654", "36151", "46706", "72598", "63446", "20221", "61556", "69421", "46308", "19089", "72844", "28826", "65304", "13793", "7424", "12129", "9320", "28274", "54599", "20845", "22882", "17912", "47983", "70371", "35093", "18565", "47984", "2370", "63", "2925", "38140", "47096", "4556", "53494", "66725", "10431", "16882", "17971", "23687", "25760", "25357", "16593", "17802", "69446", "54622", "36618", "65452", "14614", "5927", "70341", "48922", "50806", "49225", "43671", "5477", "18156", "22415", "2983", "25482", "48230", "61559", "51903", "21930", "18385", "27915", "56756", "25988", "64186", "5732", "16026", "60168", "54424", "65625", "2455", "18376", "42571", "28355", "14717", "1969", "16416", "48069", "17044", "29019", "26377", "68995", "67468", "12534", "9256", "57916", "54553", "46931", "18849", "8618", "34296", "50060", "53516", "50545", "56407", "42743", "60865", "46878", "28111", "62143", "51018", "45757", "45006", "70822", "59314", "44794", "9156", "58262", "54729", "8684", "64747", "33611", "62356", "35758", "13760", "35111", "3484", "18600", "57730", "22586", "4016", "66219", "72976", "65031", "43704", "53924", "15112", "61688", "10897", "56872", "62812", "38229", "11060", "25462", "59832", "43628", "70443", "60991", "41534", "59536", "8742", "19896", "62586", "21568", "55971", "24254", "1001", "61804", "68668", "51319", "67505", "15122", "62521", "37549", "61208", "4454", "39961", "49720", "1234", "35986", "52932", "64099", "1368", "27409", "70295", "43794", "65601", "53985", "28185", "63183", "23693", "16598", "54893", "60401", "25599", "20907", "13059", "60461", "54040", "46780", "68856", "24149", "24476", "14843", "37247", "49347", "18022", "20793", "37133", "11388", "36540", "46752", "9443", "7682", "6078", "53093", "30006", "51528", "34661", "62891", "10857", "18686", "22774", "52072", "70935", "43150", "15961", "61332", "14036", "69577", "41825", "15500", "61335", "38066", "2989", "34929", "57890", "16935", "43587", "53178", "3224", "10078", "16729", "22131", "53213", "52468", "15828", "21953", "45027", "60173", "43581", "63508", "64542", "17414", "18299", "15988", "19382", "22643", "26193", "14145", "56863", "72095", "39886", "52065", "33379", "21642", "5680", "25839", "49451", "15721", "70182", "16313", "40979", "69863", "27832", "39492", "35592", "39521", "72296", "36944", "18266", "58825", "29131", "18013", "60174", "28296", "24724", "20646", "15588", "60551", "14485", "9654", "13520", "65985", "14769", "58347", "1480", "56752", "22966", "70057", "29514", "8266", "35646", "17734", "29271", "2130", "2745", "30600", "16343", "70465", "7400", "60151", "20761", "14981", "7359", "56027", "28096", "33176", "24450", "16664", "21235", "42766", "20629", "26259", "19311", "51423", "58181", "15433", "72983", "35272", "34140", "55195", "34407", "59105", "28812", "39379", "3248", "34723", "18504", "3297", "64416", "68528", "27807", "5768", "52254", "84", "61666", "62522", "53000", "1171", "31431", "73169", "57926", "34341", "34154", "45767", "12971", "15170", "58873", "17304", "39548", "20961", "24178", "578", "48927", "34682", "3939", "69628", "56527", "1639", "56884", "22572", "71869", "24340", "485", "20434", "4634", "68397", "19387", "29635", "62571", "72125", "15118", "9714", "70495", "62071", "62444", "46094", "53382", "52525", "22339", "7799", "70275", "63267", "57173", "50853", "19030", "69193", "43111", "51248", "59615", "25444", "34726", "170", "3559", "8877", "8220", "33401", "19692", "30428", "59373", "44224", "52545", "46256", "33902", "33389", "65324", "71970", "63024", "42596", "57012", "19239", "24604", "48284", "4461", "60918", "28312", "44649", "63440", "5674", "57745", "9869", "1970", "55685", "24016", "11944", "39705", "70223", "63113", "45739", "17291", "72045", "42226", "13600", "40073", "10007", "69133", "20327", "1623", "52150", "37448", "19582", "40536", "17969", "66794", "1349", "29042", "5431", "47070", "41946", "23711", "33422", "15181", "57016", "44293", "9766", "47618", "20056", "57122", "15371", "13293", "790", "48646", "66210", "8943", "64888", "20154", "71878", "68721", "47194", "70762", "28885", "2400", "34418", "49778", "59142", "28973", "27377", "30100", "23572", "39503", "6489", "61862", "24436", "28913", "48460", "13253", "69952", "6531", "14954", "60288", "9539", "15434", "23709", "15255", "19883", "60897", "19168", "65569", "65605", "65057", "24105", "25913", "68660", "12562", "27303", "5475", "6299", "45816", "60514", "45668", "4321", "52523", "53815", "18554", "66216", "10963", "29222", "41528", "29523", "61306", "71609", "62032", "51133", "31755", "7782", "39330", "44543", "39157", "5541", "72288", "34493", "48009", "71378", "43418", "6689", "52406", "27081", "25727", "58356", "33765", "73254", "3404", "60543", "47752", "25284", "16287", "15524", "21236", "73066", "25814", "41421", "59148", "43290", "57225", "42385", "67142", "10146", "48893", "61530", "61408", "2557", "2733", "22492", "50183", "20797", "10044", "66418", "20296", "15398", "12661", "6867", "10424", "42847", "12959", "26056", "62132", "25362", "35418", "41366", "5579", "31654", "45109", "44610", "34120", "16058", "15478", "69991", "9086", "39233", "18885", "8000", "9458", "46052", "20443", "30148", "43410", "45098", "66769", "69096", "2920", "13265", "65330", "35240", "18005", "1902", "17201", "62037", "50707", "69986", "25862", "37505", "26714", "32083", "46039", "17071", "66413", "4794", "43330", "27095", "16293", "62691", "30858", "53457", "40944", "52075", "67621", "52889", "23630", "55563", "55264", "28326", "64390", "34687", "37346", "37196", "72587", "42814", "1802", "39790", "38046", "4112", "69299", "59153", "25741", "296", "4986", "58267", "43596", "33185", "58806", "31411", "38431", "5911", "19445", "39276", "15482", "48051", "59285", "60485", "2065", "59386", "14168", "19875", "62652", "50034", "48014", "72322", "23285", "63030", "66181", "27231", "6044", "29230", "3746", "13465", "31021", "26819", "41670", "20358", "22806", "22324", "53967", "28683", "19623", "23786", "60302", "27880", "55184", "2200", "17210", "11292", "73205", "5715", "31807", "42148", "3536", "63021", "30621", "7740", "48244", "228", "27901", "68404", "35375", "16120", "36727", "33400", "56316", "36679", "8127", "70500", "39994", "15847", "25330", "59042", "34000", "54700", "37720", "53119", "68847", "65738", "5531", "35404", "65146", "55670", "72050", "72252", "24583", "2188", "5507", "46767", "42872", "41338", "66060", "51970", "31176", "6718", "39028", "27251", "22759", "51690", "71105", "61838", "50315", "34562", "36389", "8175", "26317", "42673", "9598", "47530", "22171", "67662", "7436", "56487", "54205", "45923", "34958", "70643", "13936", "49796", "42689", "37244", "5454", "33325", "14147", "19127", "38925", "53731", "32000", "6436", "15158", "49306", "46544", "7333", "52815", "6083", "20756", "49223", "24258", "33165", "294", "54384", "28523", "32582", "68412", "55915", "33684", "1013", "48420", "12564", "14481", "53088", "14259", "37241", "32660", "44446", "58846", "14627", "20009", "71456", "42682", "51388", "53845", "73160", "55744", "42523", "49941", "11743", "68502", "71188", "40691", "12556", "6948", "52998", "29071", "47547", "40151", "64849", "12212", "60", "48137", "3365", "32978", "43931", "60641", "35139", "58688", "27858", "7010", "35067", "3604", "17187", "19437", "59976", "27981", "60180", "46567", "23254", "22518", "11584", "30739", "8879", "32238", "10402", "16195", "7127", "18675", "12353", "10196", "46652", "69477", "55604", "7329", "4803", "66805", "35010", "325", "6015", "30395", "14853", "66496", "70966", "4140", "56141", "58390", "5667", "17346", "2634", "4797", "36063", "53710", "44983", "32557", "36990", "33031", "48755", "8983", "3642", "71269", "46743", "29903", "9141", "6378", "53159", "33834", "62425", "19613", "23249", "62015", "8669", "19252", "60150", "22260", "1797", "57739", "1329", "3829", "5297", "42377", "57203", "55816", "28103", "6950", "58967", "8166", "41280", "29601", "37565", "38220", "53958", "983", "61946", "31673", "3500", "52136", "5744", "36155", "22559", "15271", "625", "62783", "18351", "30822", "4327", "60216", "21815", "57981", "63116", "19192", "50633", "62050", "39824", "65857", "73124", "50713", "56983", "69568", "67073", "28742", "22129", "39525", "65519", "39191", "28133", "17930", "64235", "35689", "48876", "39703", "43947", "37393", "24935", "20335", "19106", "232", "22378", "69044", "55924", "32332", "10029", "17299", "62666", "1883", "52743", "46569", "53319", "38964", "60754", "58723", "42151", "29120", "10057", "24869", "47860", "33537", "17344", "27828", "25625", "22688", "65016", "1252", "10300", "51361", "60992", "22554", "67776", "26390", "64077", "44434", "20250", "26490", "32635", "60285", "45176", "63470", "41059", "58538", "33132", "41885", "24404", "34275", "54570", "53660", "48948", "13979", "9994", "16197", "6047", "38155", "69402", "20133", "65225", "22351", "66528", "71193", "67639", "50372", "72862", "45514", "65343", "61966", "49854", "65161", "25730", "45879", "11872", "22123", "69657", "10561", "65552", "34919", "676", "58896", "29050", "7089", "59327", "9560", "17462", "55708", "11263", "5262", "59372", "14935", "64017", "53765", "65269", "38160", "39918", "23831", "49655", "66923", "70631", "30182", "24754", "34317", "42888", "43848", "40358", "3747", "28", "66259", "3501", "48054", "46495", "6035", "39391", "33002", "50213", "23361", "15079", "42927", "4696", "29346", "51554", "4581", "11862", "2103", "71638", "28876", "7794", "58383", "50352", "59993", "51656", "25909", "4119", "37768", "48778", "58824", "54663", "2869", "6485", "30992", "40708", "51616", "46808", "40179", "52550", "63967", "48199", "71237", "67085", "47514", "21609", "54852", "34469", "2444", "57727", "51720", "31304", "17943", "27644", "55165", "5106", "11140", "67333", "33192", "68120", "66635", "38863", "31130", "13394", "25118", "33822", "60786", "36988", "36963", "57377", "46934", "8993", "24852", "17140", "845", "205", "17997", "63250", "5388", "60127", "35027", "8008", "73084", "45747", "2450", "60554", "35526", "54367", "15542", "22206", "80", "26717", "8916", "52213", "36435", "30784", "26227", "4879", "3288", "44328", "50740", "765", "57828", "38570", "38294", "45304", "39780", "4378", "35005", "66026", "49736", "44407", "51650", "64968", "20967", "48159", "24388", "40899", "57098", "19988", "14924", "45756", "65798", "20123", "68992", "37080", "31257", "31970", "403", "11581", "3111", "25131", "50323", "57039", "20677", "990", "9297", "73058", "30157", "6549", "15400", "61814", "23395", "40471", "48817", "69416", "17986", "62206", "24609", "50469", "9834", "68950", "47330", "49276", "42993", "65570", "26593", "8829", "12368", "57092", "41423", "28077", "55297", "64286", "52574", "11572", "17325", "67921", "31976", "6700", "14046", "56995", "50791", "1852", "14960", "1986", "1055", "65272", "40213", "66112", "69636", "41510", "34076", "63940", "21298", "56359", "12408", "58429", "73116", "63995", "14310", "45664", "62774", "39888", "42915", "40792", "55383", "4359", "38919", "62046", "56948", "55780", "19925", "57680", "35833", "4897", "45139", "10766", "35926", "63532", "54468", "66739", "22754", "60265", "23326", "94", "55516", "39382", "3", "71039", "55730", "38014", "29083", "42732", "68325", "11611", "58767", "36657", "40972", "9474", "56665", "13159", "64222", "41778", "15368", "28125", "63094", "65653", "62861", "45345", "19007", "5772", "29036", "36188", "22268", "56911", "49964", "63510", "3903", "37266", "69685", "65690", "41064", "45979", "72937", "46641", "4793", "21330", "25618", "19744", "65640", "59232", "44618", "60277", "54818", "46262", "53790", "42264", "60252", "235", "31822", "39529", "60805", "72804", "37478", "47166", "24323", "65709", "65189", "7710", "50051", "32260", "8589", "72691", "16040", "22293", "68881", "59030", "63505", "5665", "62550", "36380", "20784", "39045", "23194", "34586", "6756", "59797", "18582", "68734", "46273", "46974", "43820", "4528", "51477", "40257", "10144", "71506", "5340", "72634", "60982", "20069", "9944", "7497", "52342", "38846", "55934", "39844", "64370", "47923", "48298", "69268", "17216", "35101", "50835", "58919", "61599", "68460", "6923", "64002", "56174", "37791", "31057", "43051", "13364", "22141", "70680", "45025", "12932", "51332", "1128", "10210", "72869", "28996", "8303", "2536", "56521", "72599", "41948", "70066", "48028", "68459", "45424", "68324", "24032", "17509", "30427", "61058", "32135", "72697", "61671", "56954", "60034", "26093", "4109", "26542", "5648", "27087", "54837", "2643", "11940", "60925", "49271", "43173", "36287", "34145", "8967", "49868", "57439", "7663", "58375", "42121", "36520", "55933", "20694", "67489", "7972", "16889", "43461", "4245", "22084", "42005", "68256", "33166", "28634", "26581", "73099", "12965", "58683", "51772", "21289", "48986", "32003", "14643", "11499", "12573", "47421", "41762", "44973", "61811", "4701", "25260", "69598", "20812", "32569", "41438", "44598", "60512", "50235", "72641", "70975", "48034", "47227", "71060", "61573", "1309", "52275", "62498", "41522", "65186", "41770", "49831", "48729", "37281", "2949", "27604", "57425", "45569", "22879", "11505", "51918", "70023", "55646", "46199", "57969", "26501", "71874", "71215", "27213", "42944", "38650", "6887", "38986", "51830", "13395", "3357", "7080", "29841", "71880", "58052", "72177", "36610", "50852", "52944", "62743", "21889", "67135", "53759", "50514", "40175", "5419", "56276", "53470", "52678", "67542", "72813", "10765", "39641", "35135", "17037", "42321", "69053", "1494", "7824", "32246", "38966", "46891", "30522", "48898", "9664", "23450", "60622", "36414", "46119", "59123", "61618", "43831", "54077", "64091", "3207", "4015", "48208", "15796", "3459", "53618", "23173", "40465", "50226", "71220", "59509", "5052", "15402", "19280", "35655", "37097", "2837", "21333", "57030", "32518", "29147", "42185", "1701", "4345", "69487", "13267", "7695", "35454", "34365", "36176", "31273", "65268", "9665", "43460", "70092", "47730", "23512", "54707", "15819", "44556", "56223", "42359", "59092", "62250", "43756", "63423", "37842", "72070", "54477", "40417", "11525", "14207", "14768", "4157", "62529", "56478", "26874", "35147", "53339", "16337", "43422", "61740", "14102", "73076", "62061", "69091", "68722", "56803", "21003", "43666", "36090", "31506", "39904", "18416", "12380", "30593", "30310", "69009", "34389", "41317", "48676", "10620", "9999", "15795", "72447", "45853", "67556", "43139", "42079", "37172", "41079", "46345", "4326", "72950", "64140", "14315", "16055", "25069", "5342", "16832", "4192", "50076", "25443", "18761", "19584", "5345", "35357", "11790", "10111", "70801", "65909", "62317", "44406", "54886", "73240", "60397", "37296", "22527", "17251", "1904", "39506", "60220", "26846", "47354", "26618", "2389", "31169", "39720", "39766", "54341", "62656", "36136", "2097", "52997", "64564", "44309", "33840", "73186", "38462", "14789", "19640", "20183", "39717", "14289", "65119", "48006", "41819", "34505", "2353", "18928", "25318", "35128", "11063", "9261", "8889", "42554", "41919", "25567", "32036", "41956", "16460", "27564", "35867", "72200", "27860", "20974", "69654", "1461", "55391", "39861", "65291", "20181", "7835", "57661", "62791", "9153", "20610", "12373", "42239", "1346", "2559", "72442", "932", "67173", "4725", "4633", "17683", "38653", "17", "43806", "46617", "72272", "66147", "54855", "12862", "29984", "71322", "45905", "20673", "44474", "10824", "36979", "72832", "39205", "49615", "14465", "52245", "35411", "45766", "42957", "57511", "25053", "44301", "73177", "22071", "39016", "43308", "6007", "20926", "53932", "67181", "73081", "8857", "30872", "48704", "72452", "55916", "3129", "18702", "72817", "56429", "21152", "14434", "69313", "25461", "45171", "66333", "71289", "60426", "49294", "8432", "20211", "5730", "50971", "17515", "10242", "4813", "22247", "34611", "7398", "2993", "13405", "37600", "2706", "55107", "20362", "61294", "4088", "66764", "71156", "71367", "41576", "31225", "62910", "50928", "14552", "8788", "7175", "6860", "2436", "9418", "72391", "12048", "42044", "2679", "13705", "14743", "36152", "21692", "56699", "21229", "47081", "55091", "68279", "42356", "7926", "40234", "16060", "64042", "10783", "41558", "58774", "31959", "44066", "24769", "66667", "31984", "23805", "73252", "56491", "13655", "30882", "43084", "26759", "1401", "41737", "23774", "56381", "26808", "12924", "18899", "14752", "57946", "32900", "60759", "71109", "30841", "36012", "12155", "19921", "54385", "3951", "21714", "34730", "64605", "51390", "20323", "52235", "50041", "6297", "47601", "7416", "34323", "39813", "42213", "11679", "41861", "29758", "43414", "36973", "69367", "34766", "12779", "12894", "57575", "19335", "43182", "9878", "22613", "66791", "11338", "64322", "6419", "10662", "17887", "51193", "22717", "64045", "18423", "24414", "45463", "48155", "24670", "63143", "40242", "42510", "6858", "29865", "53810", "38606", "70979", "25622", "71504", "11470", "60632", "47955", "10834", "52022", "47199", "11178", "65475", "5747", "56386", "66581", "72225", "13642", "37231", "25353", "22507", "46029", "29717", "54393", "55990", "14253", "46089", "66106", "57417", "22616", "36483", "18523", "40148", "65750", "39505", "73222", "10112", "38733", "21537", "47304", "16209", "13058", "34337", "40363", "71421", "1131", "25395", "2744", "26692", "24829", "5519", "10748", "14416", "25437", "58413", "63090", "10340", "8263", "38980", "31051", "16717", "26880", "60895", "57587", "35972", "28311", "68241", "57399", "66483", "40806", "33075", "66395", "26258", "44337", "39424", "23111", "34168", "19702", "18896", "26232", "41081", "42091", "45958", "62584", "14519", "4800", "2252", "57687", "21813", "59351", "5096", "4288", "69961", "28281", "67385", "22204", "51342", "39669", "68023", "21177", "9419", "18150", "6862", "42709", "18151", "41077", "30289", "35929", "65730", "23140", "70363", "73168", "17886", "44909", "11587", "65886", "18177", "71344", "55578", "70786", "39877", "17556", "36912", "35305", "58831", "33951", "37000", "35165", "55170", "49482", "13972", "17999", "59371", "783", "71194", "3679", "16389", "58668", "55577", "12443", "46061", "11907", "61221", "1847", "14808", "70750", "25225", "10671", "8176", "39952", "71401", "17821", "13085", "72403", "3383", "1286", "21034", "68562", "65011", "35018", "20723", "40948", "17500", "62626", "49800", "14512", "25343", "52746", "624", "22438", "66545", "41230", "1073", "33180", "59671", "73091", "51891", "51905", "2425", "15343", "17812", "52900", "26704", "26768", "64018", "60770", "42131", "72559", "17713", "63273", "50369", "2419", "42463", "23725", "24649", "190", "21162", "69721", "9238", "52871", "72386", "72571", "30717", "56387", "16466", "38298", "38731", "37676", "48183", "13378", "54128", "30134", "31816", "60284", "9507", "61430", "73062", "57308", "29465", "50881", "57389", "39794", "10251", "46681", "14931", "69051", "23502", "39033", "58178", "4810", "23882", "71884", "51599", "20705", "71458", "2725", "45154", "31985", "8769", "862", "51169", "47474", "49801", "68738", "12557", "36088", "15597", "68002", "13649", "14135", "13834", "59150", "17520", "29182", "38363", "57345", "66793", "11664", "60189", "25864", "47792", "16208", "35953", "51922", "60903", "31395", "19441", "51010", "48593", "20578", "38348", "56912", "71475", "50766", "46744", "58595", "264", "49307", "52460", "58130", "51886", "59738", "58761", "18940", "45087", "26241", "37305", "10668", "7045", "1739", "18346", "72419", "24918", "29105", "51021", "18668", "49438", "20231", "19648", "65327", "7020", "16089", "12077", "14115", "2036", "65383", "5857", "31657", "36465", "47325", "17579", "56829", "41877", "67427", "52646", "23879", "51355", "43521", "48947", "18563", "31317", "33953", "46187", "14152", "61874", "48951", "8249", "6092", "72259", "33458", "71235", "34128", "14509", "57733", "39359", "22886", "47424", "52070", "9395", "7261", "58162", "11537", "63540", "56781", "90", "6091", "55621", "62053", "57330", "25663", "37276", "10190", "32857", "19214", "67754", "71279", "5019", "59602", "19578", "36121", "71368", "41173", "64028", "70442", "26435", "60810", "54558", "45914", "30331", "65728", "42379", "7181", "17516", "2137", "61081", "9189", "30824", "3673", "64866", "25210", "13216", "13199", "3822", "13325", "44499", "39384", "42128", "22190", "70513", "63175", "56307", "46313", "44261", "59650", "66132", "3969", "14286", "60568", "68179", "58575", "36588", "70100", "8659", "34394", "20131", "55060", "10004", "37674", "10572", "11458", "24189", "36316", "48520", "24098", "2555", "15613", "70196", "5269", "37783", "63213", "61609", "59405", "29976", "61138", "37169", "49579", "23211", "64240", "37409", "3697", "17468", "50260", "39946", "35116", "8377", "54930", "23145", "52086", "63827", "5845", "45913", "38323", "21778", "27965", "29938", "42082", "13665", "36589", "69813", "16582", "2060", "13077", "50944", "56923", "15924", "42093", "55810", "51413", "13475", "57283", "56689", "23474", "39199", "64111", "59813", "3666", "57628", "59398", "29406", "8579", "13566", "68307", "71478", "39541", "44083", "61610", "69271", "30868", "22434", "44172", "35163", "42263", "66373", "38053", "49869", "41250", "37675", "43526", "20896", "36779", "17377", "16349", "13341", "54840", "67149", "14819", "30983", "68559", "6813", "34054", "57934", "51724", "39481", "44886", "28824", "37777", "49493", "39868", "62526", "55556", "22302", "20607", "62226", "7874", "24470", "3529", "39175", "55912", "33409", "67009", "46734", "7353", "66541", "1313", "8655", "48429", "18736", "38607", "1337", "26828", "19282", "14195", "31935", "45168", "63087", "13132", "59854", "55721", "44998", "30476", "6029", "59770", "5786", "60659", "33511", "21508", "39849", "7529", "52210", "20479", "8026", "47852", "20591", "67851", "58724", "30313", "38453", "32252", "29452", "32868", "28199", "70968", "46302", "49722", "70735", "23744", "67896", "16274", "16214", "29792", "5992", "43646", "30823", "71191", "12271", "34877", "2158", "24626", "66487", "44249", "40212", "36182", "5849", "9887", "43855", "2426", "33596", "19287", "46979", "67631", "33376", "43682", "23332", "22403", "64672", "2294", "30495", "13155", "59029", "45050", "21545", "60175", "28853", "33322", "38545", "33347", "63972", "69699", "9346", "38396", "14458", "30862", "8506", "29077", "24014", "18763", "53136", "72845", "57721", "35087", "35855", "3310", "3433", "66084", "70743", "15068", "45551", "60790", "46893", "34320", "73120", "23486", "62236", "15155", "45260", "19143", "17948", "1682", "55608", "67236", "63343", "26072", "32362", "23670", "21681", "63896", "38596", "98", "51564", "15774", "23741", "36336", "1403", "65753", "44226", "19765", "4050", "25392", "8887", "30425", "67358", "24504", "7902", "16465", "69407", "30859", "35709", "67042", "45140", "61831", "12733", "36992", "39062", "58708", "66994", "70849", "30871", "16016", "1213", "38364", "61183", "5177", "51445", "63954", "35217", "6185", "9360", "1032", "42186", "7337", "12430", "50990", "40840", "2034", "1121", "54963", "3765", "53919", "73189", "24717", "36091", "45321", "40572", "50125", "57879", "13165", "67958", "51519", "56887", "38911", "22513", "44540", "58589", "2672", "31649", "40183", "41324", "69812", "24397", "65736", "17032", "22236", "62837", "36864", "51271", "62734", "14150", "51196", "45212", "60690", "40794", "56662", "30035", "47255", "3575", "21307", "23969", "37911", "15518", "23130", "50222", "18320", "29576", "24439", "3875", "4478", "51928", "35943", "29556", "70744", "5010", "33024", "35609", "5594", "616", "25399", "72287", "20405", "23128", "13722", "23456", "29503", "32204", "28959", "262", "40935", "52788", "33257", "50838", "31553", "64159", "37459", "52104", "28822", "70774", "38787", "69872", "49147", "3715", "47502", "12224", "15657", "20446", "43708", "22859", "68079", "17358", "4595", "58439", "67545", "15567", "7190", "59733", "62655", "64943", "65676", "5412", "3301", "20341", "13274", "20587", "70940", "44636", "45770", "29479", "1877", "37164", "32087", "3596", "62899", "50516", "23726", "33119", "12754", "2991", "65587", "36941", "36648", "43360", "64202", "5084", "17235", "25418", "20632", "33449", "63744", "24048", "42134", "66891", "51298", "64103", "31286", "60431", "9635", "24950", "67373", "41233", "26040", "53185", "26787", "12037", "45934", "55559", "38704", "41827", "12390", "54549", "34511", "42607", "60439", "14809", "59784", "10104", "40429", "59044", "13289", "16533", "55108", "44391", "56904", "12866", "44459", "3837", "52898", "9817", "50720", "773", "23081", "60247", "9980", "72910", "10392", "7109", "62709", "55463", "3351", "22387", "28663", "39336", "51471", "35076", "47803", "45886", "12039", "70551", "26877", "34793", "28534", "39042", "64248", "38040", "10346", "71328", "8475", "67954", "60587", "65682", "45429", "35458", "48492", "5100", "69888", "2878", "64031", "65996", "58885", "18104", "4837", "64095", "1709", "1392", "27544", "32956", "3742", "59269", "36813", "26953", "54022", "36615", "16793", "28811", "64206", "6569", "24486", "21192", "25535", "51431", "30235", "38461", "56971", "35928", "22532", "12468", "20476", "33912", "54122", "44471", "30870", "44538", "27145", "29885", "17927", "70173", "39881", "68627", "25687", "49791", "38612", "39673", "67581", "15701", "9625", "47461", "61833", "37061", "35841", "29728", "19460", "19498", "66262", "19590", "41348", "690", "53290", "42811", "32416", "40299", "7480", "48056", "52295", "18602", "31619", "916", "4165", "31702", "4944", "31409", "51855", "50293", "10324", "6507", "64474", "54457", "31802", "61635", "64678", "53652", "35020", "70030", "57158", "42466", "26504", "13774", "22673", "25337", "63900", "2270", "46962", "43985", "30598", "4277", "38076", "46884", "71509", "26822", "47357", "41307", "72949", "46998", "18116", "19466", "25930", "10213", "5118", "53907", "63964", "11990", "60578", "41740", "47400", "17256", "25527", "29698", "53441", "2879", "59644", "56998", "20026", "3562", "69862", "45485", "8221", "14396", "39480", "43434", "65842", "59294", "8602", "26716", "73176", "55617", "6908", "50889", "27884", "44468", "53242", "35835", "46161", "32219", "46990", "21403", "5156", "23876", "35916", "71253", "71782", "55330", "59221", "64369", "57886", "54667", "5581", "11460", "30896", "21365", "67796", "68262", "36358", "24060", "17068", "39067", "63500", "27395", "58442", "12204", "19701", "69564", "25407", "39134", "53311", "53905", "50748", "12236", "22348", "35412", "57474", "53409", "48537", "60687", "65685", "29192", "27246", "25581", "44300", "62406", "71727", "30908", "39658", "48377", "56247", "69410", "14443", "58168", "14091", "65278", "61490", "15043", "63383", "13521", "62780", "15101", "62615", "8457", "4320", "12849", "4052", "50456", "15393", "56566", "18956", "57423", "30066", "12375", "24328", "15822", "16995", "34057", "40041", "30394", "41595", "42756", "32125", "15955", "62654", "27545", "40466", "41981", "62246", "147", "51227", "54471", "65611", "62043", "19619", "6366", "33949", "50146", "2422", "16398", "23626", "31024", "15743", "59686", "50593", "19241", "21524", "4421", "21283", "2427", "22579", "47895", "42744", "34679", "28797", "70709", "33502", "23870", "10945", "28160", "33419", "59343", "20659", "55565", "72459", "25204", "59892", "2834", "56742", "24355", "25149", "35888", "18489", "68063", "4705", "9187", "1421", "72956", "45532", "3835", "7122", "64806", "31982", "12602", "41312", "40130", "65437", "30676", "14266", "69055", "42610", "62600", "63924", "69079", "26316", "38425", "66073", "10256", "65614", "31091", "24112", "56789", "9313", "33816", "58010", "4370", "62152", "61723", "57741", "72387", "7019", "29975", "70309", "68226", "11805", "29660", "52488", "19323", "20669", "28820", "50700", "28063", "58552", "61924", "1148", "39732", "4612", "56707", "1657", "63668", "33958", "37537", "69815", "64652", "67248", "63466", "24945", "45877", "32445", "57286", "4083", "5838", "67070", "31690", "28645", "56593", "16081", "10312", "66781", "28803", "55776", "38538", "32389", "25122", "47587", "27554", "60157", "18510", "71500", "45636", "6333", "38488", "43973", "10580", "60442", "70125", "48494", "43090", "31315", "8656", "43288", "55978", "59975", "61040", "70211", "61470", "509", "13089", "36023", "46697", "70350", "33734", "41334", "35651", "20596", "13396", "29790", "15338", "23901", "14997", "7273", "32374", "4076", "18204", "4937", "57717", "23062", "16624", "42577", "30503", "56333", "66425", "41523", "63372", "40133", "33975", "10650", "4964", "21440", "15753", "11147", "15788", "39287", "1322", "24257", "18982", "3062", "35345", "14329", "30654", "58882", "4541", "25934", "21160", "10989", "21718", "1736", "14442", "59532", "33738", "22577", "68417", "10454", "60772", "37233", "36325", "36915", "69174", "33727", "65784", "4286", "53227", "67499", "3194", "41023", "28082", "20927", "40449", "64376", "27336", "55084", "65883", "53366", "37878", "54388", "57163", "37192", "9686", "72269", "20955", "61669", "34603", "39200", "69134", "42443", "62947", "55795", "26888", "59616", "70213", "65128", "34167", "69275", "50014", "431", "47407", "30811", "25228", "3267", "10720", "41167", "68051", "2018", "40831", "68509", "28697", "29623", "71829", "47634", "1605", "45796", "56295", "59875", "31374", "59664", "31076", "57083", "24694", "65378", "64927", "63003", "36128", "641", "69489", "12905", "498", "40688", "31062", "40470", "52695", "2792", "24618", "24364", "56443", "49798", "67436", "7364", "37306", "43838", "2827", "44587", "16608", "2484", "37364", "50123", "68843", "45179", "9666", "48983", "26865", "7842", "13381", "30952", "24027", "23792", "50259", "44813", "60735", "19129", "9958", "62573", "2259", "17627", "70337", "15034", "1931", "8872", "2271", "7818", "46770", "40102", "27514", "71079", "13334", "18315", "49333", "40900", "61613", "22261", "41628", "39140", "30981", "71591", "41820", "43582", "39976", "68073", "44265", "56701", "34004", "6822", "45227", "5425", "24853", "9424", "67380", "58853", "18927", "56517", "32164", "41278", "19729", "63779", "39381", "63163", "24126", "48762", "28010", "48048", "5245", "38245", "36044", "30651", "56149", "35221", "36976", "6912", "28065", "6014", "2369", "67626", "17789", "32567", "19535", "66251", "57522", "62547", "35507", "31456", "38056", "7779", "62513", "32462", "15698", "55752", "10428", "4393", "18757", "38530", "12377", "27088", "25335", "33557", "40674", "43647", "46914", "28643", "54446", "45446", "51998", "12118", "15257", "21805", "56706", "66433", "44437", "1519", "32118", "63428", "45995", "30610", "1649", "43112", "2720", "43331", "17792", "55218", "35896", "39782", "7563", "15354", "23033", "36486", "36006", "11082", "43850", "55450", "42656", "33924", "10086", "19073", "38991", "35315", "63728", "23905", "54524", "60715", "48623", "52644", "5523", "30131", "40609", "41901", "18818", "44451", "19140", "24412", "49452", "43343", "32137", "26816", "68862", "53300", "35386", "38387", "16031", "11468", "55600", "55811", "55304", "63151", "34572", "47196", "51241", "27531", "53925", "16238", "25806", "64552", "47087", "9949", "40582", "42236", "44140", "54588", "31773", "34298", "61665", "21016", "62366", "45578", "70715", "34489", "7147", "24339", "28525", "22360", "39354", "48309", "12797", "61087", "38279", "15549", "59933", "58584", "67620", "52000", "32985", "71982", "50079", "31695", "25117", "5810", "59283", "36289", "22512", "45930", "22332", "14919", "54016", "50084", "43660", "49653", "50712", "51035", "45926", "71584", "17667", "58680", "3163", "23610", "29125", "22249", "64311", "9633", "11009", "53902", "53080", "26009", "65591", "33571", "39696", "56166", "57295", "14676", "72313", "29814", "60411", "64653", "24990", "4099", "2525", "7391", "63185", "20707", "48786", "60594", "36962", "14529", "18838", "69793", "16829", "67394", "704", "57065", "33914", "72008", "17258", "38142", "39677", "64464", "67678", "35006", "39297", "28393", "563", "34751", "71565", "27451", "21918", "48251", "3860", "30254", "54656", "24743", "33565", "13414", "25193", "53943", "59460", "47204", "2532", "20126", "60528", "55788", "3114", "69920", "22142", "17588", "13012", "58956", "37645", "16136", "6190", "5109", "9826", "22470", "38848", "11067", "65758", "50951", "13687", "53703", "62500", "66396", "31234", "35117", "63599", "45395", "6286", "35910", "19643", "26153", "15431", "15911", "49426", "28796", "44964", "44518", "25602", "63390", "69683", "11952", "38503", "28101", "2038", "40100", "47201", "29109", "52166", "71054", "58679", "65445", "47167", "65737", "47998", "17881", "68075", "586", "66096", "59490", "5670", "2096", "17455", "974", "39204", "64793", "25521", "36037", "52330", "63074", "12835", "49284", "9089", "21820", "54871", "391", "12138", "68978", "43228", "62011", "14668", "4425", "34846", "19641", "5256", "59815", "37418", "10797", "24895", "51062", "70460", "14823", "11031", "64437", "28298", "16981", "29742", "29063", "4844", "69074", "71920", "52198", "70953", "22819", "34641", "26646", "12587", "29142", "37677", "44762", "39872", "67090", "12756", "71150", "14986", "27943", "70331", "71936", "52322", "29971", "55163", "26992", "34113", "72257", "54794", "54910", "20124", "53329", "44199", "55464", "3847", "22397", "6313", "57010", "27489", "2012", "23445", "66766", "36882", "13212", "46456", "31668", "17437", "41351", "71465", "8111", "22892", "72140", "39896", "49318", "47150", "3692", "66718", "55863", "12676", "13043", "26582", "41396", "16017", "48660", "49550", "9376", "54807", "34520", "6808", "62168", "14500", "29197", "52369", "12516", "5802", "4223", "63686", "39926", "70613", "36460", "28119", "48061", "51701", "14523", "45680", "58025", "47918", "50867", "19932", "38148", "27328", "45808", "68537", "12322", "56313", "31153", "28922", "12005", "73113", "23881", "35812", "45751", "27352", "62432", "63337", "35350", "71948", "19830", "38854", "40847", "1214", "7502", "70359", "63853", "41157", "21349", "71159", "31365", "28462", "37870", "42850", "38038", "71337", "2799", "69239", "16236", "61436", "4671", "25018", "44347", "19673", "39840", "48840", "52934", "38938", "47145", "25607", "57103", "1950", "36511", "45971", "1220", "14883", "35372", "30625", "7277", "57872", "6622", "47670", "32382", "15997", "45956", "67600", "20951", "49198", "17841", "18240", "27483", "35215", "48826", "44667", "65267", "34274", "4707", "57437", "13902", "46051", "3570", "45120", "23636", "5405", "47355", "38987", "7100", "47332", "72519", "57303", "7427", "55176", "68223", "62530", "62283", "68683", "45240", "14877", "6805", "29830", "51256", "13731", "20795", "68740", "30161", "9616", "464", "7759", "35786", "16904", "20516", "69963", "18575", "31780", "72129", "52148", "23789", "49757", "1816", "70527", "19169", "41315", "45845", "51938", "51541", "59554", "44902", "37866", "64721", "65951", "70177", "27831", "21076", "62636", "64024", "27388", "47528", "54703", "72483", "12838", "46168", "36892", "15598", "41368", "25346", "268", "22943", "60203", "2497", "70394", "27695", "21256", "50415", "25327", "44245", "4206", "8724", "53118", "51157", "21461", "67832", "69328", "50758", "13352", "21292", "18122", "26997", "25044", "2514", "69545", "9647", "36422", "55864", "19690", "35866", "54427", "39367", "41399", "1079", "9491", "23026", "65830", "16715", "29253", "42828", "62197", "69597", "28247", "5945", "844", "4687", "13328", "13233", "16281", "59689", "16997", "58003", "50227", "11869", "26907", "15973", "25009", "8478", "40581", "8683", "3088", "204", "16354", "25475", "55110", "68574", "26985", "29955", "15115", "3629", "20881", "22116", "35427", "15705", "70880", "63737", "62857", "39497", "28526", "23266", "48514", "4737", "40012", "60183", "5650", "70099", "12952", "8390", "53656", "70652", "28262", "71240", "28407", "50952", "17748", "26418", "28409", "51307", "64930", "2151", "26516", "21175", "49564", "46144", "48133", "48300", "221", "50507", "37436", "26329", "14264", "20903", "68219", "55511", "15475", "10502", "5602", "16066", "66658", "2185", "33462", "65392", "18995", "41904", "72083", "54013", "3074", "17188", "44690", "18441", "41475", "14610", "42662", "67601", "14941", "35771", "11048", "28161", "291", "46977", "11988", "38689", "59480", "32477", "49833", "10056", "39049", "54359", "42739", "38218", "56926", "33547", "4536", "47057", "24218", "61622", "57386", "18646", "35890", "49031", "40958", "28198", "19924", "42824", "68920", "1664", "12072", "49810", "23082", "23446", "72500", "51380", "72518", "13599", "61882", "35798", "46772", "24417", "1331", "23428", "53550", "69720", "8881", "20197", "73228", "9473", "9780", "1635", "47576", "16454", "46792", "66857", "26120", "31897", "20077", "49388", "4772", "435", "26395", "1157", "9738", "12499", "71461", "70171", "13104", "18126", "58117", "58662", "23667", "55209", "24955", "132", "1495", "21138", "42008", "11868", "72142", "70758", "55099", "44115", "22607", "45634", "8032", "53698", "28881", "58098", "19503", "10871", "3989", "8968", "41459", "21561", "26797", "31685", "38703", "20864", "19101", "18278", "52940", "58315", "50702", "21109", "49787", "11332", "448", "47529", "4787", "21092", "12818", "8326", "35049", "7703", "12758", "41158", "39316", "55445", "25725", "15733", "20777", "72624", "18991", "70285", "45738", "44838", "35346", "7524", "58338", "45008", "1434", "39398", "32326", "59595", "49151", "46846", "44233", "46007", "29782", "63375", "24959", "48613", "34634", "9757", "59364", "25711", "20037", "43134", "58516", "40912", "1466", "59124", "46531", "17148", "12804", "40180", "55388", "54078", "777", "10750", "31618", "60457", "59360", "50972", "39637", "60934", "50460", "46037", "35269", "20490", "62921", "29670", "36911", "47886", "37269", "14735", "29573", "44217", "8041", "35023", "21818", "13514", "65408", "43354", "41931", "1517", "58504", "26279", "69369", "66594", "27296", "60033", "48620", "12437", "14094", "31880", "32001", "48023", "25660", "11687", "62923", "34554", "32231", "10965", "22311", "24949", "30123", "670", "6510", "70562", "65224", "61631", "60317", "48859", "26233", "4956", "57144", "39413", "3227", "15948", "23918", "3209", "72416", "64651", "65248", "9728", "39622", "26379", "65865", "15718", "65072", "28664", "43124", "19264", "70664", "3218", "7819", "65013", "45383", "55009", "50717", "41894", "54098", "73233", "2274", "70669", "11808", "43813", "52053", "55737", "35755", "63022", "24459", "65476", "33865", "14056", "68439", "277", "62300", "42022", "51142", "59323", "48148", "25600", "27356", "25588", "56562", "15038", "30555", "48115", "56877", "44876", "72373", "37283", "38382", "28751", "39366", "66790", "59228", "10971", "12158", "66595", "47901", "19811", "18933", "68925", "7713", "13609", "2262", "55941", "42796", "33601", "67267", "68401", "10666", "11475", "47744", "44976", "73134", "62633", "55891", "13300", "56642", "12454", "25895", "24069", "55586", "2091", "70693", "71539", "44758", "24936", "34424", "68162", "72431", "10958", "49541", "9267", "28501", "3516", "5981", "40349", "43247", "30200", "40108", "18811", "18704", "48079", "67529", "5999", "39757", "17853", "7056", "16308", "2491", "37800", "58982", "10186", "17489", "64775", "29729", "62971", "56236", "71917", "15454", "34654", "71585", "62419", "38276", "62697", "10254", "47140", "24457", "29423", "31843", "53439", "12107", "57807", "47533", "51901", "2288", "72584", "32442", "38497", "38672", "62877", "61391", "28042", "41991", "25824", "31325", "51030", "46709", "36968", "18146", "58256", "38028", "18894", "46603", "72015", "21737", "65917", "37309", "65695", "53889", "22592", "64141", "70049", "62591", "14744", "4977", "45221", "13213", "1257", "69129", "15916", "65173", "70074", "43750", "44494", "53356", "49697", "53939", "67779", "7851", "73150", "26489", "22118", "40296", "57317", "66323", "27302", "68777", "37482", "27953", "6875", "21380", "54207", "67043", "42771", "3571", "40976", "56759", "70003", "49449", "11932", "42924", "22333", "66884", "54756", "69684", "69351", "286", "65451", "63880", "38048", "49289", "5455", "14470", "37659", "11389", "11360", "24818", "34839", "34717", "69473", "22290", "54604", "14887", "68072", "37776", "30309", "19443", "20953", "69357", "12916", "61147", "65174", "12948", "14404", "26321", "6758", "37711", "6737", "60316", "26183", "33733", "29177", "26167", "10604", "45409", "49888", "22313", "45100", "15520", "71429", "43675", "25655", "10425", "28794", "19843", "19308", "43219", "48015", "65932", "19650", "64795", "39787", "34245", "47256", "52606", "14904", "21837", "14383", "53969", "17715", "30685", "65017", "69359", "62404", "16064", "44770", "50403", "48197", "40930", "37297", "73020", "44102", "66898", "38030", "58852", "2143", "36565", "2696", "49553", "35568", "6787", "48510", "24925", "63245", "55658", "19224", "2582", "19184", "49061", "61847", "34670", "14537", "9986", "57632", "7374", "21123", "38214", "9114", "27929", "52209", "61116", "62563", "48659", "17302", "26554", "25024", "27851", "13652", "59603", "5489", "60546", "31002", "41856", "13122", "13124", "40453", "40730", "36253", "42890", "8018", "58430", "45125", "36704", "44396", "2630", "7324", "27961", "46208", "6267", "34978", "44502", "30229", "52029", "17284", "54929", "5128", "35303", "28552", "34657", "53141", "726", "20293", "64415", "62712", "69105", "20028", "60329", "44986", "23402", "59728", "44100", "4475", "25526", "9685", "69289", "32592", "51366", "18989", "29685", "55225", "60986", "25819", "70339", "22930", "10120", "31466", "44822", "5040", "2190", "33697", "14613", "25595", "22677", "37780", "24074", "60695", "41569", "52247", "19534", "4118", "43171", "18734", "46826", "626", "30826", "16388", "13430", "57139", "47838", "68221", "56004", "70573", "592", "59023", "59467", "62689", "14706", "22534", "30748", "63125", "37803", "47156", "26995", "27130", "26762", "12510", "19992", "12164", "18119", "26981", "49140", "2690", "16649", "55016", "32738", "14414", "30245", "18949", "14913", "32335", "49840", "31184", "45203", "33090", "61750", "4305", "30153", "27597", "33804", "20305", "10701", "33345", "42747", "62570", "62029", "42193", "48366", "36327", "40297", "71840", "23045", "10285", "19313", "35487", "56469", "71333", "26825", "68012", "64226", "7954", "47666", "19829", "21059", "22676", "54302", "10079", "37453", "3200", "52062", "7572", "63986", "6235", "41943", "51995", "32211", "71639", "63925", "38995", "6643", "41582", "21498", "46835", "60242", "28668", "653", "59663", "35509", "20098", "31895", "25929", "43667", "15467", "20184", "50400", "29711", "68970", "38185", "42884", "6722", "6564", "61007", "14445", "31971", "48677", "69375", "12462", "3803", "3264", "2293", "50686", "60422", "37420", "71499", "30761", "57400", "31768", "1944", "9554", "37454", "1468", "66229", "28231", "31178", "2691", "16903", "23415", "38942", "5433", "26428", "29751", "3890", "27844", "49769", "25366", "24221", "21705", "39742", "57531", "41705", "70296", "8431", "16908", "67228", "36070", "2899", "41987", "6843", "61284", "64864", "40723", "10777", "37105", "72220", "70958", "70585", "20023", "3272", "29665", "33137", "50196", "38726", "42561", "67853", "63059", "8373", "31723", "8963", "49277", "4545", "50998", "16144", "2129", "27464", "53379", "37203", "21286", "38278", "51190", "4017", "12531", "25624", "20330", "17067", "62397", "25744", "57577", "14153", "1459", "12528", "55943", "70329", "11678", "32460", "53576", "25237", "48633", "32387", "10048", "58293", "60307", "53097", "10518", "19628", "57605", "54988", "72205", "68696", "66432", "30650", "38058", "46872", "62527", "40782", "20230", "2946", "1824", "70544", "58340", "31073", "70207", "69076", "31414", "38724", "15395", "28904", "53992", "18267", "59777", "26597", "55991", "41646", "69891", "51448", "66614", "20487", "31172", "8623", "72713", "14078", "38472", "7878", "54838", "73104", "68102", "13551", "44292", "8901", "11903", "29421", "29998", "47298", "40447", "46673", "2331", "21892", "44773", "54276", "52782", "26139", "18129", "124", "66083", "23839", "54885", "7936", "32885", "5606", "65689", "21482", "19243", "41089", "34625", "11555", "18705", "47205", "4203", "18801", "54594", "33160", "22949", "36785", "72985", "10884", "22872", "67162", "68144", "66779", "70812", "23291", "35629", "36852", "61517", "25029", "40643", "51754", "38660", "58797", "72383", "6181", "70636", "47123", "61844", "37589", "52987", "49804", "33630", "6546", "7526", "51119", "10786", "57", "27741", "53869", "65930", "64458", "53483", "69846", "41414", "41541", "56631", "60699", "11895", "18802", "61205", "36698", "60444", "2835", "37964", "53982", "23796", "71093", "68550", "57938", "42923", "28438", "50958", "5072", "2483", "52829", "40246", "67422", "55593", "30919", "46515", "11467", "29207", "33238", "56677", "5876", "22805", "46099", "36861", "44507", "21911", "68952", "31943", "51853", "1389", "40872", "13091", "62755", "2145", "57285", "60762", "69307", "18145", "35691", "56787", "29205", "42936", "4630", "9199", "12624", "15214", "45214", "22266", "48802", "12953", "1833", "1989", "8256", "42099", "2868", "57126", "62569", "25553", "14775", "31733", "58380", "56457", "40789", "68031", "18524", "68455", "8577", "56700", "55503", "52858", "12046", "64196", "71781", "30956", "19768", "55686", "11519", "39213", "36210", "35940", "65788", "9700", "53513", "31622", "3945", "46725", "26891", "4331", "50595", "47740", "31210", "56020", "32516", "26470", "57157", "53523", "62190", "22593", "29252", "26771", "70560", "21846", "61062", "42558", "46558", "24406", "3857", "34183", "27150", "4108", "41100", "18091", "46043", "25643", "26922", "45079", "4603", "26498", "62422", "69881", "6816", "62099", "32399", "10717", "11420", "30063", "3854", "38550", "41498", "7224", "910", "24419", "71078", "15217", "68145", "45392", "34580", "34680", "22557", "7771", "33899", "41831", "18543", "36706", "12749", "61820", "23753", "2905", "1039", "16079", "41408", "22020", "63936", "7623", "56748", "40218", "39680", "67423", "33383", "31869", "18722", "32756", "67574", "10041", "31838", "5873", "64893", "125", "60573", "24545", "41761", "49060", "34051", "55052", "66940", "60228", "43563", "20758", "42753", "15649", "41241", "34494", "56485", "37552", "69228", "36665", "64828", "56032", "30198", "38738", "4075", "17672", "71501", "46542", "839", "57044", "20057", "63045", "58779", "64863", "52809", "70917", "7967", "38780", "58534", "49809", "9456", "51012", "6406", "20242", "66324", "46297", "21018", "33201", "22561", "43274", "41559", "5117", "65065", "21031", "39674", "2591", "43815", "18823", "60084", "28896", "47595", "8841", "3478", "6577", "27276", "23978", "71173", "57374", "32051", "21043", "56200", "25141", "3147", "66452", "3179", "17973", "54887", "35752", "28841", "46508", "50601", "30388", "35832", "72766", "30703", "34304", "42720", "28377", "33089", "36820", "73183", "12571", "28580", "3063", "30718", "31332", "70458", "35258", "48889", "25455", "3940", "17686", "67088", "67331", "2346", "59107", "43518", "3362", "26761", "73223", "40733", "41712", "72376", "6480", "45700", "51046", "55479", "30800", "66977", "68492", "51522", "3677", "52041", "70077", "40159", "36303", "56261", "51456", "56765", "61972", "50216", "68427", "39821", "19671", "31056", "16919", "61841", "52461", "56050", "26050", "71119", "19346", "51739", "5899", "59446", "18167", "63887", "11829", "42313", "11691", "31975", "24120", "46166", "19093", "19063", "65088", "69127", "56330", "46496", "3956", "6902", "21829", "34827", "42484", "23998", "30341", "19726", "42416", "50209", "23043", "65969", "66139", "6593", "72797", "11891", "43594", "31268", "65564", "18622", "694", "2962", "65593", "64412", "27010", "67419", "41552", "12540", "52598", "49821", "40791", "24309", "64264", "31860", "49278", "56907", "16435", "50074", "32534", "7986", "19326", "5385", "27815", "31068", "39416", "56939", "8214", "19907", "69955", "43244", "39306", "6645", "30481", "61022", "3740", "13623", "69389", "3982", "48345", "23814", "24761", "64594", "55876", "38484", "42887", "21193", "48313", "11860", "51751", "57967", "336", "51409", "25857", "28026", "46753", "68211", "9045", "32799", "70373", "54245", "6237", "12487", "5984", "60064", "38552", "20257", "28983", "29541", "65271", "10865", "32188", "62762", "23575", "26651", "66629", "24002", "50575", "71801", "21617", "55814", "5376", "55022", "4648", "8136", "65275", "34351", "47524", "1140", "5545", "18759", "50184", "45772", "67524", "40771", "50840", "56220", "2746", "56191", "45855", "40206", "29281", "28738", "3112", "43804", "59747", "34315", "4374", "72422", "44825", "40262", "1713", "60185", "30934", "39901", "59270", "53316", "54962", "49731", "42659", "8450", "65461", "22011", "1325", "34366", "33510", "27960", "13390", "33182", "20085", "57315", "66552", "15517", "5905", "29143", "3175", "13767", "59422", "60827", "3752", "69681", "40570", "12177", "46155", "54301", "14567", "62158", "61615", "12918", "29320", "1347", "32009", "70229", "64996", "60680", "23627", "24013", "56992", "48193", "6282", "20522", "50448", "27102", "35848", "19316", "11256", "47097", "67520", "11507", "65906", "44380", "34061", "39810", "68819", "43116", "70091", "31116", "22273", "2888", "36555", "67244", "12260", "68306", "54336", "1024", "56326", "39068", "22810", "53929", "47135", "42455", "7432", "70711", "59167", "36131", "48976", "41695", "20847", "30262", "21158", "53034", "51258", "67915", "30466", "11482", "22210", "28808", "64789", "61591", "58749", "39797", "4284", "16626", "5509", "39320", "40344", "6071", "52544", "30239", "45444", "39103", "32517", "68568", "27859", "16161", "29918", "62677", "5366", "5503", "34950", "19354", "66522", "47716", "37497", "22436", "11873", "27156", "62842", "48546", "26206", "33695", "8224", "65789", "24723", "43356", "30292", "53413", "7530", "7543", "53017", "17493", "20888", "41815", "25558", "64588", "23358", "59223", "45272", "22756", "32934", "57809", "15829", "45130", "27477", "37256", "22418", "1581", "27500", "35729", "25185", "17553", "30691", "16994", "20381", "16947", "33514", "22747", "6778", "36260", "72901", "63641", "5695", "72909", "33876", "15056", "25079", "3560", "67571", "5956", "40929", "68236", "37732", "40927", "57229", "27640", "43252", "6075", "32412", "7027", "63129", "12536", "12448", "33111", "44401", "67712", "47436", "16410", "70707", "12875", "72533", "10483", "64237", "18000", "56953", "2830", "13931", "51933", "43986", "41242", "23039", "46759", "58449", "5301", "30634", "36651", "31523", "2221", "46894", "15375", "38716", "61554", "36417", "30947", "31295", "399", "27951", "65927", "64824", "62079", "10482", "6148", "23764", "33882", "32271", "50544", "3091", "9066", "25720", "51824", "37982", "3974", "37604", "59701", "5271", "72735", "21212", "52259", "42918", "15809", "20851", "68327", "27734", "11427", "66839", "8722", "9000", "54627", "24953", "24960", "41876", "25389", "66937", "67458", "28845", "64543", "27066", "62123", "40209", "178", "2496", "61848", "65053", "24691", "43969", "31872", "519", "612", "55970", "40992", "29092", "46510", "35172", "46778", "39019", "52509", "17379", "20891", "34480", "41367", "52345", "3635", "2959", "48046", "1238", "6846", "9559", "53553", "41968", "21022", "22650", "45970", "69627", "39063", "7095", "69869", "24543", "3950", "59688", "2699", "31047", "71046", "28031", "1484", "37030", "13656", "28998", "45694", "66646", "50311", "44710", "52272", "28309", "12994", "32232", "23664", "24005", "62177", "2919", "8081", "44470", "7762", "22693", "11211", "33862", "47742", "42211", "28378", "54141", "64991", "60800", "28108", "60464", "30390", "2451", "43828", "1212", "318", "361", "37351", "34764", "60062", "21824", "31006", "33101", "14682", "63635", "32611", "9797", "13438", "2182", "55697", "12565", "30780", "27663", "6110", "46560", "48380", "54500", "50940", "30534", "50508", "62608", "486", "58841", "51746", "68822", "10731", "43393", "11729", "13597", "40362", "1504", "22564", "47780", "20033", "62271", "12930", "72082", "1116", "33748", "43359", "25214", "18547", "60548", "29810", "34702", "70140", "24924", "48084", "71973", "3688", "10615", "21201", "27090", "72429", "1344", "4341", "20637", "71463", "3066", "52712", "42058", "40520", "49313", "65850", "20528", "24298", "66988", "12788", "70627", "36502", "19824", "38395", "49416", "13111", "56352", "29614", "54542", "29966", "22187", "505", "10864", "23932", "19927", "59068", "73132", "30279", "21961", "10477", "48233", "56594", "51367", "65107", "66708", "66561", "45329", "55543", "49611", "19912", "48870", "30920", "11892", "19737", "53946", "55112", "7165", "50336", "45729", "19376", "34769", "37592", "60720", "35265", "44060", "11931", "43399", "70071", "68938", "23200", "66554", "63579", "48338", "46394", "24468", "14357", "35426", "28016", "23727", "16074", "219", "33614", "51634", "50678", "36282", "20778", "18603", "45162", "28232", "27106", "58657", "2609", "29909", "15298", "27455", "60959", "43371", "1636", "5176", "2978", "60666", "64508", "26520", "50431", "5543", "22784", "52863", "14772", "35913", "37052", "24321", "21367", "24059", "19522", "71050", "47706", "7602", "27304", "58322", "5465", "9170", "64924", "12836", "63429", "17477", "20715", "72231", "64121", "23495", "62452", "4449", "60088", "69466", "63578", "48062", "23086", "42919", "44368", "66225", "17617", "43803", "59437", "8694", "545", "50538", "65381", "50542", "8395", "60076", "8917", "59918", "10426", "47579", "71980", "71894", "14255", "22538", "31681", "36206", "66285", "11549", "59389", "21006", "57517", "65214", "35584", "40586", "16683", "60498", "10302", "1249", "60479", "28306", "19998", "64283", "33833", "23314", "67805", "7188", "48511", "46234", "21375", "10017", "7774", "39716", "22958", "43929", "27242", "34211", "65290", "57550", "659", "10099", "65814", "3197", "54142", "64408", "22515", "15382", "48237", "30306", "71523", "26308", "52708", "32573", "63837", "28937", "27075", "1889", "17900", "4878", "66417", "43327", "31359", "15419", "56782", "44432", "58905", "14235", "34433", "11740", "18459", "54419", "779", "46086", "1747", "56137", "7397", "24563", "26207", "66905", "51232", "38307", "56287", "8313", "63546", "15794", "66864", "39887", "28499", "67892", "8535", "15900", "45750", "13746", "41491", "10627", "65087", "55069", "20621", "6995", "28537", "879", "43357", "61769", "17059", "16732", "6839", "72699", "8340", "4251", "34273", "3492", "37193", "31992", "44264", "45117", "46708", "58713", "43511", "53365", "44190", "58111", "19521", "43085", "23290", "42204", "71377", "61464", "10316", "38259", "32075", "20513", "29737", "3686", "65028", "44613", "5258", "54206", "28225", "46147", "42332", "46339", "50922", "70036", "61793", "28903", "19012", "50739", "63945", "56325", "46677", "55259", "71255", "30967", "5796", "32888", "32006", "39508", "44339", "46857", "23037", "57201", "14820", "27656", "7239", "21263", "28707", "40223", "23074", "24271", "63493", "60609", "10074", "41133", "13966", "7793", "16974", "45704", "57206", "52405", "52162", "71022", "66882", "46713", "62100", "7718", "45106", "40973", "26843", "73059", "29351", "70611", "55115", "12889", "18864", "27789", "44599", "5687", "66729", "37425", "38152", "58583", "4198", "6611", "64163", "72204", "44905", "10728", "44402", "48681", "61057", "55651", "15201", "15090", "25413", "47159", "50239", "65078", "58523", "17975", "49431", "47785", "43738", "41231", "9808", "59862", "47992", "45754", "46235", "60331", "48562", "60159", "1818", "51793", "70987", "59942", "51073", "54343", "10804", "18218", "70327", "25969", "63413", "72030", "12479", "3938", "66647", "50199", "42629", "11775", "23413", "45422", "8324", "72354", "57865", "19049", "56633", "16902", "68625", "5266", "49596", "63935", "65870", "52750", "5587", "31568", "18444", "26039", "31252", "19111", "54998", "9047", "8935", "685", "42303", "55199", "20361", "43312", "6065", "63047", "29506", "68286", "66676", "33486", "1240", "70587", "54160", "1440", "42050", "32751", "13779", "24087", "68533", "63713", "7264", "53164", "12051", "53782", "18060", "33492", "44912", "24", "59478", "48128", "55642", "42055", "60313", "49116", "20142", "51560", "31764", "41657", "22490", "2666", "26826", "52375", "37805", "61728", "19519", "15039", "61479", "14342", "50496", "54166", "1085", "53317", "47235", "60418", "25347", "8253", "46701", "57837", "7688", "66178", "60010", "57698", "21797", "17622", "41654", "5355", "67286", "11089", "30073", "23189", "67666", "47339", "29117", "54045", "23526", "69630", "27178", "19989", "7008", "37704", "56234", "60256", "65092", "18992", "9441", "26730", "21761", "38844", "16419", "39102", "65305", "63648", "17431", "44447", "13088", "27360", "26949", "35338", "6395", "50796", "64020", "27583", "48590", "55120", "61674", "63687", "41627", "46479", "27077", "37808", "8411", "14314", "56413", "14250", "68864", "6379", "66657", "8351", "59641", "34830", "47170", "61668", "57426", "61897", "72388", "9828", "26693", "21654", "21579", "363", "7439", "45541", "9465", "52945", "63637", "6909", "35314", "63118", "46499", "2908", "64253", "32128", "36757", "41570", "37700", "21119", "68479", "72280", "22150", "6089", "22251", "37375", "33863", "51470", "22041", "52967", "41602", "36538", "31397", "60538", "36875", "61810", "22868", "21626", "57796", "68382", "17635", "57844", "37127", "5842", "7048", "31382", "54899", "11534", "36701", "20017", "11202", "42388", "9636", "13120", "289", "68770", "65142", "23005", "12250", "64906", "71186", "39064", "1311", "42452", "50613", "16355", "54816", "47136", "16836", "45169", "59589", "16500", "42284", "57666", "61364", "58686", "62767", "67938", "28531", "44363", "35290", "18419", "45887", "38021", "5334", "21362", "27133", "45013", "26030", "35150", "52660", "11331", "58669", "24453", "52183", "47184", "17595", "35925", "5294", "1857", "65760", "44112", "62017", "33038", "60530", "15200", "20201", "44629", "3490", "11782", "65618", "54753", "38510", "38358", "58123", "31164", "61055", "31995", "59022", "38397", "10183", "20679", "26156", "4792", "4430", "54290", "2839", "47556", "51957", "39828", "15263", "10372", "9746", "62892", "49013", "28882", "32098", "47874", "32257", "62817", "5306", "62822", "36195", "12649", "1410", "26751", "29907", "33856", "20423", "63189", "60575", "19770", "27582", "70187", "9579", "57604", "12502", "21071", "37385", "44283", "5657", "66654", "14121", "3698", "3924", "1418", "46703", "23675", "33515", "61442", "72167", "24372", "43589", "20393", "18862", "4658", "49480", "50911", "29798", "61258", "43845", "34423", "2543", "32005", "67284", "5354", "48365", "23373", "25308", "39185", "49862", "69242", "72575", "55470", "53370", "1838", "41768", "57390", "37647", "26075", "22635", "58787", "47497", "19310", "40826", "27861", "33887", "20853", "41246", "43987", "24977", "62833", "51439", "34815", "72120", "15136", "54677", "37817", "38295", "71608", "25088", "44139", "3914", "24039", "21102", "47371", "48833", "17388", "66987", "6753", "60586", "52377", "48854", "14029", "34613", "14635", "38137", "68806", "44369", "59475", "24629", "49676", "37936", "15168", "28420", "59005", "45966", "29492", "69960", "24979", "36221", "22976", "13499", "72545", "71361", "65136", "58603", "69534", "457", "67780", "11552", "30461", "62010", "34967", "33011", "29721", "57161", "27435", "14531", "23702", "57162", "40555", "38236", "23040", "58914", "3153", "31720", "32192", "53817", "23968", "40767", "23506", "11528", "45309", "50488", "59424", "67914", "31110", "11445", "36491", "56059", "59136", "36954", "60032", "40217", "48272", "10947", "44327", "37120", "28381", "10198", "47713", "15750", "53495", "38475", "46918", "17642", "7396", "37485", "47580", "45509", "59102", "44294", "57186", "16134", "15222", "23958", "57005", "12357", "43612", "47448", "37348", "33640", "54685", "2324", "30577", "23365", "12591", "110", "33903", "16455", "61425", "36789", "5605", "64612", "56405", "12488", "61872", "29475", "61152", "49837", "36331", "56301", "66334", "11692", "28280", "5624", "13717", "64421", "55468", "12457", "44969", "51632", "38489", "70078", "18954", "59408", "53613", "72620", "13282", "44121", "2062", "56996", "29812", "20122", "3436", "15364", "15855", "4511", "62773", "14846", "61415", "31064", "29491", "46426", "28159", "58433", "48647", "38149", "65166", "47303", "16356", "12010", "57013", "57855", "66885", "7589", "58563", "18390", "45198", "8609", "28114", "47657", "26469", "23649", "34196", "18048", "14184", "59209", "48600", "62022", "29478", "34241", "17069", "25143", "67486", "63765", "42528", "25796", "15233", "48685", "26394", "29411", "26602", "26112", "72762", "44373", "50007", "53110", "43893", "10257", "61393", "65354", "46143", "2248", "20137", "31287", "5154", "50657", "68386", "25060", "33053", "6080", "56272", "26769", "21139", "3768", "71177", "52504", "12958", "11130", "72924", "60231", "11569", "55378", "55826", "38036", "52571", "14583", "14199", "51847", "45900", "50709", "53834", "17116", "40336", "16368", "3384", "66044", "31608", "21730", "55989", "44994", "10886", "9426", "45042", "56126", "18379", "60532", "16300", "30038", "21213", "41454", "6236", "71704", "59463", "13066", "41343", "28076", "64842", "51414", "64446", "39118", "67942", "60946", "29324", "13467", "15905", "9995", "71072", "56660", "36800", "59773", "40211", "12742", "28540", "69458", "47971", "10643", "8726", "6537", "16992", "28647", "70533", "31675", "49068", "69035", "17270", "49956", "64995", "68333", "70027", "8706", "63127", "59354", "65971", "1869", "13923", "36609", "26076", "6238", "58999", "38059", "51022", "49923", "10683", "56708", "33392", "4385", "64343", "22533", "67018", "14421", "38101", "68699", "36425", "29886", "18635", "43854", "73133", "32664", "9732", "57662", "5285", "38895", "5532", "12643", "22691", "48949", "66342", "50598", "21735", "70144", "45037", "56903", "31133", "65725", "51939", "17725", "68083", "30775", "43923", "53983", "47978", "58048", "15363", "1422", "25109", "53849", "66451", "46414", "29290", "62627", "36025", "462", "26522", "14157", "14082", "44677", "30771", "17147", "44192", "11366", "64454", "5688", "57140", "69882", "28604", "16702", "13887", "4853", "55741", "45758", "57880", "3618", "1885", "60294", "10846", "34986", "23781", "44295", "63131", "25915", "12450", "19361", "60639", "13406", "60324", "55862", "10499", "48136", "59844", "42039", "63157", "47649", "19148", "62462", "44061", "4358", "68763", "69142", "46012", "26095", "7669", "43974", "31239", "24904", "39620", "35583", "20773", "23848", "15004", "36796", "32910", "37759", "66175", "11114", "38010", "35871", "5721", "69471", "3611", "51514", "29683", "32621", "38917", "63437", "29675", "31916", "41490", "7108", "810", "14755", "45644", "43231", "72756", "2766", "15780", "55656", "29952", "37937", "40646", "54220", "53972", "13198", "51630", "14624", "36060", "29360", "7941", "70632", "43977", "33", "51400", "31758", "60588", "65300", "65374", "32840", "5844", "67330", "34414", "25094", "7596", "33586", "12947", "35046", "41852", "17293", "31401", "50805", "5947", "71643", "7370", "47813", "44688", "70644", "2411", "68605", "35984", "11956", "18226", "15494", "8425", "30486", "43075", "18170", "12603", "67991", "54971", "46769", "14616", "37211", "22782", "52996", "4058", "40413", "55798", "36017", "44260", "28513", "12341", "64269", "7532", "28012", "13562", "52593", "39569", "21869", "28539", "63377", "5183", "6450", "22755", "33046", "34393", "44836", "70606", "54949", "6651", "8277", "36362", "34286", "33237", "52361", "51816", "31489", "71616", "37702", "7082", "13615", "64436", "39865", "43984", "60432", "35723", "70803", "4935", "67065", "6422", "20428", "43584", "9674", "21826", "66726", "58698", "3573", "18499", "28652", "13363", "1524", "39942", "57059", "24552", "14351", "63326", "37068", "32625", "37917", "487", "71778", "13633", "17898", "6309", "69703", "69062", "12258", "50128", "2476", "55791", "12581", "40184", "3093", "62938", "63237", "50568", "68497", "20447", "61743", "56465", "56177", "38447", "20583", "39951", "24941", "21849", "52769", "50030", "64060", "72441", "14533", "7437", "23016", "14882", "8526", "68737", "7649", "54435", "53037", "22974", "29617", "26540", "34732", "67414", "72831", "5776", "30960", "31993", "14070", "45067", "37983", "3602", "12618", "36522", "66308", "37329", "49576", "7787", "54113", "27318", "37646", "15856", "33837", "26595", "5959", "33917", "20101", "32040", "44223", "18430", "66187", "7485", "22919", "51524", "17412", "27454", "35889", "70490", "44475", "68997", "28473", "70520", "50831", "7471", "24688", "42072", "20152", "60292", "42168", "58182", "52845", "27198", "28640", "33770", "9770", "26740", "59542", "41583", "25052", "3530", "22929", "46688", "5685", "26068", "57299", "24283", "501", "67643", "8245", "40882", "48356", "38274", "71208", "32503", "56255", "15582", "28974", "29243", "59971", "37509", "23493", "69662", "41573", "9252", "32854", "15595", "15966", "51036", "24876", "55508", "47877", "3283", "44484", "25992", "37018", "56649", "67350", "4727", "48437", "2498", "22423", "48883", "22453", "7939", "9420", "12000", "10339", "37742", "69425", "8647", "21557", "31111", "56762", "4176", "6445", "1576", "60388", "47080", "26510", "54801", "37050", "60962", "63316", "43177", "3984", "45291", "61186", "5363", "17319", "4505", "68639", "11293", "34502", "71409", "50744", "70108", "58044", "64147", "16692", "63961", "42899", "43672", "50169", "61110", "141", "47534", "70263", "66776", "43224", "21443", "42977", "25718", "6616", "18911", "23337", "16491", "20303", "60068", "6039", "15100", "45537", "38772", "19019", "5704", "5888", "4440", "47224", "12845", "28975", "67477", "14985", "57218", "50037", "8536", "62454", "947", "31669", "37530", "55315", "39571", "22081", "23548", "61277", "21147", "25082", "54664", "155", "19860", "20921", "60361", "47278", "35640", "8211", "10718", "40458", "42319", "44731", "2473", "19840", "24424", "41287", "20121", "9502", "53852", "7302", "13745", "51908", "58532", "63089", "11881", "39595", "20894", "5997", "58894", "65198", "66816", "57252", "30769", "47844", "41617", "32463", "50009", "71321", "47099", "68189", "2394", "43669", "4841", "16470", "51966", "36939", "18188", "36510", "18723", "19105", "19617", "967", "22762", "29141", "66747", "54713", "29179", "67663", "54745", "15884", "66512", "63128", "23136", "7523", "19776", "4646", "4718", "4060", "36481", "45258", "4597", "39166", "55465", "26351", "24310", "40081", "23834", "56544", "42839", "26562", "1321", "29518", "52490", "47669", "58975", "40377", "18568", "55319", "70639", "21208", "61617", "13819", "7223", "39098", "40122", "38719", "50411", "4616", "65033", "8645", "25971", "1265", "43186", "52570", "58518", "61039", "19020", "28693", "17019", "43344", "57151", "5954", "44572", "50476", "16149", "59823", "48104", "7768", "35159", "4524", "62816", "27727", "45960", "51297", "46906", "31498", "18449", "49993", "9367", "73148", "70620", "6199", "4264", "57921", "58879", "55582", "16061", "24756", "60289", "48993", "37781", "31296", "56736", "28524", "33229", "4224", "41748", "37347", "29420", "27562", "37739", "52860", "14654", "13947", "57104", "69431", "920", "33561", "9028", "16614", "16740", "55661", "48830", "42277", "3296", "58850", "52358", "1568", "45827", "63760", "10626", "18088", "37446", "59442", "57354", "19858", "26950", "48470", "72739", "8567", "57874", "13857", "71128", "17865", "64201", "22489", "25128", "3110", "15703", "807", "19836", "22291", "12831", "51115", "27153", "24859", "41785", "49709", "68273", "39011", "56796", "71564", "37965", "69180", "54653", "23597", "42225", "18942", "65720", "59694", "3990", "61851", "60566", "48610", "15095", "22569", "68924", "14971", "44251", "53672", "72723", "48025", "48186", "36097", "41817", "57251", "13974", "41533", "37877", "38562", "4468", "49846", "59281", "73031", "65015", "19919", "33247", "27950", "19328", "63830", "51443", "53156", "55159", "47068", "55105", "65844", "64046", "57519", "50291", "63046", "21024", "50127", "52993", "15211", "21872", "29267", "39450", "53461", "53430", "53771", "9460", "30675", "57159", "15941", "17932", "6062", "33275", "47855", "28153", "8151", "69371", "33305", "10723", "44765", "40111", "43086", "44415", "52917", "11322", "69481", "40440", "9933", "46018", "62737", "44684", "66722", "12041", "44062", "55551", "17934", "52985", "16563", "70857", "51775", "71348", "2359", "44695", "9852", "34724", "34980", "16132", "26989", "44828", "1975", "57138", "4973", "65420", "41507", "45054", "3729", "57503", "37571", "16351", "67720", "39936", "1036", "68975", "43494", "19760", "41669", "68084", "16987", "59774", "33444", "58796", "71644", "46614", "33587", "57397", "26734", "51120", "276", "61443", "38074", "45792", "712", "5717", "58562", "37719", "60448", "60336", "56146", "44212", "3977", "18227", "5474", "54164", "24777", "46781", "16024", "26553", "27895", "6866", "13117", "55448", "15965", "35210", "21035", "42060", "9388", "64649", "4639", "25108", "29203", "37257", "22497", "22088", "28521", "13712", "45385", "62389", "45663", "875", "15082", "9055", "27512", "53015", "48900", "60799", "22629", "69220", "4895", "17952", "63598", "12473", "70797", "40914", "27472", "60624", "69361", "66538", "53585", "69649", "45924", "57762", "57262", "39281", "8573", "38041", "51362", "13032", "7960", "29198", "38163", "39568", "24841", "17036", "1342", "3963", "64160", "47215", "2572", "59472", "18978", "41538", "45340", "25442", "70084", "40529", "46819", "36473", "58902", "11882", "66608", "9856", "45209", "31390", "4680", "35475", "63848", "10419", "62941", "43492", "50788", "10862", "34329", "49937", "72497", "55628", "27200", "52536", "72780", "16773", "39443", "13097", "23041", "1738", "71013", "7253", "9451", "26705", "37437", "9779", "65851", "65246", "61366", "50594", "69916", "23468", "17888", "43028", "35177", "64812", "61997", "10015", "4923", "61085", "50458", "29922", "58417", "33606", "19471", "37614", "17723", "6969", "45722", "45399", "58942", "27433", "31533", "47139", "20346", "38347", "27484", "18760", "28846", "3461", "6854", "1037", "53631", "22223", "72179", "33541", "23075", "21400", "64724", "24875", "58759", "53099", "35541", "22468", "28713", "28121", "29923", "11979", "11087", "35100", "45331", "55", "29859", "2121", "38767", "44844", "29846", "23032", "19820", "63379", "20804", "41598", "32461", "23985", "43714", "19672", "35610", "19081", "11736", "47678", "34539", "33753", "68563", "32745", "21254", "25797", "1137", "32661", "40913", "60731", "37147", "45959", "36218", "15189", "30585", "29228", "26035", "40946", "37846", "69830", "71268", "57927", "39032", "18084", "40266", "49523", "61608", "66748", "12800", "9203", "12715", "70494", "34686", "58574", "30833", "55630", "3324", "40011", "58319", "71029", "52768", "64901", "28786", "21131", "10226", "10368", "68204", "63067", "66242", "70096", "66280", "5895", "36050", "31795", "55576", "41816", "1125", "14174", "50512", "57766", "22262", "64375", "41344", "15962", "11880", "33907", "3655", "4889", "20521", "23555", "2675", "60274", "48892", "580", "63628", "55254", "61976", "52528", "48225", "39400", "72833", "15232", "1933", "48307", "38032", "19454", "2917", "55362", "371", "58521", "28735", "6469", "28596", "65628", "60219", "37575", "58524", "29889", "12677", "68889", "37850", "34716", "64398", "68636", "21120", "60525", "44959", "68888", "45118", "33183", "58899", "13568", "22514", "54378", "60643", "47606", "37873", "68859", "38783", "65638", "18994", "39534", "52455", "39638", "33496", "39217", "35683", "41091", "25577", "46980", "42618", "42778", "21656", "15272", "48421", "57407", "28712", "49215", "45479", "44011", "2242", "26791", "49621", "40795", "28136", "70779", "48406", "29915", "13280", "4054", "404", "44418", "17807", "64441", "28599", "30281", "27916", "60725", "12880", "25459", "21941", "42401", "62881", "11632", "32755", "59829", "25068", "23027", "40735", "55695", "16517", "51532", "15577", "5982", "10621", "67728", "12801", "56199", "41202", "11138", "1161", "19547", "36032", "40013", "66199", "6768", "72798", "53127", "50668", "47086", "71481", "45892", "69423", "6326", "21707", "45621", "66058", "64912", "27684", "34148", "37501", "33529", "66304", "37815", "25638", "7361", "54057", "20186", "55012", "16824", "35880", "58530", "58661", "70047", "12141", "11452", "4389", "15771", "47130", "65743", "57700", "24970", "25641", "42698", "43513", "27470", "43352", "14449", "70976", "6601", "33569", "46066", "58423", "33512", "69040", "62131", "32675", "65483", "66892", "55849", "21978", "72758", "51354", "30329", "7007", "55778", "59200", "36203", "29483", "25810", "41698", "38877", "37652", "2430", "6746", "44771", "31160", "24506", "41578", "67438", "67137", "62638", "35725", "10083", "53561", "65322", "5054", "4621", "6515", "10117", "69442", "47640", "29086", "12079", "48827", "48981", "57710", "36842", "31015", "40784", "35356", "63834", "19869", "22587", "41797", "58093", "23772", "48811", "41656", "65889", "57788", "2165", "55751", "37479", "23406", "12699", "11797", "8650", "23228", "44480", "7330", "1574", "25153", "70063", "18336", "7059", "20790", "35859", "24706", "37232", "17710", "25830", "56258", "33052", "68868", "1982", "22857", "11613", "59761", "34451", "34217", "46335", "24831", "41225", "40542", "2345", "60080", "60208", "37070", "11077", "64717", "48882", "24442", "72777", "56861", "32938", "63027", "57858", "73045", "43294", "2972", "12827", "14807", "15724", "26541", "66901", "44858", "51228", "8156", "58948", "59489", "22061", "30619", "50611", "10397", "16007", "13276", "58635", "4835", "307", "33885", "2049", "48476", "12231", "44873", "13181", "65285", "50579", "30926", "36245", "26263", "49038", "50218", "17087", "46658", "55948", "50010", "12427", "25780", "22457", "16204", "3953", "48657", "23022", "7381", "22789", "17845", "32444", "23057", "23135", "46663", "47906", "58462", "33959", "39364", "25032", "55367", "32832", "70454", "7449", "27550", "37889", "398", "70870", "7632", "64530", "38374", "19740", "615", "69735", "30284", "11483", "47050", "11271", "50743", "71263", "29949", "6664", "43254", "60286", "38801", "4429", "30612", "11498", "70272", "62442", "8269", "36093", "42873", "40233", "22488", "5031", "55527", "14580", "51602", "43012", "11234", "43007", "25147", "69798", "55843", "16777", "51315", "28292", "49080", "44211", "15929", "7786", "72121", "32151", "71876", "10555", "1051", "33937", "57481", "12342", "48096", "5788", "66375", "45909", "67610", "37611", "71691", "32447", "35657", "53237", "23845", "32743", "53106", "20584", "9710", "17333", "45495", "27128", "33440", "49159", "60750", "49660", "24631", "49582", "47047", "33295", "51627", "36773", "58834", "16409", "52111", "9299", "45576", "13846", "69033", "42327", "1684", "48969", "60143", "57249", "9927", "16674", "7804", "60808", "25872", "9076", "41810", "45301", "48113", "66934", "52305", "50033", "531", "67947", "30500", "48247", "48982", "11187", "66332", "72123", "62407", "2499", "12388", "72707", "34598", "56962", "28428", "33605", "44197", "23294", "44028", "2722", "58813", "5167", "56643", "29181", "71062", "40174", "71657", "24040", "55854", "33397", "13794", "48637", "58328", "31870", "47908", "69488", "39284", "57422", "53542", "12303", "29240", "20349", "30507", "64897", "64984", "49872", "48304", "15140", "23812", "13988", "10735", "12558", "54296", "30356", "23639", "14469", "55900", "46095", "50302", "21142", "18525", "32612", "19456", "23611", "54018", "48527", "32453", "13959", "46573", "6325", "68237", "32433", "44046", "39781", "11670", "50979", "14363", "20960", "68676", "46126", "4615", "66507", "14111", "61955", "23093", "14476", "27760", "50382", "50834", "45189", "24072", "66013", "33415", "49405", "25119", "12321", "66127", "15745", "40060", "30251", "67152", "58068", "62375", "58057", "2565", "60195", "31144", "6128", "2705", "1334", "31752", "56225", "25869", "45363", "65095", "48033", "62954", "24739", "12632", "6381", "6609", "36835", "27245", "30948", "23790", "26361", "48607", "8489", "52285", "47879", "24500", "71918", "52271", "15231", "40894", "65701", "13828", "16998", "42991", "57356", "51341", "10968", "1547", "53604", "57499", "45573", "4681", "63330", "39038", "18075", "56770", "62808", "68773", "3652", "33093", "41034", "20072", "55469", "16472", "62681", "13677", "57873", "16651", "10918", "50163", "46449", "6444", "34115", "36379", "50543", "38707", "42865", "64224", "42464", "23316", "10353", "58625", "22931", "16895", "2379", "21853", "36626", "57506", "15555", "33732", "43014", "62148", "5980", "13669", "46696", "64922", "12156", "22729", "39242", "38005", "49122", "63915", "71667", "1087", "8030", "2075", "45582", "18486", "43948", "32950", "36018", "30821", "61486", "6703", "33712", "33007", "50248", "23346", "14386", "52337", "11701", "57671", "71780", "70964", "27116", "45999", "2950", "34085", "72850", "62347", "65668", "27541", "4071", "43500", "33083", "27310", "48989", "42", "42979", "58567", "35804", "39564", "13074", "20874", "22925", "28624", "50726", "72310", "57645", "14705", "68284", "15334", "17488", "33227", "70826", "51032", "53760", "9214", "6449", "22914", "67994", "10053", "53122", "33785", "5094", "17247", "238", "70450", "43922", "7681", "53272", "38882", "58274", "324", "67830", "24832", "64182", "69435", "34344", "53719", "58112", "48205", "38213", "64029", "21760", "9455", "16532", "31413", "20035", "47387", "13190", "42281", "52662", "49127", "7017", "48387", "45627", "63058", "10631", "59564", "28394", "55160", "2570", "58234", "29327", "51185", "46405", "8264", "49765", "53692", "70029", "13062", "6615", "13621", "56747", "17088", "2099", "25175", "32092", "46353", "8671", "7920", "50731", "68454", "3696", "24897", "47867", "57955", "1250", "63792", "5637", "22077", "19831", "10861", "38842", "37888", "103", "65236", "8610", "46952", "24499", "53562", "10548", "71839", "9931", "60069", "32791", "30912", "868", "2773", "25801", "49472", "11295", "8765", "41746", "62685", "40681", "18205", "26713", "30280", "1836", "8529", "53299", "26975", "69261", "49467", "24823", "13339", "53434", "7116", "34990", "64531", "60608", "23023", "10031", "3257", "8640", "58265", "26909", "15527", "14109", "73005", "21063", "67323", "31797", "27458", "55423", "35793", "52605", "8737", "42486", "35388", "66755", "59791", "17989", "22605", "13781", "35527", "47149", "68402", "67211", "1981", "66376", "61537", "53007", "21442", "10607", "16029", "30295", "29084", "45468", "20494", "44883", "11689", "18448", "16872", "8477", "20328", "70915", "36643", "16888", "12513", "54327", "64624", "27317", "53498", "26114", "37137", "6172", "61663", "12243", "27628", "8680", "33226", "34943", "46241", "6260", "33241", "7215", "23280", "1198", "25696", "52780", "613", "22990", "20377", "38579", "51684", "4771", "16791", "57253", "68986", "23492", "26280", "56041", "22851", "3425", "34856", "62634", "8643", "8368", "30594", "41922", "19691", "47947", "64480", "26320", "6229", "22124", "39214", "9173", "18190", "4927", "3616", "21754", "28661", "1698", "10894", "19754", "61314", "57818", "7790", "67875", "13825", "61029", "57073", "72074", "64501", "54586", "58656", "26205", "62235", "19436", "48558", "756", "49241", "24158", "46299", "16229", "60705", "6942", "17029", "31510", "35810", "24415", "38678", "49117", "29187", "60872", "60853", "49230", "45122", "21608", "40719", "7405", "42559", "30018", "60633", "56737", "6518", "50359", "26900", "38498", "35137", "32072", "45834", "58063", "10569", "43979", "53715", "22528", "56850", "70284", "60392", "3543", "62595", "9271", "24998", "62537", "56618", "15297", "58637", "5432", "20628", "71161", "9398", "7496", "71896", "65867", "8516", "52665", "19546", "43166", "51825", "11841", "71238", "16515", "13944", "28993", "31046", "24250", "59160", "4491", "19291", "6008", "59055", "55030", "9758", "21768", "67027", "48140", "1945", "54270", "69080", "72820", "16022", "7709", "62859", "52115", "49822", "857", "70653", "31503", "60377", "35415", "32758", "62429", "58706", "13644", "67818", "52482", "53233", "52156", "58750", "39374", "15", "42415", "3406", "62267", "62730", "7935", "42588", "41517", "15358", "38874", "10715", "49498", "62013", "56091", "56441", "16353", "1318", "36847", "69608", "42700", "54154", "43382", "67580", "22130", "34795", "59717", "4666", "11855", "23138", "54132", "62355", "59545", "51940", "12485", "54240", "52200", "48433", "16480", "7036", "43148", "44460", "39870", "65599", "36138", "35547", "7018", "26169", "1825", "36514", "1300", "6202", "28175", "16844", "24948", "50360", "62688", "17005", "35359", "71516", "64515", "69981", "62667", "42390", "31168", "26147", "42826", "42614", "1722", "36891", "59482", "27446", "50562", "20611", "37928", "58461", "63235", "70874", "42012", "71218", "55106", "56950", "27580", "59235", "17578", "42367", "51791", "36359", "7914", "38078", "37925", "36922", "31538", "27312", "64951", "9563", "32203", "70523", "46602", "15202", "26192", "59762", "50245", "51668", "5614", "50273", "5824", "64156", "57208", "64366", "2329", "70112", "19995", "22156", "32663", "65773", "54598", "18820", "22827", "27523", "68953", "61489", "13721", "51576", "64272", "26034", "21803", "45852", "55229", "10552", "28172", "56180", "46963", "38288", "18697", "44632", "60935", "12312", "60264", "66057", "17787", "50765", "69599", "54289", "43176", "51027", "34910", "13553", "69144", "6076", "24193", "45224", "29140", "42567", "40808", "24805", "66292", "13803", "18612", "24384", "14303", "63859", "69692", "30950", "2215", "16457", "8296", "14947", "19525", "60371", "38392", "3960", "26747", "14072", "63526", "61451", "59061", "17143", "53612", "71911", "2986", "2633", "64941", "63494", "51705", "51859", "67781", "4152", "29061", "8929", "68689", "35518", "31977", "16759", "22231", "25136", "42898", "65359", "36201", "71398", "67684", "35318", "18515", "11193", "60599", "12697", "36401", "41484", "41456", "60628", "9095", "70581", "31045", "5604", "26026", "61288", "34090", "6575", "16087", "28334", "51679", "14212", "24211", "65277", "68631", "45236", "34705", "26203", "9816", "26957", "65286", "12859", "32739", "31647", "53490", "56738", "3949", "29174", "24654", "12694", "56774", "14285", "50301", "68278", "9923", "19494", "17295", "7823", "9091", "15550", "10817", "49773", "63173", "16126", "39755", "60994", "63312", "57674", "34655", "55901", "28758", "2509", "64872", "67637", "16680", "8289", "25283", "58092", "5837", "58968", "60576", "48627", "66371", "40123", "18321", "34909", "26083", "4673", "71343", "37886", "16451", "56670", "58247", "9287", "2161", "44351", "69414", "13425", "21602", "50790", "11240", "37267", "5600", "56211", "8199", "26353", "22894", "65347", "31248", "8319", "13436", "8165", "26557", "12383", "14664", "51873", "17482", "62839", "49381", "71508", "1272", "2615", "37136", "43793", "22162", "42192", "19500", "52510", "4062", "7053", "72226", "17261", "10157", "54851", "20605", "21615", "28863", "36659", "37531", "50265", "19045", "37918", "32571", "23680", "6861", "70234", "26315", "15172", "41326", "36547", "51288", "2541", "8378", "9018", "16980", "31714", "64560", "54973", "18415", "36349", "55287", "72936", "71548", "45452", "2747", "9447", "26402", "38940", "68260", "1643", "39432", "9512", "51777", "72675", "6283", "49642", "15401", "34308", "34060", "45400", "19319", "57556", "54283", "5965", "18349", "65259", "55070", "12647", "8241", "18362", "45312", "436", "3878", "25893", "11150", "73200", "174", "9023", "29964", "68442", "46488", "60370", "6542", "36112", "20764", "57323", "41849", "24479", "4184", "8990", "24791", "59873", "29028", "17290", "34483", "55492", "54031", "42339", "44812", "58519", "57420", "23367", "41934", "33943", "54249", "68535", "14791", "17998", "24407", "3186", "63867", "58077", "8593", "12980", "37225", "65556", "51045", "25499", "26307", "40618", "60694", "57540", "71113", "9629", "48465", "57198", "68239", "53531", "10583", "54684", "70775", "57072", "3338", "72067", "54979", "25431", "46078", "20076", "8312", "63441", "3375", "34956", "68900", "31058", "22869", "44154", "67221", "20396", "24587", "7270", "32762", "6396", "53702", "9859", "6620", "60509", "63780", "30914", "31905", "37586", "45335", "56987", "70947", "67172", "31096", "1572", "49069", "35298", "1708", "64494", "56371", "47028", "29432", "48026", "31161", "53582", "46440", "21578", "52748", "32873", "36515", "35259", "6365", "2805", "36000", "10703", "48498", "56587", "62639", "101", "28626", "49947", "61585", "65370", "4413", "2348", "32628", "56231", "66", "67759", "24547", "37823", "59593", "5836", "53217", "4504", "19715", "37968", "59324", "65863", "60016", "31187", "6411", "47071", "53947", "3054", "18289", "59818", "428", "44797", "57387", "52204", "51792", "35682", "13418", "65194", "58827", "46923", "22685", "24623", "69212", "48058", "27497", "45820", "6732", "56035", "46933", "4348", "61105", "59916", "43493", "27474", "13858", "24982", "29727", "3437", "40602", "34223", "56975", "31054", "16933", "71596", "33045", "16257", "71604", "34574", "22852", "22977", "46649", "33875", "56943", "18157", "60878", "1251", "33140", "63150", "22641", "53647", "29383", "15106", "18372", "11244", "69396", "27913", "22347", "59239", "7210", "17935", "44371", "59921", "20067", "62096", "36881", "61133", "28709", "60325", "13127", "24760", "26465", "25213", "17171", "50093", "15954", "49263", "63619", "9892", "46451", "1260", "19667", "47466", "36212", "7950", "19790", "46173", "63367", "70997", "4360", "1767", "66833", "26758", "15230", "9656", "45068", "49047", "49752", "63921", "60779", "15057", "49344", "48669", "63182", "12750", "50842", "26061", "2759", "35456", "4593", "60375", "18796", "50577", "24525", "19762", "1753", "47635", "524", "48675", "51350", "4596", "38290", "64268", "24183", "61725", "30837", "16403", "51255", "43693", "53641", "21905", "9572", "42190", "33498", "42198", "12207", "18072", "62805", "34224", "11626", "21155", "40002", "72253", "4902", "20043", "61132", "64534", "68158", "18078", "23364", "59886", "52087", "65999", "15273", "6362", "67615", "17156", "32765", "59293", "2039", "42437", "6619", "72768", "15316", "31849", "36632", "38957", "16618", "18135", "58306", "53367", "20269", "49166", "43972", "6610", "35769", "65943", "15347", "3844", "21553", "71359", "71908", "29067", "44400", "52912", "71091", "37392", "56811", "56171", "26168", "12768", "8705", "48529", "46605", "65474", "7110", "46290", "51231", "30581", "62433", "23273", "29284", "63009", "53063", "64768", "16901", "51485", "65391", "71788", "38691", "69638", "68912", "44938", "53019", "22794", "53707", "65594", "56399", "35572", "24448", "40563", "36611", "33113", "33164", "53184", "13057", "63550", "59103", "7707", "8484", "40374", "67165", "40658", "11155", "44321", "54137", "51286", "64676", "64631", "1552", "19325", "30187", "30366", "24781", "48246", "17550", "20769", "48970", "62926", "36120", "41874", "9582", "59112", "54769", "66559", "65615", "5312", "61325", "32494", "55399", "71925", "68028", "69676", "60070", "29185", "48038", "9333", "54338", "33288", "35982", "65589", "16977", "59746", "21166", "53030", "51178", "3542", "39031", "45146", "39685", "54454", "58549", "13141", "23607", "34411", "41616", "1615", "48397", "19844", "23763", "32314", "72141", "55844", "68983", "30348", "26941", "20001", "54617", "31335", "58222", "52832", "28783", "2475", "16754", "13951", "42880", "10923", "48757", "71212", "48213", "3413", "16438", "19798", "44739", "60638", "3773", "62018", "72433", "26523", "3506", "72800", "67091", "36984", "24051", "20671", "61771", "9044", "40051", "5577", "48097", "47950", "32144", "25984", "64504", "39186", "56304", "57722", "7516", "55702", "72421", "7221", "28798", "9029", "27107", "46087", "37639", "4754", "4281", "39929", "30497", "29937", "12188", "58385", "57096", "39551", "68192", "72032", "45014", "70133", "60737", "7187", "49819", "55961", "29285", "62305", "18184", "24327", "32609", "70354", "67355", "34973", "9962", "29741", "28097", "26445", "65523", "24335", "3534", "52766", "33317", "48760", "48650", "44837", "63297", "56439", "21659", "41195", "58922", "23198", "12332", "28585", "16243", "18228", "6743", "53482", "60988", "62489", "575", "5736", "72084", "48943", "29208", "44110", "632", "49971", "35882", "15462", "23186", "6913", "11986", "4876", "61577", "8297", "29809", "64844", "32508", "984", "13942", "43498", "48203", "36994", "40454", "18191", "37382", "52498", "41134", "54552", "63501", "59487", "31637", "45531", "64963", "16115", "71944", "29440", "34309", "49608", "38978", "6016", "34544", "55980", "3401", "72761", "46473", "14582", "33233", "38047", "68984", "58190", "30551", "40236", "20127", "28338", "21216", "5852", "17810", "10993", "35778", "19949", "71166", "46845", "45020", "58604", "71914", "5726", "69847", "48178", "8953", "14903", "54373", "43882", "44593", "68858", "67899", "28517", "47753", "33064", "67376", "5553", "66288", "13843", "66047", "22486", "61071", "40970", "66170", "61092", "47554", "68523", "14338", "18234", "50732", "67475", "618", "17529", "69859", "48122", "6769", "6554", "25580", "53569", "6383", "3393", "8107", "5851", "37943", "67397", "63608", "13983", "56766", "19352", "12480", "67852", "31508", "60826", "4812", "1371", "30259", "41497", "20214", "70591", "33650", "3904", "59349", "52738", "54501", "14446", "8144", "25447", "40738", "3031", "48582", "19714", "67867", "23174", "32205", "9308", "54743", "28245", "16357", "57337", "47381", "18056", "9978", "65301", "46950", "65068", "20116", "69675", "8731", "10687", "51612", "57245", "11395", "69776", "61659", "51421", "63345", "40893", "27891", "55031", "62417", "58336", "30135", "29857", "20624", "69922", "68134", "50092", "15623", "33343", "6965", "40611", "21973", "61070", "40684", "55373", "63605", "9900", "26859", "25855", "45555", "373", "36845", "38793", "37664", "29480", "43322", "44564", "15506", "48411", "13069", "31433", "72097", "72113", "16942", "24458", "13191", "2000", "14326", "72837", "40825", "49687", "46076", "27268", "4186", "4929", "61271", "25511", "58663", "27710", "50965", "66558", "66040", "15212", "59477", "67629", "58634", "52103", "19349", "28387", "66054", "54423", "38531", "54730", "14603", "45375", "43956", "57620", "46588", "60797", "21304", "42087", "21592", "27138", "26226", "66951", "61297", "14817", "54644", "23654", "37814", "23134", "18007", "52217", "18053", "28644", "69377", "8751", "27802", "29566", "8379", "36689", "35789", "21249", "23155", "59707", "72115", "56412", "40161", "4521", "42961", "1683", "48663", "8314", "66801", "41805", "54134", "51051", "15726", "45097", "26437", "27855", "56902", "38592", "55312", "47293", "52274", "62121", "25612", "47350", "34629", "45210", "27171", "71512", "42182", "7394", "26574", "71748", "72335", "56988", "21766", "53058", "44552", "57309", "17145", "54614", "68349", "18693", "2223", "42928", "11778", "21353", "13935", "38146", "32883", "28612", "3959", "67784", "59533", "43545", "23904", "71931", "25422", "21518", "41610", "17979", "48668", "46885", "23944", "62985", "18707", "43083", "9741", "56384", "33122", "9937", "20074", "27327", "65485", "4439", "45602", "4361", "53491", "59056", "1202", "65181", "63662", "16440", "47541", "40033", "41296", "14188", "60332", "62847", "3901", "69918", "57450", "72588", "35114", "40331", "9240", "304", "52124", "52125", "49351", "34013", "8532", "35425", "30606", "33313", "14023", "69978", "16580", "13175", "15683", "6655", "49356", "67283", "40170", "36145", "19092", "69237", "69549", "21110", "32176", "9269", "11948", "40080", "47126", "67488", "30409", "43833", "71883", "70687", "7231", "11672", "31790", "29780", "49327", "45231", "37681", "2535", "54319", "18358", "22096", "73056", "19855", "16959", "22817", "52756", "49542", "35825", "47353", "64592", "33304", "19397", "59312", "15708", "67256", "12609", "55639", "2172", "40931", "5830", "18062", "46307", "29369", "22225", "25132", "62340", "70656", "70955", "37013", "60451", "32797", "8678", "65205", "50728", "20059", "23097", "26711", "34467", "44222", "29686", "66434", "62841", "23182", "36364", "23180", "56857", "56685", "36036", "20733", "15040", "9143", "4433", "19203", "48829", "21912", "69670", "51702", "34609", "8355", "27596", "13374", "52601", "63099", "29529", "28799", "3889", "30523", "35686", "56430", "41152", "23070", "69204", "49354", "5372", "61413", "24018", "42527", "19001", "5467", "37744", "59619", "45521", "24498", "67593", "16770", "69902", "32583", "31025", "49818", "46557", "48924", "25723", "48490", "27923", "20840", "16534", "54755", "44026", "53597", "15399", "53586", "43262", "43479", "28054", "57896", "48029", "30630", "26959", "55326", "16430", "16225", "63585", "35213", "50880", "34281", "56669", "41425", "15293", "15603", "38283", "12990", "25386", "25296", "14875", "48166", "23334", "48410", "3802", "28207", "11014", "8866", "72058", "37044", "13046", "55570", "51207", "37612", "52659", "38682", "30112", "9768", "60243", "16948", "51610", "57549", "14398", "40583", "70488", "31378", "43236", "22202", "53368", "41800", "45366", "18047", "14322", "51123", "57961", "59443", "47472", "66948", "21496", "8010", "32383", "63884", "35206", "62172", "21038", "71859", "25489", "57637", "47538", "48694", "18904", "36766", "26343", "20475", "28385", "183", "7877", "23730", "53654", "24516", "17498", "67421", "43441", "15432", "806", "49393", "31398", "43913", "1472", "13939", "15629", "50420", "12035", "21402", "34094", "66420", "25946", "9178", "7690", "71947", "65894", "56769", "45807", "26209", "56190", "70648", "27372", "8223", "11011", "62619", "10282", "45411", "71582", "50973", "25650", "31074", "53253", "6264", "56001", "55177", "20348", "65184", "7004", "11700", "40431", "54923", "65064", "2688", "41097", "7216", "8998", "6764", "34125", "40499", "29927", "57985", "51648", "36669", "17020", "44409", "21980", "13515", "22335", "21724", "15159", "26837", "58692", "45161", "43222", "4736", "17309", "56514", "64117", "46758", "56322", "47737", "54371", "65973", "70984", "54732", "1106", "17378", "13516", "64559", "2658", "48296", "47599", "67890", "52638", "61807", "35400", "2263", "58167", "53931", "6835", "27308", "14793", "49077", "40281", "2790", "19449", "64456", "26727", "39958", "26257", "28419", "51600", "63310", "10512", "50773", "20547", "56879", "73141", "62319", "50737", "18467", "56318", "52364", "64743", "64860", "37692", "23275", "15193", "5807", "27658", "58792", "27158", "37629", "18356", "16176", "38494", "8398", "60516", "64036", "56324", "9837", "5201", "3787", "28919", "9049", "11320", "52921", "53767", "20228", "26829", "69791", "39532", "10491", "25496", "11285", "2528", "63807", "38539", "2819", "59425", "12062", "33544", "2490", "64801", "64581", "25802", "8950", "70734", "38974", "35463", "14345", "65446", "19660", "3082", "33035", "65444", "21566", "56854", "65957", "20046", "7498", "64573", "58253", "47970", "69399", "11920", "45653", "3549", "15327", "22005", "4708", "60703", "24034", "70256", "67479", "56040", "55480", "502", "47322", "29911", "5883", "34647", "46700", "71397", "41259", "8047", "70977", "15516", "7745", "54568", "24519", "7720", "17763", "73063", "5254", "32201", "14534", "22628", "30255", "48383", "39474", "44624", "60425", "72016", "5149", "25585", "24030", "37595", "11266", "70751", "53042", "44687", "51348", "51173", "43996", "28584", "45270", "21708", "9247", "24188", "47585", "21269", "17745", "64127", "67392", "19772", "18125", "47394", "22659", "54203", "17275", "50673", "49953", "32342", "25678", "2886", "29354", "3717", "19923", "69941", "23230", "66069", "16744", "64451", "39396", "14667", "58613", "25105", "13743", "48728", "18098", "62110", "48481", "24727", "65963", "53822", "59759", "49481", "14543", "38331", "7146", "39121", "62798", "55387", "7469", "69774", "60670", "33301", "6589", "6217", "33042", "70818", "32114", "50679", "63805", "31458", "1339", "61485", "64841", "9706", "51335", "61893", "49607", "71210", "16138", "33624", "33675", "65458", "72586", "47016", "64414", "25096", "27739", "10638", "30336", "16371", "72006", "68494", "46304", "5295", "45197", "53385", "9755", "23582", "7284", "13108", "42773", "61389", "24776", "61011", "57261", "55957", "38661", "44979", "62539", "18178", "11716", "58801", "34206", "39630", "57763", "65921", "37610", "14455", "9715", "18404", "23817", "19935", "24628", "61903", "58773", "11951", "2051", "31699", "68723", "36713", "47221", "29417", "57606", "41101", "31034", "826", "535", "22679", "33941", "7990", "18866", "46764", "58094", "13572", "71815", "26767", "60091", "33074", "19183", "1075", "53284", "34959", "36177", "12365", "10357", "47695", "52566", "51004", "36965", "2668", "24053", "21982", "37046", "16931", "68515", "30012", "60698", "61399", "15179", "33483", "67691", "14301", "66213", "2566", "46309", "68801", "10648", "70767", "16755", "47404", "45690", "6107", "6400", "73210", "15638", "35474", "14124", "70037", "11257", "64700", "33751", "51536", "36809", "63825", "29400", "53294", "61319", "49700", "22739", "1113", "56404", "8833", "31803", "67176", "62027", "41208", "69760", "26245", "50058", "11998", "32096", "41502", "37154", "2328", "17987", "50936", "16211", "12086", "26378", "976", "10772", "51289", "62876", "30588", "67674", "51381", "4602", "15148", "66412", "6525", "61434", "15976", "50026", "10020", "59887", "32468", "29610", "29305", "47958", "38009", "71487", "15970", "27117", "35283", "52168", "39495", "26417", "50849", "60926", "31467", "10318", "54761", "27147", "12503", "63228", "4023", "22948", "59704", "42699", "51055", "71529", "55558", "61557", "29145", "37146", "48680", "21223", "71514", "38037", "71998", "30670", "52161", "3308", "53358", "51835", "21444", "1720", "19560", "38934", "11401", "17730", "35782", "46698", "28361", "20505", "31460", "10463", "48878", "57638", "71425", "44842", "38034", "33621", "25", "28827", "14343", "31334", "40418", "58475", "27115", "45347", "57988", "66036", "4242", "26116", "59205", "17704", "25340", "59407", "72912", "58082", "10121", "24410", "66099", "15000", "14955", "154", "40272", "12641", "58855", "71721", "52095", "44393", "68007", "16656", "49374", "62565", "64554", "26476", "34555", "30778", "16234", "72655", "28414", "14854", "60493", "69712", "66269", "5008", "50151", "43235", "41000", "52352", "29913", "5310", "22143", "67905", "20079", "68581", "53081", "53314", "24680", "33827", "62916", "3058", "50844", "68980", "67647", "17332", "62810", "65714", "46016", "38224", "43690", "27708", "54055", "33471", "53514", "4366", "1665", "19305", "24886", "20421", "864", "36738", "15825", "14093", "64081", "21877", "8015", "51681", "52507", "39029", "24928", "37461", "40598", "21544", "29531", "61947", "21445", "10920", "54734", "16536", "35124", "12309", "32624", "27648", "7940", "32940", "53593", "39202", "37941", "43890", "7506", "71418", "6415", "14784", "56391", "21738", "28805", "53922", "72516", "69839", "2160", "48749", "35788", "52427", "35466", "20780", "65568", "6574", "15061", "21727", "39796", "64347", "49960", "45300", "67604", "45288", "19746", "4448", "18028", "14835", "32818", "63342", "19415", "65869", "60723", "60301", "47701", "46063", "14720", "57388", "69727", "34225", "54440", "54079", "41795", "64877", "58254", "30546", "30014", "44116", "48978", "12645", "69310", "36563", "31808", "40254", "55282", "62583", "40455", "13850", "9905", "27478", "2", "49477", "56617", "69380", "28899", "31209", "35811", "10736", "43342", "67589", "39699", "48581", "28702", "57804", "42755", "31155", "14761", "68961", "31100", "49535", "59347", "28461", "57043", "434", "60386", "67644", "34644", "57845", "63044", "7178", "6674", "31678", "36840", "66160", "71520", "35444", "55868", "1184", "25964", "36816", "47622", "16433", "4691", "72332", "26421", "1503", "10567", "32471", "17982", "36784", "59289", "37577", "49566", "53646", "62515", "7058", "3029", "60021", "16876", "45614", "36161", "2664", "11406", "3624", "19639", "6204", "47000", "45964", "21011", "1048", "41932", "21863", "42483", "14892", "17511", "13603", "42647", "37084", "27623", "9822", "64677", "67399", "1025", "59590", "64771", "57327", "43226", "56038", "31592", "4969", "53421", "36398", "54484", "53875", "4328", "41189", "16100", "37767", "41993", "56676", "28140", "70318", "43634", "69255", "52497", "21958", "58221", "18192", "58983", "38436", "53888", "10060", "63301", "63364", "58353", "42216", "48409", "9556", "14352", "33672", "44575", "17983", "13433", "72605", "69913", "16093", "525", "40149", "62408", "51614", "46457", "37311", "63133", "62438", "53784", "32626", "4278", "12978", "14309", "72715", "64738", "30451", "23411", "25008", "0", "42999", "43543", "70543", "16920", "2072", "38972", "59525", "21925", "66886", "16841", "4122", "2267", "62936", "3320", "1481", "56079", "59956", "13239", "6801", "37801", "31263", "38771", "58299", "48032", "47285", "29836", "66318", "37556", "6825", "41003", "64777", "41905", "40109", "54402", "36251", "21326", "53948", "882", "68905", "62794", "42652", "35671", "50747", "44207", "58745", "37859", "26176", "949", "71164", "34039", "57331", "43585", "8209", "51411", "24726", "66902", "22618", "1760", "32501", "59755", "70104", "26145", "7553", "34282", "36109", "35202", "23780", "3505", "64107", "36264", "49925", "14381", "43812", "69259", "26960", "32050", "32301", "35997", "37496", "5077", "21775", "67677", "46634", "22010", "62924", "20082", "56569", "8848", "33360", "24789", "55895", "28739", "11335", "46714", "11530", "69887", "1856", "64799", "63334", "60044", "20048", "14369", "26803", "54870", "52044", "20941", "51556", "63951", "20322", "13439", "65511", "52019", "26367", "25858", "56337", "6409", "62293", "39027", "38651", "20", "41143", "5093", "39160", "47504", "39996", "71736", "30040", "13868", "54936", "41291", "42485", "27921", "29418", "19481", "68394", "71095", "7971", "63376", "8371", "6935", "37560", "8620", "46814", "14297", "43626", "14554", "759", "25832", "35770", "22993", "19565", "16661", "28615", "71581", "59867", "16950", "18957", "59341", "69432", "23820", "64925", "503", "41464", "56084", "16979", "36640", "5580", "73194", "61916", "59076", "9509", "29806", "40855", "17718", "41690", "24772", "61310", "42136", "40461", "51364", "24971", "52094", "25142", "56008", "46222", "18186", "17236", "38325", "51124", "38744", "71137", "71892", "26667", "14587", "5702", "49873", "50105", "38091", "65439", "42448", "62620", "8227", "60963", "22235", "854", "71766", "36821", "5538", "22876", "54747", "58943", "24750", "26575", "24974", "31556", "27002", "11667", "51276", "25246", "27475", "14911", "25746", "1089", "34053", "68626", "56294", "50602", "30788", "25487", "27274", "37246", "26455", "44012", "39981", "2882", "41741", "24985", "23966", "45981", "72993", "19250", "56000", "49768", "8860", "22424", "52459", "54123", "11157", "2282", "62252", "63979", "50663", "27503", "26724", "19054", "30167", "15144", "68094", "37245", "5762", "67742", "27539", "65496", "7575", "46868", "28467", "5686", "32434", "9964", "59264", "59870", "24332", "56709", "20453", "12652", "17130", "23837", "46081", "2952", "53564", "67533", "34065", "30359", "20165", "45305", "20458", "55520", "67716", "26720", "68282", "27555", "69535", "56369", "34193", "62827", "66100", "9501", "4695", "55404", "43849", "9033", "70475", "10602", "34244", "40276", "24784", "53278", "1217", "44645", "39648", "17633", "4466", "40140", "21452", "5636", "1030", "42015", "52901", "40067", "30847", "25886", "15315", "5790", "70117", "9720", "70315", "20422", "51269", "19879", "36884", "68416", "50746", "53266", "65472", "38116", "23913", "7145", "31589", "2611", "42478", "69329", "68450", "12392", "30214", "944", "59600", "71625", "25261", "20290", "71489", "56388", "39152", "43873", "58276", "25667", "5690", "46139", "7675", "11906", "71334", "10202", "3414", "26117", "19351", "66713", "31769", "26148", "40232", "41728", "46318", "24864", "2338", "72426", "35041", "51633", "10265", "3052", "18909", "5909", "20325", "60416", "57256", "42300", "57760", "2409", "52026", "56718", "62894", "29746", "7283", "24046", "36193", "32690", "40748", "56726", "51469", "31888", "57419", "52286", "58984", "28491", "73208", "27108", "62753", "23459", "54577", "37906", "69808", "17460", "71709", "6916", "21924", "69490", "12635", "28313", "22670", "52388", "31289", "55528", "56393", "9638", "29611", "56048", "51842", "23622", "52719", "7805", "24420", "8708", "28095", "69102", "52973", "43022", "72469", "71836", "7269", "1803", "68469", "42697", "44205", "48415", "72891", "47553", "46287", "27163", "5273", "66623", "59255", "68314", "41032", "1729", "11131", "44481", "29211", "31954", "72110", "71791", "55974", "18296", "41581", "56", "11738", "70231", "64183", "61873", "27917", "37155", "60090", "13303", "29900", "14031", "7157", "31212", "13875", "24306", "5987", "8548", "70240", "38886", "49559", "25170", "31473", "59691", "29494", "60223", "45016", "35420", "27152", "7737", "26397", "57119", "15776", "4507", "34742", "31989", "22898", "20776", "25750", "894", "20692", "41285", "58705", "43009", "6738", "46032", "31796", "24741", "13455", "68310", "20334", "6055", "56658", "8497", "50823", "29022", "14505", "28303", "2220", "25483", "14648", "62902", "8649", "38336", "18863", "60672", "27919", "37416", "4928", "65121", "67270", "45615", "3261", "28914", "4444", "72941", "51095", "1776", "1388", "32290", "27887", "28730", "44464", "2516", "16617", "70492", "28547", "10248", "28976", "66969", "3397", "65674", "42904", "16431", "46118", "57742", "6332", "21456", "64904", "28733", "32524", "65089", "31491", "12171", "6494", "6914", "68339", "56398", "24638", "9454", "17101", "21539", "4410", "6349", "55432", "32671", "61324", "69853", "16162", "8123", "69550", "39608", "15408", "54106", "29783", "8854", "2647", "45038", "55467", "26855", "8448", "7876", "25047", "10030", "45287", "45371", "14022", "60917", "71406", "57543", "16092", "43332", "33603", "41652", "47458", "51222", "67224", "32019", "38631", "59090", "41509", "35898", "48167", "32437", "67234", "25405", "7034", "6856", "18491", "72710", "16160", "41771", "22771", "5834", "7619", "50912", "71849", "17673", "6624", "63601", "13065", "19458", "12966", "27027", "33266", "52888", "63952", "49010", "48780", "68805", "2489", "57494", "4117", "29287", "59667", "43102", "29917", "46945", "42034", "14751", "70739", "68104", "45136", "14105", "42030", "10420", "28569", "28801", "21189", "32212", "20416", "61282", "30033", "22480", "22825", "33327", "45104", "27864", "56844", "30659", "19341", "33339", "23641", "23872", "334", "5026", "66846", "38684", "36697", "13696", "28386", "23527", "39535", "19707", "63549", "47929", "58272", "57569", "3201", "19585", "72339", "30246", "34527", "52142", "48318", "38909", "47567", "53043", "10847", "69767", "45759", "15508", "49508", "55409", "40829", "68690", "3853", "23765", "16611", "24281", "20826", "14", "15729", "49747", "24676", "20054", "43955", "41227", "58401", "32076", "43805", "53952", "177", "71058", "58918", "20203", "55356", "22846", "54180", "12354", "19429", "27067", "3151", "34892", "71507", "7114", "72161", "10021", "48192", "59418", "66659", "27485", "55913", "21185", "15804", "12083", "13366", "24646", "50573", "22006", "15957", "7352", "63234", "3891", "66655", "18276", "50416", "47491", "10313", "32119", "25253", "42820", "2547", "999", "42514", "8997", "57412", "32816", "11172", "46523", "6296", "61883", "37032", "39530", "71038", "25041", "2226", "69020", "18470", "5200", "61388", "18110", "32438", "13570", "23532", "43792", "15679", "53693", "59284", "18493", "33156", "71276", "11039", "26190", "54767", "59710", "36041", "70557", "12193", "7401", "41127", "29013", "7385", "30333", "1923", "21199", "24491", "36633", "32719", "19654", "41754", "36747", "53395", "49879", "19966", "4644", "31593", "43055", "13867", "27637", "35966", "51015", "27771", "60979", "49229", "2180", "8228", "62330", "54247", "42983", "40135", "76", "13863", "17940", "37986", "29087", "45436", "29532", "21358", "574", "3170", "46365", "11447", "56847", "70616", "32279", "31369", "53918", "36568", "44070", "15667", "35231", "10525", "11003", "59420", "41301", "19028", "32676", "53534", "27705", "57101", "48847", "50388", "29858", "28992", "14412", "15863", "13320", "55596", "54266", "17480", "41272", "26977", "6820", "29829", "7304", "31170", "57568", "47537", "44131", "68362", "2434", "3456", "25195", "40023", "11469", "69374", "46295", "37862", "46195", "2127", "48469", "27208", "44944", "70206", "48257", "10734", "25989", "3556", "28567", "44775", "63998", "72466", "9769", "7091", "49593", "18031", "68107", "40495", "6626", "26972", "50461", "70960", "44495", "25708", "49005", "59724", "66602", "56641", "3469", "27976", "26236", "49281", "38989", "57578", "22495", "14570", "69463", "44049", "11325", "6154", "31738", "6809", "19212", "3778", "17564", "55513", "2243", "27579", "62754", "9786", "6285", "50012", "40049", "50470", "10249", "44515", "27639", "21779", "59177", "3262", "63596", "69443", "8215", "68829", "66978", "41215", "41166", "39923", "11054", "52495", "5308", "48638", "18858", "14293", "26439", "3689", "67084", "58148", "23403", "25942", "28038", "48316", "55412", "30683", "33737", "58546", "44423", "65120", "44892", "63784", "13026", "52710", "24035", "62374", "51537", "62463", "3366", "37391", "25419", "8251", "46410", "22510", "32090", "59077", "44772", "59638", "68959", "73198", "10543", "20204", "21150", "73096", "55228", "70583", "14457", "36399", "24600", "7344", "30516", "49568", "66120", "36471", "38085", "33210", "343", "9676", "37934", "15476", "71317", "16531", "10744", "59917", "36841", "59091", "46175", "72209", "13489", "52158", "38334", "4201", "59448", "17225", "8942", "55911", "60999", "59810", "49350", "53988", "7366", "71571", "18484", "67671", "12218", "21234", "9895", "1745", "2268", "62324", "42407", "53313", "38962", "39911", "67225", "28228", "33018", "16141", "51539", "26558", "71170", "54858", "48563", "41791", "55539", "47941", "25569", "65098", "50205", "69447", "20765", "73145", "45138", "32598", "25095", "67041", "3579", "57924", "73245", "62739", "12532", "560", "14045", "8083", "2665", "11804", "44365", "51200", "66687", "31920", "23187", "54086", "1133", "3142", "40097", "11677", "51068", "49959", "34347", "13262", "45204", "31743", "2238", "29891", "8232", "68539", "64026", "63745", "65766", "58030", "8243", "50107", "25421", "33387", "64410", "71341", "39392", "27463", "65132", "33620", "4072", "40961", "17706", "5797", "54299", "40017", "9107", "41866", "44625", "460", "68551", "27447", "52930", "6028", "50502", "16297", "64149", "66992", "54819", "54420", "23325", "54050", "26633", "3836", "65364", "28163", "60953", "14888", "33504", "7714", "71550", "45620", "31837", "64220", "17941", "10240", "14825", "19136", "42470", "43266", "19142", "64831", "11550", "52634", "41643", "38252", "16583", "52301", "70321", "69451", "33279", "9353", "33003", "62997", "14477", "23545", "36526", "56784", "68901", "41839", "65484", "42453", "45286", "57366", "10247", "19480", "63941", "22092", "20798", "25266", "59798", "18247", "37856", "26873", "22356", "19948", "52354", "48962", "20514", "59612", "7687", "64053", "59307", "28201", "50417", "22390", "34806", "72565", "38156", "24116", "32776", "10494", "20833", "14281", "14026", "26146", "9310", "52735", "63023", "29988", "70055", "30510", "6567", "12287", "51871", "37797", "53238", "14349", "44943", "3873", "53281", "40519", "42902", "51417", "32482", "18655", "12084", "66294", "61510", "16384", "58561", "36237", "41915", "45927", "69202", "55983", "65755", "62186", "60480", "44977", "34701", "69618", "72219", "68276", "14410", "44000", "20053", "37082", "38954", "21986", "25642", "14005", "8187", "28888", "24381", "1317", "50233", "61685", "42885", "29996", "37270", "33168", "58752", "15333", "1832", "824", "70602", "19088", "67200", "5616", "6672", "29756", "10753", "18861", "44098", "29969", "3871", "34660", "52518", "4181", "6872", "45748", "20339", "28444", "52951", "40837", "9622", "57054", "56274", "23475", "66754", "66985", "26163", "51822", "13478", "59510", "61030", "39303", "58467", "3528", "39325", "11425", "40942", "11412", "44405", "66092", "57067", "72238", "71804", "62642", "64488", "46512", "68424", "46859", "7544", "37371", "46784", "37261", "21783", "56560", "10960", "26893", "27939", "58901", "65754", "59257", "48966", "60506", "54495", "53823", "10277", "10996", "6341", "32939", "36480", "50090", "16367", "8173", "1730", "2098", "6523", "72449", "24751", "25894", "71472", "69152", "7327", "35491", "11339", "63479", "10721", "58468", "43292", "58558", "38193", "14996", "20752", "64426", "59060", "41892", "22373", "56473", "46464", "60212", "4270", "51969", "3783", "36734", "29562", "9301", "12637", "886", "43020", "45002", "3195", "38528", "61828", "18946", "6040", "61455", "28684", "71291", "28259", "7743", "6023", "522", "40867", "68998", "39971", "69417", "72486", "39452", "9687", "62889", "47588", "6145", "69994", "16159", "210", "68440", "49133", "25786", "53059", "41673", "18762", "67004", "57076", "36770", "10704", "45269", "38330", "26637", "60721", "66008", "49131", "70981", "72642", "2711", "49748", "6255", "54995", "14095", "18035", "39114", "26475", "61478", "60885", "20460", "28320", "38893", "18083", "65260", "14287", "40750", "38173", "11142", "57418", "15109", "33685", "17042", "26689", "9695", "10294", "12361", "372", "16415", "47661", "73147", "37189", "62862", "62105", "11088", "55332", "43163", "5350", "14316", "68788", "66462", "38207", "52684", "13281", "46255", "21927", "36903", "9064", "65243", "1177", "7191", "50465", "42875", "56652", "41252", "64918", "72603", "37442", "50897", "14660", "27716", "72217", "45897", "62959", "24851", "18698", "21462", "56846", "36118", "50159", "31356", "48625", "19083", "27047", "43184", "62475", "40758", "3358", "45404", "25717", "19773", "730", "21597", "47545", "19600", "27053", "37493", "70245", "61680", "19000", "45327", "42241", "42038", "14269", "48725", "12981", "70965", "5169", "42746", "25356", "17985", "41683", "40486", "72096", "33563", "10550", "47710", "988", "30093", "55253", "12476", "64374", "7454", "19307", "34811", "44313", "30405", "52092", "46687", "10188", "32171", "49497", "8845", "16486", "24892", "46393", "40406", "59919", "26800", "61772", "7161", "29965", "4163", "22085", "28910", "65616", "50997", "68638", "46978", "55655", "49850", "38015", "69290", "60545", "72592", "19249", "65585", "60948", "64561", "62970", "55931", "64627", "15359", "8164", "41163", "14219", "68932", "19674", "61978", "39037", "66888", "33594", "20650", "8847", "44077", "7727", "60572", "21696", "7869", "52318", "72523", "35279", "58609", "71547", "2522", "1242", "32727", "2780", "23352", "42881", "21197", "33463", "57888", "22169", "11221", "59912", "53248", "32942", "66309", "25456", "29122", "60709", "63483", "35174", "22498", "19112", "69759", "18152", "24225", "53257", "35424", "43160", "38904", "5445", "5564", "20820", "3160", "490", "39727", "59233", "54826", "48674", "17754", "36913", "42780", "48232", "18255", "46928", "11484", "28332", "5251", "66767", "16868", "6263", "53481", "58083", "51473", "72852", "40444", "17486", "9389", "19448", "1492", "59972", "62298", "12766", "18380", "36917", "16808", "46427", "14225", "21087", "7562", "49379", "31925", "70413", "27105", "63678", "68933", "44431", "3976", "38670", "15951", "41310", "4829", "60741", "55785", "4775", "73202", "63816", "39010", "45880", "68948", "2460", "48432", "42000", "1941", "40700", "53772", "21787", "39489", "17613", "18790", "2474", "19587", "59340", "52721", "9694", "47687", "16572", "20146", "72678", "24508", "67275", "20411", "46327", "53083", "69120", "7443", "24405", "53911", "70008", "13206", "11704", "46238", "67332", "72385", "1035", "73041", "637", "23745", "5792", "46856", "41478", "67190", "62288", "14753", "3068", "10367", "68791", "13094", "64013", "18134", "26947", "45015", "3585", "32599", "58156", "28264", "41473", "10507", "58931", "50329", "63053", "4372", "49364", "65845", "57559", "20024", "19097", "5384", "29571", "39229", "66945", "28894", "36413", "59681", "27042", "50843", "38544", "59929", "23121", "55104", "4102", "72651", "30022", "68130", "41256", "30403", "67963", "19132", "2750", "56989", "12865", "70941", "12612", "17267", "30904", "34093", "3681", "18586", "66322", "1357", "52438", "36127", "58035", "40985", "36324", "57649", "37616", "64557", "46065", "29540", "9875", "36466", "18147", "55792", "15820", "34590", "68573", "34992", "12765", "35433", "66098", "22714", "55893", "38416", "4499", "40813", "38444", "17624", "71897", "47129", "36308", "37958", "49211", "5787", "35842", "12929", "59811", "51663", "14426", "20150", "22603", "18637", "51917", "67628", "70279", "66728", "57244", "18089", "39928", "68206", "62318", "1189", "12913", "67375", "44002", "44270", "56021", "21348", "46296", "26449", "69003", "11141", "25011", "61650", "10315", "18667", "20380", "18375", "21214", "36244", "52783", "71295", "1822", "14189", "15033", "11380", "72379", "52203", "52278", "62240", "40698", "16660", "10344", "65310", "48258", "56604", "52960", "68415", "48102", "71329", "45389", "45261", "61693", "43528", "29082", "31602", "61923", "25611", "15735", "13527", "28028", "65813", "3154", "65537", "31514", "45673", "25097", "36135", "34384", "25320", "41306", "2340", "8137", "31680", "57210", "58471", "16877", "26926", "34530", "65769", "12024", "721", "18755", "13451", "6652", "31475", "3097", "72568", "26669", "15543", "20162", "48135", "60908", "43623", "22922", "56167", "3215", "18469", "70221", "21798", "14408", "24463", "37405", "58110", "19959", "10325", "59523", "11353", "17292", "56684", "43329", "20908", "43901", "23090", "33094", "40328", "50566", "49437", "4110", "32780", "53877", "12369", "38944", "56693", "44699", "38580", "69587", "29530", "68536", "64357", "51799", "13647", "20254", "16847", "5968", "62799", "36043", "39271", "58995", "65328", "55779", "34046", "5221", "38468", "30083", "11565", "18219", "45333", "8104", "43315", "68675", "58766", "62373", "33688", "30048", "71605", "15339", "14539", "65822", "65665", "5482", "58214", "10314", "51110", "15499", "57440", "33080", "49474", "20408", "42683", "21336", "8097", "43832", "70422", "60906", "32754", "48818", "8928", "18985", "51184", "71619", "71123", "39043", "68421", "34571", "64785", "55134", "56741", "36102", "29659", "25945", "17055", "13493", "60408", "56590", "53016", "240", "72530", "61240", "66286", "70840", "70189", "44420", "46486", "9646", "56758", "44067", "23011", "44768", "39390", "60791", "64881", "13350", "41148", "9121", "8562", "16117", "52185", "48977", "66979", "34255", "10011", "58599", "58600", "39549", "60580", "32491", "55984", "51580", "55495", "70365", "15353", "13674", "63644", "20437", "23895", "34352", "39853", "35873", "30008", "14876", "40311", "55908", "46165", "31892", "17944", "23967", "39219", "27169", "413", "65633", "21618", "40530", "53722", "37829", "17657", "30082", "25838", "45699", "20730", "70931", "70268", "39195", "49628", "2994", "27331", "35545", "68513", "36656", "10281", "8905", "41289", "39036", "11243", "6605", "45894", "18210", "32329", "15959", "5399", "38988", "49708", "53023", "21207", "27226", "30655", "52412", "60892", "52926", "61535", "39291", "8662", "31610", "51460", "65227", "19616", "48491", "32997", "4386", "19743", "42806", "17168", "13432", "15660", "2386", "25381", "55471", "70095", "51069", "54170", "13744", "21698", "57631", "118", "44840", "66709", "64565", "36123", "26673", "17636", "15851", "39462", "29006", "63016", "8170", "15903", "5914", "41308", "55964", "40541", "71793", "10237", "67429", "35112", "15789", "39056", "23205", "42560", "30365", "54337", "33911", "66077", "55273", "22776", "45447", "61636", "330", "8748", "44056", "43942", "7112", "50467", "36392", "33178", "39969", "27752", "71183", "41105", "27551", "30746", "15492", "50641", "40926", "32043", "53489", "61625", "23983", "73088", "32258", "12432", "66792", "62542", "24533", "35144", "36553", "37175", "23962", "34142", "20964", "50650", "62039", "25564", "7599", "8074", "21163", "60346", "6708", "66002", "49682", "53529", "4272", "62208", "47362", "29633", "57871", "41161", "70086", "21623", "2712", "714", "18699", "58868", "22712", "28308", "44126", "53962", "39476", "42815", "25822", "59560", "15936", "57282", "60533", "72738", "20612", "46751", "33749", "3620", "44269", "35964", "28905", "19015", "34643", "40178", "11522", "10730", "442", "69855", "46668", "46382", "4782", "14634", "24429", "53987", "59736", "39970", "10413", "53138", "9154", "20537", "31611", "39119", "23389", "59827", "62275", "34944", "30370", "50610", "40082", "1432", "70754", "13196", "2070", "13999", "68106", "61096", "14821", "25587", "792", "4759", "62672", "28030", "39150", "44081", "24557", "47004", "3755", "67281", "6492", "64544", "13503", "67945", "24620", "16728", "33333", "44753", "36830", "10234", "60478", "61598", "65803", "48869", "60756", "34071", "21781", "59807", "35017", "53579", "16787", "9439", "5748", "13758", "10363", "8229", "1954", "22622", "33754", "55506", "24157", "5246", "56711", "60362", "36597", "35649", "54629", "64641", "35044", "40424", "40967", "33892", "21740", "72561", "15031", "46216", "19258", "59575", "21313", "31774", "12819", "46461", "32064", "57171", "45935", "71576", "63351", "17205", "70432", "64098", "30961", "15846", "9788", "15535", "35334", "70836", "53389", "37444", "37605", "33929", "42690", "17776", "36233", "11959", "35631", "4953", "59234", "34664", "14617", "5641", "30053", "41132", "23751", "40828", "7087", "28194", "59757", "3614", "5392", "49913", "19488", "47965", "24497", "70239", "58099", "39736", "35483", "64895", "35449", "72779", "12992", "17599", "66408", "55969", "62758", "39927", "14563", "68659", "6493", "39065", "17508", "18832", "33948", "29580", "17815", "43411", "12214", "2017", "43487", "16866", "65891", "64230", "71229", "9069", "31420", "10250", "52840", "52843", "71221", "29787", "30635", "62119", "54892", "26188", "41960", "7522", "52772", "2046", "33828", "15256", "14639", "69980", "31347", "50036", "951", "20062", "57548", "55530", "54265", "63272", "9002", "64340", "63254", "18401", "29316", "2043", "65965", "24796", "68168", "67729", "26951", "10986", "30529", "5559", "57669", "15116", "35393", "71224", "28200", "33910", "27469", "34663", "18049", "45829", "72861", "1510", "72734", "19333", "4329", "3440", "49876", "43998", "49571", "37421", "8294", "12266", "60341", "58946", "65984", "46209", "3762", "40659", "8793", "20693", "18977", "69874", "4030", "61254", "58541", "6917", "51006", "40721", "54332", "41920", "71972", "23221", "68947", "26194", "57761", "38154", "15699", "41547", "35214", "12777", "47863", "6706", "69842", "55197", "55443", "58517", "37381", "56479", "72664", "38480", "45365", "32572", "15065", "28423", "65199", "5298", "38343", "50639", "18198", "18016", "9863", "16262", "12660", "66402", "43910", "3986", "3017", "20180", "56804", "11715", "6988", "4710", "72019", "15772", "40805", "20635", "59172", "47176", "62554", "24308", "21448", "13182", "66214", "45619", "31232", "29769", "61513", "35158", "2247", "57767", "68271", "45616", "68117", "71877", "32937", "72081", "5676", "57616", "18773", "59413", "11712", "62467", "8861", "66906", "64316", "67709", "7778", "9302", "32369", "43180", "47722", "3870", "68761", "37722", "28286", "4104", "47475", "34477", "11016", "374", "40565", "61403", "31205", "20598", "31858", "57020", "19216", "21399", "14771", "1377", "61051", "32346", "25425", "27598", "72492", "59698", "4804", "7685", "2875", "5565", "12810", "62992", "57684", "58061", "5662", "54944", "52949", "59079", "45597", "54522", "16395", "47542", "49669", "48394", "20899", "27035", "47798", "24168", "62930", "23391", "17561", "32121", "24810", "49334", "33416", "64348", "66553", "19477", "17861", "3379", "62949", "42335", "62403", "18330", "31741", "8207", "53232", "66636", "24799", "29874", "44795", "39353", "61214", "49051", "20402", "22209", "24026", "63689", "850", "58747", "17829", "53187", "52639", "66916", "42268", "36769", "35820", "73214", "46853", "72472", "69116", "32734", "26458", "20593", "17705", "55339", "15305", "25219", "12909", "32286", "71928", "56657", "57134", "69505", "65958", "46460", "49272", "35413", "1374", "53635", "67998", "13312", "6629", "66248", "66761", "61375", "23940", "43525", "50043", "18398", "8014", "67988", "59249", "63795", "8420", "7679", "10820", "54680", "46590", "11824", "50815", "28406", "44157", "46547", "54895", "2388", "24682", "32316", "71685", "5561", "6150", "9793", "27239", "62867", "8375", "15656", "22772", "36740", "17458", "17017", "2602", "1626", "54618", "12814", "9612", "15883", "17569", "40587", "10877", "49190", "39368", "39101", "61431", "34174", "61523", "7470", "25610", "19780", "15063", "31355", "41685", "8510", "50405", "20737", "56866", "59800", "2076", "44198", "51487", "23871", "20801", "46619", "33703", "26794", "71251", "53406", "37473", "13787", "72021", "65627", "406", "49744", "2907", "42259", "72489", "22114", "68473", "61317", "50286", "11749", "3580", "51127", "55682", "58788", "1886", "25987", "65518", "46153", "53140", "38662", "40648", "5230", "12626", "47228", "933", "1994", "6018", "62130", "45361", "59299", "55240", "69585", "27055", "23972", "7061", "11037", "4519", "55505", "13476", "72757", "24131", "33433", "61594", "39030", "46850", "72425", "25647", "33637", "47686", "21979", "42848", "32345", "44737", "30342", "60081", "7222", "43771", "19181", "36717", "69247", "25840", "60590", "49572", "4703", "44228", "20508", "37293", "47571", "47956", "63293", "23084", "19009", "24710", "22720", "40978", "64549", "30421", "66877", "12421", "61576", "72929", "16041", "35574", "17954", "46976", "27629", "38126", "10966", "44904", "42606", "10384", "7671", "32605", "37500", "5801", "18304", "59116", "38786", "23982", "61069", "7905", "30138", "26330", "772", "37764", "57242", "50229", "11711", "64606", "11509", "23852", "72282", "28543", "3339", "48739", "57279", "21858", "1147", "60927", "28315", "20539", "17149", "54005", "17033", "70601", "40188", "29139", "54361", "4946", "4086", "6186", "69726", "23716", "5153", "382", "36263", "27473", "26251", "37975", "53894", "39839", "37468", "20836", "51041", "472", "62989", "25668", "23877", "68765", "65312", "19881", "33076", "63953", "18583", "20843", "3483", "17822", "9736", "45364", "67039", "8800", "48917", "42181", "22781", "33974", "71126", "43763", "15720", "24478", "9873", "24874", "61052", "60140", "34029", "27400", "11832", "22758", "2381", "67136", "64419", "16575", "51680", "38209", "37361", "42439", "61801", "34671", "64074", "2648", "12732", "34782", "17563", "54596", "50558", "11575", "16339", "12442", "12902", "8108", "16151", "62174", "30639", "1452", "53538", "24780", "36748", "49279", "5101", "38721", "19609", "70418", "41775", "67774", "62496", "52266", "17021", "52249", "40018", "31479", "2844", "44299", "6369", "26086", "12567", "58672", "2044", "61578", "28229", "63800", "22186", "28451", "25016", "64463", "19478", "49240", "71652", "17581", "67873", "62264", "23382", "26626", "15643", "66374", "72835", "68902", "4356", "24029", "25264", "38308", "45872", "14325", "5336", "10304", "7254", "65209", "21395", "44169", "73225", "630", "22609", "25685", "28413", "58673", "25525", "6032", "30432", "71753", "69641", "35402", "61193", "28349", "17083", "40839", "3967", "5494", "13589", "50357", "17400", "11875", "18950", "16665", "20957", "54120", "70828", "6888", "56619", "23318", "56957", "15096", "37060", "30177", "38864", "58173", "6328", "71178", "8606", "15281", "57810", "33818", "66284", "60701", "60309", "51709", "8525", "37670", "48253", "28105", "26467", "10350", "35999", "53057", "13604", "36080", "23854", "52420", "56052", "22844", "63000", "4063", "50122", "21548", "27344", "15761", "46024", "23400", "59857", "1619", "1734", "21387", "17456", "65831", "7946", "41714", "53837", "22849", "5891", "30226", "32292", "53884", "24336", "16270", "8625", "35554", "515", "39912", "16056", "36464", "20524", "34774", "3006", "70311", "31994", "55583", "63990", "59058", "41213", "58959", "27602", "9672", "46292", "31499", "58422", "37319", "15463", "49683", "23447", "1201", "24595", "27330", "49732", "68710", "53228", "41082", "53568", "27358", "30508", "43557", "66751", "10474", "20973", "25955", "26872", "3217", "68463", "60458", "46057", "19453", "61950", "68616", "13048", "473", "36824", "58490", "27187", "18480", "39710", "43866", "7171", "39858", "41073", "22511", "11223", "14249", "40554", "70324", "44359", "13063", "46162", "21776", "66976", "53736", "34762", "72731", "18224", "60318", "12441", "68702", "42334", "7077", "46441", "36352", "51609", "25280", "26739", "48789", "23752", "68489", "55680", "45217", "1864", "18825", "67694", "37477", "59501", "12773", "65228", "45788", "11673", "267", "50724", "41016", "25148", "46847", "3157", "64825", "42705", "16490", "55334", "27181", "47842", "50056", "35766", "22294", "37790", "65512", "32282", "62436", "9630", "37417", "14742", "23438", "36823", "55668", "69979", "72099", "31683", "56012", "18561", "54173", "3473", "69060", "28802", "3369", "10229", "55153", "55873", "12597", "43385", "11019", "69426", "28191", "2909", "9215", "67154", "56045", "51581", "21624", "31904", "953", "7367", "27970", "62724", "23473", "54119", "27723", "15709", "57641", "23860", "51198", "71926", "59433", "33635", "15389", "49994", "30049", "28246", "32152", "66052", "36348", "37578", "6059", "36596", "59149", "21308", "48182", "18815", "20302", "39395", "70904", "14910", "33340", "30830", "54739", "57978", "9506", "38937", "14492", "23095", "40713", "40409", "57997", "4344", "20034", "1000", "45464", "59488", "65097", "41499", "32014", "14120", "19636", "58835", "18196", "63464", "6090", "20603", "46562", "39162", "63462", "30795", "41957", "9957", "9946", "1154", "38559", "29508", "11117", "15277", "37197", "19888", "41198", "21027", "762", "30465", "51912", "70288", "61746", "46799", "54638", "24276", "55601", "62234", "72811", "18442", "15579", "34034", "70325", "50883", "29502", "14008", "15706", "21763", "19042", "30105", "26755", "34038", "54084", "53515", "17094", "44068", "31236", "71153", "34098", "60696", "20092", "40600", "47856", "7070", "57689", "16114", "11922", "14736", "66059", "23258", "45323", "10524", "50424", "66604", "72543", "51658", "19170", "5439", "13932", "49425", "9945", "54947", "31981", "63260", "70629", "18414", "51982", "56981", "32150", "27183", "53658", "64145", "21178", "40155", "63012", "5712", "42101", "58596", "61611", "36807", "32431", "61229", "55351", "57868", "10729", "11040", "28756", "29954", "62134", "49711", "33374", "14707", "33700", "38619", "20029", "3140", "35306", "6921", "68389", "4767", "58544", "65719", "1597", "46628", "66126", "57799", "16", "49194", "56037", "4901", "2170", "52251", "36300", "49009", "64772", "71730", "30450", "4179", "40512", "15081", "16562", "61981", "38491", "7650", "49632", "65048", "43377", "65130", "35498", "48219", "9341", "40163", "30835", "4265", "2629", "16905", "22649", "2569", "51117", "12399", "26084", "50618", "39078", "31814", "40778", "3511", "57541", "2511", "22456", "67949", "30735", "31240", "50048", "401", "72791", "69780", "34235", "58989", "32995", "38118", "56899", "922", "82", "13585", "19210", "57395", "56462", "73042", "31923", "46721", "32167", "3057", "44304", "38808", "50651", "167", "52370", "7584", "65923", "42807", "14790", "13901", "27461", "41689", "52796", "72618", "13316", "70704", "72453", "47377", "64616", "46849", "43098", "71331", "20299", "27278", "11208", "56810", "822", "51700", "30412", "218", "13996", "41119", "38429", "2083", "61422", "26284", "31924", "6273", "45199", "5418", "14448", "59840", "35084", "26608", "67834", "29631", "39670", "22338", "51911", "5724", "22126", "14750", "60026", "2604", "33495", "6895", "33207", "65038", "1261", "42146", "24712", "8142", "55832", "46599", "55098", "42750", "55433", "50376", "36827", "37655", "52244", "40881", "60985", "8874", "2402", "24944", "22694", "24863", "33300", "48281", "37251", "17733", "17750", "14422", "48267", "65740", "32265", "3374", "58029", "63332", "39263", "14358", "2707", "4363", "8590", "21357", "34610", "37526", "31689", "33881", "21046", "14922", "30681", "32146", "52269", "28051", "16478", "57027", "51952", "18092", "6698", "37016", "16122", "144", "26350", "5880", "69615", "5029", "49501", "53545", "62072", "30770", "38873", "4840", "17384", "40525", "9906", "55762", "44059", "25201", "66908", "6524", "19807", "6079", "20091", "9919", "22144", "67465", "42802", "46564", "32025", "61295", "35517", "3707", "55398", "39009", "48088", "16163", "67640", "4741", "65424", "63931", "30781", "43583", "7806", "16393", "33554", "68364", "62416", "35473", "9186", "34073", "40069", "49421", "26836", "58751", "48228", "31388", "52732", "35220", "49337", "5536", "22227", "61620", "58351", "30745", "12270", "23269", "50002", "59051", "48926", "45670", "49678", "9221", "59894", "50114", "44623", "61302", "1522", "49728", "14604", "19491", "33564", "4089", "72", "15036", "39144", "65910", "49911", "5228", "51818", "17869", "45917", "64926", "22764", "49557", "44835", "49786", "59143", "17234", "59496", "60495", "42316", "45552", "43146", "4056", "72581", "3495", "5346", "17122", "28837", "6367", "9452", "53254", "62829", "8346", "69540", "15687", "13227", "44506", "19356", "11396", "48770", "12300", "61596", "26140", "43723", "49299", "27920", "35799", "6009", "26773", "37555", "61220", "16411", "66786", "57773", "18408", "45806", "65572", "19225", "34443", "69117", "19605", "43780", "38710", "35909", "27442", "14832", "56928", "47029", "11747", "13523", "61322", "50179", "17089", "49737", "14770", "59576", "68794", "53757", "3888", "72301", "26172", "34715", "32174", "44361", "22454", "5518", "42149", "4411", "47084", "5609", "71002", "47446", "53171", "54295", "26345", "52344", "40963", "47707", "31372", "32162", "3619", "57212", "44997", "15861", "72770", "38652", "3148", "69113", "46623", "71597", "15937", "20702", "31254", "24055", "66110", "41913", "66746", "42891", "60162", "18306", "71277", "41199", "18366", "51048", "26131", "48862", "33154", "29246", "47872", "2057", "3856", "72103", "59837", "57346", "36003", "32467", "30227", "71266", "2542", "58768", "49183", "45870", "55102", "1735", "54775", "71258", "68232", "63527", "68529", "70792", "4150", "31424", "10601", "41641", "44748", "4814", "53799", "21394", "55397", "7845", "526", "4924", "62879", "69551", "19682", "62180", "42424", "45985", "37563", "9169", "61958", "60372", "121", "11080", "12489", "53289", "28209", "47454", "9281", "67241", "14762", "67706", "59524", "64243", "47830", "13369", "32556", "56966", "43894", "24106", "69191", "54003", "32371", "42129", "60513", "46062", "33129", "42145", "23808", "6841", "54035", "62362", "17913", "61225", "63604", "44927", "53746", "37525", "21165", "50296", "54827", "65703", "52499", "17690", "55370", "34756", "30030", "45688", "50348", "29688", "68445", "57528", "72657", "14595", "66501", "38049", "8492", "55573", "69809", "56915", "62181", "3003", "29224", "27803", "17753", "72629", "22504", "46312", "10785", "32526", "13852", "46966", "51129", "72051", "9184", "3519", "58039", "26487", "60949", "59735", "63600", "17453", "1043", "1529", "54407", "25735", "57094", "13692", "7570", "46579", "39310", "34293", "58407", "41177", "32917", "70737", "29366", "19024", "10066", "54691", "48687", "39572", "31734", "70699", "68768", "17957", "6497", "43877", "14625", "49081", "70316", "42077", "15652", "5299", "27576", "14403", "15882", "12215", "57490", "21318", "36068", "47847", "54520", "45063", "4353", "30195", "61785", "13162", "12665", "37129", "9678", "7207", "10375", "19733", "48074", "69817", "47014", "68816", "24737", "17597", "45595", "69252", "22211", "46989", "26917", "30107", "1640", "64200", "64297", "1521", "30627", "20913", "25061", "50898", "17799", "689", "4349", "63901", "5869", "51233", "51587", "62649", "12316", "57627", "59367", "39724", "69178", "483", "36022", "6898", "14313", "2812", "29700", "33396", "47548", "45229", "39488", "333", "45393", "68446", "38211", "7991", "40983", "67432", "60640", "10266", "69269", "25515", "10359", "71102", "41004", "70586", "13251", "11216", "72628", "7747", "61572", "59267", "55025", "45250", "55819", "10542", "53551", "18875", "51771", "6152", "7992", "51794", "64113", "6791", "19558", "19071", "259", "35477", "34825", "7212", "40435", "28923", "58085", "29785", "21810", "32869", "24734", "31547", "2657", "28216", "19835", "25467", "4258", "46729", "3691", "70781", "59841", "48634", "32047", "16012", "59336", "61097", "33354", "11326", "64260", "53581", "60909", "42473", "30379", "25230", "39394", "56794", "19474", "44332", "35988", "63767", "25927", "55921", "51206", "7780", "54333", "55329", "27568", "65595", "16294", "39995", "12919", "48813", "61890", "11835", "71786", "37874", "67317", "71650", "5426", "24284", "66807", "57108", "66016", "70604", "14499", "52529", "23853", "36384", "12521", "21215", "43138", "1259", "53963", "21273", "20985", "48890", "10201", "2876", "60846", "24878", "36064", "59470", "7368", "34994", "72396", "12452", "49059", "61206", "161", "565", "43154", "19379", "64021", "740", "63669", "19929", "33800", "38565", "39869", "1897", "35373", "68259", "67291", "35241", "7319", "53661", "8880", "6160", "53271", "887", "4891", "60969", "1699", "45635", "22792", "39584", "33900", "34263", "33199", "70848", "42104", "48783", "1159", "47013", "25958", "63192", "60529", "69043", "69778", "67564", "48865", "344", "13724", "21217", "20147", "37864", "45245", "42238", "16252", "6103", "47719", "34299", "15531", "1412", "4979", "51065", "4145", "25440", "19029", "36826", "58257", "44899", "54557", "1958", "24000", "70923", "15017", "13894", "22861", "8086", "38696", "7482", "54455", "12726", "10389", "46545", "43483", "3107", "51240", "60463", "55483", "57870", "38045", "4419", "52139", "59881", "40553", "21259", "70847", "62153", "2831", "46823", "5725", "13588", "68789", "38876", "50503", "71358", "8549", "50966", "66782", "56754", "1350", "42643", "50736", "64644", "60822", "8", "61954", "42869", "30721", "46937", "16943", "18176", "13625", "67269", "22146", "53361", "61089", "10714", "20480", "51695", "51305", "5448", "2689", "10866", "26480", "45649", "1814", "22308", "10492", "17909", "59904", "9621", "19275", "63387", "48243", "7705", "33615", "72001", "53372", "45884", "16303", "20115", "45512", "21640", "23274", "64745", "22382", "20995", "58172", "10919", "30944", "42789", "43735", "27434", "49372", "11849", "640", "36802", "62508", "4074", "55453", "57360", "56856", "54613", "29545", "55420", "10870", "68817", "15348", "17297", "7130", "53053", "59252", "49688", "60267", "40065", "50421", "32158", "50676", "51113", "10807", "68633", "21729", "2663", "64800", "12304", "10818", "16552", "11198", "8033", "38534", "16344", "65634", "73027", "59794", "1194", "39470", "34316", "67782", "15791", "20141", "15427", "63365", "6360", "49658", "4153", "56087", "48004", "2529", "54822", "15812", "59883", "31535", "2035", "17339", "8339", "37390", "43275", "5377", "4625", "71451", "21033", "2155", "29802", "18111", "44254", "48515", "9467", "18951", "29577", "35478", "63176", "53539", "63109", "14049", "26514", "19364", "37828", "57905", "1712", "9642", "13299", "122", "71792", "60176", "3576", "12595", "13010", "66129", "58142", "21575", "61103", "37286", "5111", "65425", "51872", "31476", "68169", "1651", "22389", "1963", "23952", "49290", "11374", "6465", "47550", "26903", "34859", "24633", "59382", "6105", "16705", "30649", "67050", "3119", "38782", "39074", "29134", "63290", "63188", "29646", "72499", "56483", "14374", "64279", "55024", "20003", "54694", "53107", "46117", "19245", "24549", "52243", "47209", "20350", "33842", "43661", "67611", "37516", "27261", "32298", "23806", "71923", "6133", "64164", "23028", "27230", "40196", "27247", "47699", "49989", "13459", "539", "62210", "28734", "7772", "31312", "70233", "39116", "41430", "45775", "67010", "9104", "59864", "49667", "4730", "4761", "23439", "33134", "10848", "58678", "17331", "16282", "49694", "39039", "25183", "65964", "3108", "63564", "41945", "12712", "40580", "73040", "72927", "15319", "34026", "311", "19638", "6828", "45631", "15105", "33897", "29536", "24480", "25092", "32794", "22239", "47273", "40863", "26537", "6804", "26366", "50533", "56779", "7628", "70869", "53333", "59551", "23514", "11916", "42166", "39709", "7872", "67768", "18087", "57429", "26134", "32253", "28755", "24634", "10563", "14847", "66326", "20125", "26601", "44001", "37345", "1181", "20682", "46671", "42920", "14773", "40610", "63492", "63403", "52876", "1204", "9343", "7275", "36512", "20075", "20239", "51058", "44846", "52367", "49958", "22461", "2463", "24365", "45658", "64052", "55728", "21534", "31798", "10158", "24477", "69880", "12044", "55149", "69764", "52837", "7279", "50306", "25491", "72677", "62482", "46136", "65767", "53018", "16232", "28577", "28017", "34903", "47092", "71823", "46352", "23542", "42413", "24842", "50653", "49726", "42053", "59779", "68926", "53169", "53955", "4424", "11413", "54042", "46436", "13851", "11356", "30001", "60405", "62887", "3867", "5668", "1196", "26782", "29386", "310", "25920", "60905", "119", "63386", "20976", "12681", "21049", "28954", "26493", "4243", "58862", "69583", "67268", "43885", "37195", "64909", "45922", "64815", "27093", "65010", "38223", "51573", "62414", "16215", "73146", "6474", "63752", "14006", "57115", "66142", "37262", "10814", "30580", "21351", "44779", "60711", "52439", "12406", "14311", "23002", "54670", "37959", "43789", "64106", "51900", "8727", "24363", "27001", "49199", "52827", "11962", "35122", "1974", "4559", "32866", "37662", "48702", "32027", "6372", "10941", "54418", "32989", "27074", "20979", "61822", "17189", "34725", "17831", "58465", "26889", "68869", "16456", "29229", "34084", "63145", "28227", "67717", "41496", "51491", "28093", "16640", "58581", "38205", "37125", "60255", "57829", "2913", "34080", "35063", "44627", "41153", "7769", "47465", "70916", "65379", "17555", "11631", "18061", "36356", "2616", "48771", "58352", "54800", "41172", "2806", "62461", "38072", "56967", "40907", "60440", "66391", "8617", "8716", "42515", "11614", "19479", "11298", "25268", "7668", "64339", "31875", "19827", "68161", "15264", "33789", "10388", "10217", "54392", "846", "38400", "35047", "7155", "72336", "19857", "15175", "56157", "53022", "14507", "50955", "1924", "21845", "47173", "58817", "50324", "24632", "37463", "31827", "33048", "34253", "64422", "13179", "4655", "41822", "2577", "60648", "39566", "61189", "46749", "18717", "24290", "7645", "55517", "24128", "44013", "25145", "35993", "69070", "40709", "16929", "62286", "16607", "64948", "8244", "71182", "49017", "64207", "44463", "3916", "32489", "17755", "69635", "51221", "38231", "58438", "31829", "21026", "39666", "13683", "48661", "29593", "1020", "45786", "46598", "65249", "42207", "55856", "38646", "27816", "16550", "53938", "54404", "61239", "45624", "55454", "5218", "27517", "14459", "27491", "61001", "26710", "29752", "62068", "6807", "59447", "4174", "70328", "4042", "25738", "53609", "37869", "11666", "41283", "20814", "39964", "58305", "69394", "46639", "64956", "8044", "65059", "12363", "34136", "71964", "4365", "42693", "29533", "45230", "39334", "61350", "62811", "6004", "57318", "5885", "35615", "53474", "38117", "56545", "27519", "52950", "62427", "24142", "58933", "9874", "56144", "56283", "71146", "36163", "60494", "50170", "32184", "56425", "9296", "34853", "62832", "43269", "65990", "53144", "34659", "5633", "43939", "27370", "4642", "70449", "35502", "16328", "55488", "31656", "2330", "35481", "45996", "72760", "20620", "35270", "5144", "21277", "41873", "42731", "29000", "29534", "69054", "43056", "54043", "53521", "64744", "19166", "44878", "43774", "30869", "4609", "19895", "41769", "35252", "18579", "48413", "72661", "27114", "35263", "41917", "63973", "17215", "35611", "72513", "47439", "47401", "58988", "51574", "70672", "54673", "27324", "28654", "31027", "37452", "30268", "7458", "36899", "27452", "4396", "23718", "1624", "39562", "17659", "31403", "25887", "20950", "3563", "31832", "67874", "26349", "15644", "59486", "14400", "28278", "19787", "38541", "21777", "45581", "23207", "22328", "45513", "64506", "30339", "29894", "15544", "46151", "50534", "26621", "47200", "11879", "71899", "7846", "71575", "25364", "43865", "44919", "6443", "19669", "7629", "51375", "20937", "55894", "29049", "71707", "9730", "13038", "57648", "60629", "31300", "23847", "69674", "10290", "16719", "7088", "7149", "53512", "68564", "14179", "32691", "8117", "59970", "65222", "57768", "12735", "68871", "22699", "35542", "69050", "69145", "37229", "56946", "33990", "63893", "37010", "4987", "59753", "66928", "30183", "48181", "19298", "53615", "65749", "17000", "16990", "23288", "61045", "18123", "62346", "40552", "60039", "27109", "50482", "72159", "17035", "20788", "63167", "48263", "532", "21716", "9342", "35521", "21486", "64232", "10658", "29982", "58411", "45990", "64261", "24196", "63565", "61042", "14515", "38999", "52466", "28454", "20378", "20462", "65093", "38702", "51199", "45264", "73038", "23349", "36890", "33806", "32323", "5471", "67854", "2323", "41204", "30351", "47638", "29090", "34027", "26596", "791", "7867", "56889", "29220", "44220", "63927", "34302", "46265", "25257", "56893", "59457", "22428", "60928", "62849", "52799", "6442", "21630", "280", "2841", "43339", "68142", "16783", "64510", "38764", "4182", "64516", "51943", "54864", "25269", "598", "10253", "41277", "14559", "70726", "34777", "14760", "4367", "16212", "1854", "60742", "13838", "42144", "18629", "20376", "19371", "3416", "52035", "40724", "67692", "69222", "52671", "38865", "45866", "33417", "58978", "18345", "700", "55292", "59012", "6720", "6513", "5866", "38826", "1579", "13243", "6815", "1788", "48807", "18285", "63509", "70625", "69645", "31367", "27402", "58134", "65126", "46720", "21585", "8336", "56397", "33735", "4467", "67656", "63191", "64629", "53045", "17170", "37469", "8409", "2992", "47772", "25338", "9719", "12221", "27968", "54131", "56506", "1928", "15909", "70236", "54196", "39596", "66150", "33297", "10221", "30316", "21916", "46666", "6102", "49033", "26376", "64551", "3120", "16549", "64039", "54814", "25348", "71075", "62735", "59019", "71820", "28595", "12490", "40578", "67919", "20482", "26299", "10457", "41667", "50974", "50031", "30613", "70430", "44057", "62732", "44717", "32099", "33125", "18518", "2573", "63517", "11013", "56771", "15060", "9268", "38080", "72643", "69465", "39511", "56129", "26691", "6170", "54000", "10450", "69325", "39768", "24917", "43751", "48065", "65148", "6013", "56935", "26252", "52866", "65699", "66935", "10269", "43821", "30496", "58865", "9911", "42492", "41542", "30561", "65801", "59415", "5229", "66615", "2118", "37807", "19968", "35619", "68867", "30184", "7444", "59517", "15984", "3809", "22065", "57839", "63622", "30422", "16966", "65856", "10430", "54999", "52517", "12563", "50947", "66799", "19505", "42228", "26249", "61626", "47344", "10296", "35756", "8582", "30147", "35809", "29656", "50537", "5998", "28045", "64225", "19823", "22648", "48940", "23665", "41210", "40550", "68004", "25742", "66596", "68148", "7919", "24679", "8638", "24488", "8613", "37731", "51187", "50478", "69895", "57036", "2859", "68545", "47555", "55711", "40705", "3704", "16678", "19423", "32284", "27662", "32565", "61875", "69929", "69069", "54619", "9644", "4613", "67531", "38320", "53537", "64242", "17942", "63746", "44801", "58059", "44045", "60718", "44751", "42116", "46593", "28773", "30373", "9590", "54409", "24180", "130", "2281", "59570", "1955", "12614", "52239", "2307", "10912", "58163", "22640", "47899", "37905", "60270", "40991", "10970", "67935", "30658", "64334", "8601", "2159", "4255", "68554", "21891", "48517", "43457", "51718", "34096", "120", "16502", "49613", "51106", "27809", "9385", "50068", "45893", "9476", "71467", "60468", "43512", "30400", "34858", "23770", "19172", "39974", "28873", "9851", "37015", "44746", "7536", "68065", "43868", "71151", "22553", "18468", "28578", "39594", "56427", "14700", "67125", "47177", "35736", "46197", "38042", "11957", "61995", "41942", "17830", "36239", "48131", "47408", "66410", "30855", "54321", "44302", "37881", "38690", "17521", "27788", "34295", "40366", "52543", "13311", "28870", "45867", "61698", "32024", "65583", "49652", "8580", "11108", "19267", "68281", "18130", "33224", "64599", "19625", "34513", "31597", "20095", "7633", "56542", "8480", "57471", "2889", "73013", "21517", "35791", "70985", "10332", "42117", "5490", "7651", "27428", "41788", "10710", "3016", "14330", "44651", "67668", "29986", "65480", "61372", "50982", "38664", "29073", "16952", "38082", "30511", "55436", "59117", "40588", "11866", "71009", "2513", "61448", "41405", "22285", "9596", "24047", "19457", "20832", "51447", "63222", "62804", "8216", "72049", "1023", "67322", "13242", "72727", "66569", "5404", "10812", "1326", "25630", "60915", "1058", "46719", "22821", "45702", "13442", "60843", "64396", "33268", "67356", "41019", "53856", "29396", "55925", "32640", "14201", "62702", "46160", "55350", "3649", "59696", "6219", "47573", "28866", "8167", "2638", "15888", "67212", "10635", "36391", "24452", "41330", "33084", "40345", "22178", "10025", "68011", "52294", "30540", "52246", "52664", "33378", "69486", "46103", "53683", "39491", "21920", "72809", "67364", "39947", "71386", "43367", "58279", "27731", "70477", "3429", "44533", "24208", "20279", "55166", "57194", "28800", "4912", "5699", "56697", "18611", "26238", "6311", "36284", "11340", "30629", "29210", "50913", "66593", "23740", "9284", "64338", "13662", "54650", "8591", "19446", "49154", "64835", "15728", "45953", "39365", "72457", "43684", "10073", "13815", "15300", "50915", "45684", "13110", "23453", "25414", "31449", "70607", "37427", "20615", "52214", "51108", "59383", "38401", "70232", "65563", "15969", "65293", "23609", "24082", "16173", "4037", "66717", "55416", "12657", "4885", "6793", "22122", "45322", "34169", "64848", "54345", "27685", "72299", "62381", "766", "61870", "68657", "3999", "3367", "68886", "40483", "19462", "26427", "72168", "69106", "45942", "12364", "34854", "4022", "31919", "6591", "36240", "52471", "69977", "2023", "57358", "18391", "34453", "39338", "9830", "61033", "71851", "52821", "26481", "25804", "15824", "33879", "52366", "3668", "53945", "19800", "41872", "5622", "22282", "50289", "6550", "1336", "56456", "9749", "39253", "55478", "72196", "47611", "16452", "39000", "39619", "47827", "14013", "36256", "51725", "3714", "18271", "69004", "26338", "6560", "62128", "46699", "34516", "52447", "12670", "36894", "27957", "29484", "4716", "22853", "13675", "65560", "62244", "42726", "64466", "36354", "53629", "42980", "13664", "36162", "39274", "70768", "39894", "33769", "117", "39871", "68224", "23736", "13081", "36156", "70994", "66347", "58736", "795", "12344", "65394", "60412", "56459", "192", "51189", "51442", "66855", "52800", "13653", "39987", "9072", "61043", "4354", "41390", "72529", "67990", "51992", "32646", "61367", "11838", "3347", "19467", "14915", "61473", "60417", "73181", "10192", "36442", "40757", "25329", "18620", "35352", "69267", "39921", "26866", "2361", "36058", "21899", "8242", "54972", "4477", "14191", "34498", "17102", "43881", "40177", "44047", "15330", "5738", "57711", "44715", "63104", "47189", "67636", "6228", "49301", "59417", "48093", "831", "40147", "34831", "4142", "8078", "69503", "42218", "69352", "1483", "14749", "71035", "54316", "33559", "11349", "8762", "22905", "59966", "63005", "35003", "31730", "58002", "43541", "4997", "61176", "9969", "9272", "39771", "19043", "21832", "34938", "8960", "4588", "68146", "13643", "5166", "52798", "69354", "41474", "72647", "69831", "55023", "53885", "68963", "33648", "68555", "12376", "36344", "66787", "62447", "1573", "51832", "11201", "43994", "61127", "48904", "37507", "72635", "7379", "19294", "31228", "26934", "22824", "56061", "55549", "43766", "10136", "65183", "41848", "3370", "70730", "60942", "49443", "57971", "1150", "14161", "10203", "22352", "26151", "62602", "18261", "21054", "45945", "29652", "18764", "3491", "44188", "10239", "44935", "36991", "13420", "61670", "36703", "54728", "647", "597", "49771", "28055", "14370", "57191", "71482", "40933", "54925", "66378", "53956", "59212", "27492", "41843", "58808", "15798", "47823", "31014", "10876", "3786", "30640", "64477", "48597", "17996", "24778", "71327", "43430", "15775", "31903", "46802", "40000", "6113", "16873", "22180", "3131", "51698", "51762", "17907", "49001", "61709", "24020", "42895", "15340", "997", "10169", "58748", "12798", "38268", "39333", "52417", "17009", "3869", "2077", "7963", "49014", "2521", "26796", "7334", "23640", "55284", "47969", "45567", "17883", "17492", "65061", "29349", "45554", "15910", "57737", "13365", "43892", "43305", "67119", "57132", "28971", "38458", "38548", "30624", "39307", "10299", "42584", "30525", "18713", "21662", "33660", "57923", "67559", "11179", "50885", "69457", "29598", "48428", "22946", "3106", "16135", "53291", "5059", "47711", "25486", "51267", "19854", "72742", "53541", "6683", "50591", "20506", "59085", "42644", "37141", "41393", "13106", "69393", "72880", "8752", "52696", "61918", "31329", "7987", "285", "65790", "29397", "28810", "45226", "10948", "27853", "28984", "38001", "23392", "23659", "18006", "60269", "19428", "29910", "69041", "46128", "67363", "8574", "35249", "63562", "32195", "54494", "2112", "24237", "55611", "20557", "58508", "24207", "67401", "65477", "9129", "59876", "20495", "23508", "68351", "71464", "16441", "49063", "27085", "55716", "31974", "10842", "43451", "44015", "19246", "59885", "56719", "47949", "19416", "50609", "1508", "67661", "51897", "34658", "6692", "47851", "71440", "70286", "69904", "67897", "72273", "45353", "42254", "29925", "49238", "46717", "22525", "11883", "16219", "52036", "39250", "11748", "13689", "24705", "494", "23866", "37260", "37104", "57841", "30576", "27350", "2687", "38107", "65204", "40472", "43061", "61806", "59096", "63503", "27259", "57267", "49169", "46022", "33921", "35987", "57478", "72091", "12105", "42159", "19417", "34709", "13238", "2291", "36678", "65372", "26932", "5850", "41437", "17404", "21771", "16756", "59549", "39806", "24080", "15821", "45470", "81", "33740", "14027", "44189", "28830", "9438", "16930", "62135", "25614", "11807", "64015", "14215", "37779", "60681", "57703", "56678", "8341", "69797", "47819", "42457", "55683", "44890", "68865", "33774", "22318", "60651", "38162", "16268", "56530", "24366", "28781", "46047", "40059", "32980", "34228", "6336", "52627", "62448", "3188", "16869", "65975", "48831", "24172", "4807", "31926", "37827", "26886", "40337", "55457", "29150", "35288", "45975", "35715", "70436", "66389", "34635", "51450", "26589", "72243", "51601", "14536", "17524", "13948", "69336", "46275", "67182", "3387", "19752", "68561", "26110", "63843", "53900", "45603", "1460", "1786", "67724", "31567", "68546", "14294", "60389", "34305", "51864", "59852", "36320", "34876", "11674", "59926", "71199", "63357", "9782", "44757", "31722", "59740", "19717", "1967", "42713", "39435", "24721", "61117", "64358", "21439", "48571", "36722", "44982", "62592", "49286", "20493", "16034", "61404", "9659", "62994", "10698", "47786", "62719", "7340", "33470", "53089", "21095", "61968", "68071", "49982", "64423", "23593", "68429", "30893", "26776", "16648", "51878", "43944", "23739", "6673", "48608", "64092", "44148", "39220", "58930", "18172", "56838", "51788", "24361", "42797", "19532", "44680", "61005", "31098", "38", "72279", "11390", "45419", "835", "69665", "45655", "5277", "26650", "44247", "32341", "47982", "426", "43719", "50656", "10452", "22183", "37548", "35679", "38804", "27016", "39147", "29893", "26457", "45500", "5703", "72448", "17956", "70483", "26106", "32968", "38131", "16329", "7534", "35546", "54235", "15694", "30072", "67410", "52957", "40250", "12345", "22697", "1006", "64853", "35467", "39197", "67280", "55126", "70306", "40925", "42488", "33337", "14712", "17531", "6890", "49629", "73070", "31376", "30617", "52770", "57836", "71056", "38604", "6844", "8514", "56785", "69625", "37331", "23072", "71349", "18445", "71496", "54149", "47207", "68188", "17875", "67883", "33871", "9809", "24493", "41895", "6439", "15646", "6927", "41130", "67452", "49955", "70728", "30493", "10827", "3512", "30401", "70374", "11084", "26288", "26788", "9437", "51838", "3703", "6418", "89", "17010", "38328", "29804", "2937", "44869", "30286", "68082", "40287", "69910", "12159", "69199", "68022", "68880", "62951", "10162", "30222", "35530", "7724", "477", "24472", "28947", "25851", "67627", "48221", "22936", "57438", "19191", "58026", "64535", "72794", "60290", "57090", "9800", "65242", "44187", "20132", "71665", "28941", "14514", "8050", "34373", "61387", "63309", "17695", "23887", "18183", "14475", "57751", "36798", "66763", "41087", "58936", "24571", "60706", "52426", "35567", "20515", "17804", "35313", "65606", "45391", "73170", "41154", "32650", "6767", "20728", "57780", "65924", "71628", "28558", "50409", "52814", "67034", "60919", "54346", "40405", "65315", "58500", "67792", "30404", "21015", "23767", "11688", "1913", "50942", "54268", "4405", "3322", "49880", "7969", "30231", "1219", "70158", "50308", "15509", "42094", "54280", "68919", "59353", "68092", "13236", "51809", "36543", "21098", "39812", "55998", "499", "20038", "68039", "57033", "36469", "18990", "24861", "14076", "47562", "66780", "65325", "59428", "25232", "65124", "59948", "55789", "18432", "56809", "63399", "41717", "31753", "64086", "29592", "534", "27078", "1771", "6319", "20342", "5870", "68115", "17639", "20922", "60929", "62088", "70227", "51659", "8592", "40920", "10562", "17908", "44728", "4825", "9063", "50021", "10394", "14647", "46615", "57701", "43", "68264", "1447", "56883", "36077", "10490", "34649", "32810", "62594", "15423", "48448", "11935", "69211", "38003", "67072", "42329", "30532", "47516", "2301", "47067", "66674", "13345", "40354", "12395", "17474", "48502", "1328", "8458", "27084", "41329", "46955", "15496", "68322", "44918", "51989", "18102", "27987", "51688", "72849", "37913", "17966", "35647", "13892", "31705", "61989", "54179", "46386", "55375", "39931", "54380", "63199", "72755", "22158", "2169", "33272", "71875", "52842", "52709", "69610", "5548", "62803", "71044", "11820", "44037", "18726", "38242", "36513", "31383", "33283", "65369", "4692", "8719", "30344", "648", "4583", "14035", "63535", "57686", "53450", "39762", "18298", "33545", "63828", "69200", "19563", "52276", "59344", "37979", "18217", "39701", "39180", "61952", "64894", "19614", "69714", "67484", "50964", "33689", "59941", "38210", "49161", "34872", "32824", "47653", "9276", "8321", "30151", "65821", "71133", "67022", "15304", "16382", "16655", "60281", "21398", "41764", "19593", "10206", "22411", "43553", "666", "14943", "51572", "13701", "42386", "69480", "58543", "19439", "40263", "52290", "46662", "66090", "53524", "37213", "62361", "40538", "32876", "53883", "61571", "28907", "6874", "28722", "29594", "18238", "423", "47061", "3455", "9242", "31652", "50246", "11136", "39574", "19318", "5900", "41811", "11348", "41644", "65746", "43408", "44467", "15369", "65617", "64433", "48798", "56970", "64275", "16342", "43258", "43925", "45538", "36844", "42598", "49141", "31670", "51237", "54080", "47470", "2814", "41361", "24418", "2029", "10258", "11532", "63026", "34855", "14112", "53435", "28908", "49995", "36273", "63126", "57228", "19969", "8385", "67000", "5014", "21782", "9059", "31003", "21113", "62646", "3712", "15250", "46467", "48408", "59197", "42196", "10028", "62000", "50767", "29119", "61082", "42156", "35054", "51956", "20255", "38792", "25571", "12791", "65004", "6548", "53354", "11854", "9218", "48576", "17714", "28470", "14086", "4266", "30350", "17991", "55262", "65178", "24586", "18682", "43309", "61849", "39054", "53805", "28917", "35185", "68016", "610", "41451", "14394", "44850", "54974", "62227", "47119", "22778", "16279", "58268", "4114", "28422", "8055", "67398", "42634", "6490", "66570", "69725", "57555", "12415", "40689", "3325", "40260", "54331", "47106", "9631", "65138", "51795", "14376", "60491", "51808", "10089", "45482", "55279", "39421", "65463", "54006", "56862", "1227", "43002", "41170", "67579", "40467", "20015", "59550", "21509", "20163", "18314", "12679", "63489", "30054", "66072", "72108", "47771", "46387", "61079", "23066", "67168", "59028", "9293", "70651", "59210", "58197", "22842", "32575", "48732", "6642", "50970", "42914", "39914", "48719", "49795", "32588", "44517", "54640", "11987", "20464", "19997", "12705", "27346", "46201", "36527", "37617", "12872", "65590", "60898", "8217", "8926", "64070", "20120", "15928", "54543", "33259", "26217", "47100", "60394", "56263", "57560", "67797", "38901", "25003", "41999", "4395", "16554", "65622", "9374", "25865", "69166", "5133", "6586", "20727", "30232", "32975", "73006", "4079", "42716", "15647", "40873", "41550", "52195", "13131", "12378", "24375", "52576", "28698", "69734", "42100", "44867", "10214", "66071", "7854", "52063", "73024", "71096", "43567", "60087", "10232", "21963", "45316", "55322", "48117", "6845", "3865", "72138", "32887", "21070", "15504", "51923", "44788", "18496", "72721", "2974", "20531", "16823", "54146", "23141", "31445", "10173", "62312", "49128", "65696", "31462", "35519", "48297", "22610", "15589", "863", "50665", "35348", "55267", "53697", "2736", "37471", "26450", "9074", "18569", "7441", "18070", "23184", "14350", "56060", "23914", "70945", "1438", "71510", "58335", "43749", "7945", "6747", "33506", "10757", "19939", "31987", "38043", "64468", "57598", "4313", "28146", "21023", "5243", "17487", "4471", "14962", "29676", "28582", "11451", "22366", "43870", "62875", "40342", "12013", "52411", "723", "19003", "70058", "1686", "55932", "14495", "39203", "70310", "3893", "52336", "55202", "6981", "36019", "17890", "28706", "69687", "30996", "8603", "35844", "2212", "4169", "31193", "62813", "51057", "23747", "27397", "18522", "17972", "8433", "46463", "17387", "32392", "48299", "14787", "28965", "11471", "27667", "2601", "43234", "67378", "49764", "69279", "58697", "19138", "16319", "53493", "20869", "27038", "12664", "47499", "19108", "67006", "1600", "70250", "13616", "52736", "44709", "61179", "45284", "54946", "22818", "7664", "70289", "42578", "66895", "48022", "54697", "24068", "64550", "16887", "33172", "67613", "60841", "20541", "33106", "32234", "43696", "69153", "25807", "70599", "21939", "24378", "48445", "59376", "12234", "31204", "46090", "35695", "3238", "48036", "13762", "7495", "69541", "9744", "65664", "43777", "5618", "63453", "3581", "27361", "61550", "67155", "28187", "58913", "59416", "14055", "2929", "52643", "23092", "9053", "173", "68049", "12060", "47286", "4249", "13050", "66153", "51666", "53934", "63832", "9190", "45782", "71364", "11166", "28737", "66428", "68227", "16962", "33779", "497", "33223", "8201", "33197", "34700", "1671", "7652", "27897", "47725", "18268", "38477", "10439", "53840", "45989", "43452", "13742", "50813", "36440", "15196", "4589", "33219", "67319", "57099", "2488", "68312", "66464", "10515", "39209", "7138", "42175", "22270", "50331", "19278", "73213", "34870", "47240", "10369", "1485", "5224", "37861", "62483", "7111", "30315", "58236", "40269", "23265", "1314", "63889", "12387", "1437", "13940", "28764", "39093", "70534", "22245", "48126", "4", "20370", "15177", "62066", "28653", "6797", "72003", "18081", "34042", "48819", "67483", "18114", "32948", "70943", "27223", "66435", "71887", "28752", "28256", "47946", "45741", "68157", "46091", "7477", "14906", "62149", "3736", "47508", "49340", "10205", "27930", "21576", "37340", "3434", "43034", "43744", "31028", "46540", "71735", "12110", "66155", "30929", "32041", "34095", "50317", "52688", "73180", "62696", "57914", "34961", "59732", "48055", "73080", "64166", "51270", "62752", "50631", "11561", "25466", "63612", "2754", "28020", "34370", "43954", "46563", "23430", "38910", "66516", "58067", "22455", "32350", "69765", "35638", "67525", "26538", "30712", "3343", "14122", "43197", "8654", "67756", "28388", "7654", "25089", "39602", "31216", "16642", "41511", "53687", "29796", "45467", "65245", "14780", "69340", "71139", "25042", "13549", "29024", "34378", "14615", "38828", "24241", "42397", "57435", "51977", "42465", "13380", "23703", "71324", "44443", "36395", "21047", "67298", "16669", "6665", "68576", "43698", "53728", "64352", "25291", "54785", "36383", "64522", "42931", "23015", "4790", "55342", "68220", "21481", "2414", "12313", "25748", "48960", "25339", "72214", "72302", "58889", "39247", "49538", "14902", "33893", "7174", "71963", "60560", "14568", "60108", "40176", "41046", "15581", "58416", "26080", "2410", "64473", "5697", "10801", "28705", "49291", "50172", "47787", "7384", "20331", "57047", "49715", "62320", "18378", "48344", "53843", "19722", "21607", "73004", "69468", "1707", "29681", "69889", "20735", "68141", "12165", "38008", "54306", "64850", "68949", "7357", "9171", "49758", "44961", "69382", "7892", "61733", "30984", "8604", "32589", "36153", "63200", "36663", "11418", "35875", "52315", "24535", "37715", "17532", "56378", "42028", "54991", "50466", "58591", "4575", "41168", "8021", "46635", "7843", "7667", "39820", "4508", "10540", "67313", "893", "49516", "6446", "45263", "25923", "156", "66430", "32145", "8397", "45524", "38125", "24822", "44082", "35558", "13822", "66145", "11361", "43153", "59651", "55896", "22346", "33428", "41918", "56117", "67300", "70102", "45540", "56757", "20275", "5205", "70485", "29839", "43732", "26060", "4580", "15617", "46655", "33277", "44348", "47072", "13208", "1091", "25279", "66317", "4732", "27254", "21066", "25034", "48067", "7626", "29144", "25513", "43784", "2337", "55247", "26007", "50451", "44782", "35829", "47122", "62905", "14826", "8528", "51557", "19034", "10182", "5598", "71678", "43232", "65980", "62665", "3353", "70280", "26754", "34767", "21687", "3356", "34079", "50457", "52581", "11894", "64750", "50328", "43325", "22809", "12404", "62596", "34499", "43752", "21170", "33644", "21148", "32915", "12861", "30290", "27687", "25085", "25673", "13613", "1409", "52299", "70852", "61088", "13297", "19300", "34585", "2578", "495", "14738", "27289", "47222", "16067", "35002", "1888", "37723", "40056", "68764", "31149", "48175", "68046", "73188", "53039", "37106", "62390", "70372", "19866", "44975", "35126", "3342", "8184", "1182", "62912", "65514", "29200", "60536", "68030", "12109", "72044", "6614", "47692", "42283", "65039", "59637", "26031", "66808", "54298", "4495", "49871", "54067", "72938", "39580", "4699", "11059", "63305", "6880", "54797", "6536", "23951", "50818", "28072", "26260", "44999", "2084", "45402", "19994", "44738", "64692", "49546", "24854", "27387", "53224", "320", "36014", "54366", "3858", "71904", "68337", "52573", "49890", "56109", "31211", "26639", "72430", "35222", "17995", "70729", "49475", "57484", "33157", "18879", "6800", "28770", "52335", "32545", "55180", "39809", "15075", "17984", "3910", "33023", "47388", "39885", "11668", "27699", "33371", "35106", "58910", "70763", "22355", "12782", "71262", "56819", "31392", "50864", "32723", "30558", "69965", "26970", "12972", "14896", "62048", "7219", "28877", "69921", "65776", "42325", "41040", "22741", "68147", "19796", "20711", "2662", "70320", "38618", "48641", "41951", "55011", "17443", "28848", "22392", "53953", "27770", "49627", "47179", "3169", "56013", "45266", "39058", "40315", "17794", "71825", "2199", "5871", "5460", "63139", "6578", "63568", "46105", "72508", "11135", "28177", "6337", "3894", "67115", "27037", "38409", "64610", "11844", "40190", "14461", "64658", "29552", "47243", "8079", "48887", "17666", "46368", "405", "71168", "12719", "72702", "33767", "53519", "2354", "38147", "12973", "70277", "56715", "29726", "36004", "7704", "25358", "62323", "45856", "43701", "38241", "71641", "12418", "13002", "70721", "569", "49986", "70593", "71664", "34011", "20568", "5191", "24227", "10603", "3173", "57545", "32199", "42916", "25538", "64217", "28275", "27228", "62263", "20900", "50281", "30779", "14983", "54636", "45073", "49679", "19908", "67612", "45330", "10150", "70536", "20554", "3622", "40346", "23296", "55962", "60769", "70668", "25906", "44706", "20589", "65244", "25961", "71819", "63974", "8888", "70298", "19016", "27806", "65233", "48881", "26625", "44993", "39882", "15270", "7479", "42086", "53547", "68543", "59771", "6488", "11354", "26122", "64630", "49614", "26548", "66809", "317", "36569", "11699", "20527", "26878", "54562", "53125", "19412", "24603", "52739", "71304", "3103", "3064", "70332", "1666", "52327", "10404", "55456", "14946", "67108", "71559", "57215", "48264", "56552", "32717", "15742", "29718", "27826", "6486", "57688", "19926", "50675", "46820", "42563", "21167", "5769", "57520", "8989", "13519", "58631", "6985", "26675", "18395", "13632", "10462", "23067", "6952", "1312", "5020", "2968", "72303", "25313", "21825", "40226", "19204", "72072", "2433", "6180", "10551", "18534", "22860", "19353", "65983", "56123", "43091", "20356", "18382", "67434", "24698", "44476", "36198", "41196", "14385", "55824", "35860", "38603", "33651", "36265", "23480", "49770", "2384", "47005", "37867", "11384", "52751", "6464", "52585", "48350", "44967", "58014", "34372", "22775", "64935", "62060", "69256", "15092", "8318", "6414", "62281", "65902", "21546", "28623", "68357", "56297", "22687", "24661", "7142", "41257", "8611", "20739", "3043", "56500", "61826", "16383", "9068", "9037", "55220", "53261", "17111", "64626", "45476", "28561", "14881", "30068", "68113", "44978", "42542", "38936", "37489", "30193", "45298", "50436", "22201", "9688", "68769", "60276", "8062", "16606", "37547", "2317", "51586", "69896", "21432", "43125", "1225", "42333", "50088", "66315", "36667", "31963", "16874", "55674", "65944", "43157", "33939", "30196", "22967", "48666", "43423", "36914", "63187", "12330", "59275", "12634", "52841", "55879", "42573", "66619", "8325", "13150", "60030", "65557", "5158", "4235", "53309", "51748", "46775", "43456", "19812", "19023", "65647", "6563", "29950", "70101", "54139", "55914", "71592", "68025", "59137", "35675", "40919", "69876", "20168", "25547", "20065", "2856", "67250", "37324", "50996", "9553", "37428", "36255", "53500", "53896", "4828", "43816", "63361", "40562", "64589", "7399", "22191", "5617", "27768", "56576", "55719", "68814", "46433", "47700", "36833", "4155", "19814", "72596", "35694", "69034", "15280", "69031", "63265", "55678", "58394", "12066", "55560", "32787", "3745", "11577", "15108", "21996", "26369", "9901", "41398", "22684", "18118", "46497", "16864", "55923", "46860", "43605", "4865", "73050", "10123", "13698", "62999", "24621", "52914", "11408", "62231", "67550", "67412", "6482", "56553", "50523", "34210", "23264", "68008", "46145", "27868", "1565", "17826", "39299", "36494", "19118", "26973", "10790", "31578", "35741", "19301", "61437", "49173", "21248", "42664", "17631", "66579", "31052", "68840", "62553", "67342", "39159", "14436", "13426", "59900", "7884", "15114", "24253", "55880", "40433", "47316", "56229", "4143", "53611", "17338", "40441", "22565", "10881", "28396", "37691", "41916", "15051", "40626", "13766", "35091", "59968", "64596", "67840", "11950", "19037", "61922", "62664", "57532", "37727", "28411", "4720", "67657", "51265", "44707", "27540", "13950", "29186", "53858", "23270", "72157", "18283", "63602", "26630", "3021", "28013", "60234", "1765", "16360", "3300", "52562", "61825", "44008", "48788", "25578", "15392", "16110", "61170", "17679", "48459", "12416", "4887", "28222", "22856", "40620", "35961", "71485", "52021", "50175", "50493", "6284", "36214", "67238", "10774", "20917", "56112", "58684", "52130", "13741", "34276", "20943", "7894", "56638", "19841", "45639", "61428", "39704", "42487", "47011", "4105", "73107", "45711", "34860", "10999", "63442", "26805", "40706", "51875", "29046", "63975", "39846", "71689", "72239", "61930", "24262", "65091", "38747", "71144", "9366", "39826", "61035", "54941", "66085", "8122", "50204", "19006", "68074", "7730", "61102", "6596", "23784", "5921", "68823", "69514", "34631", "63010", "29759", "48334", "30505", "41284", "25090", "16893", "40450", "45522", "46922", "58492", "2824", "69642", "55808", "38892", "55418", "23771", "64625", "63370", "59988", "33255", "14262", "30223", "58050", "28069", "69559", "3303", "26546", "16004", "15739", "58189", "68336", "11955", "28230", "31579", "26656", "36932", "23567", "35630", "59471", "14474", "70866", "59263", "70821", "10286", "52282", "64308", "12228", "26248", "54230", "14125", "1288", "34313", "18413", "50753", "21584", "35024", "12247", "22255", "35105", "67134", "31481", "50247", "48008", "69736", "21355", "18538", "44084", "61999", "69820", "68461", "20740", "6174", "16584", "68481", "23511", "986", "69634", "34331", "15731", "12042", "58895", "61247", "46115", "63358", "61327", "47919", "20564", "34504", "7882", "18871", "45516", "15491", "40043", "49790", "57465", "38796", "2197", "61020", "42932", "25040", "71286", "31319", "27753", "48791", "37938", "31946", "34201", "9481", "36893", "52113", "63084", "49949", "60239", "60956", "32074", "55567", "16833", "19508", "50267", "31527", "64519", "68122", "53479", "49343", "72485", "18307", "5669", "54589", "61184", "51962", "18225", "26408", "65879", "23454", "64153", "677", "30125", "49429", "68799", "13052", "56671", "44350", "12904", "16528", "57889", "2203", "1627", "24371", "59939", "32172", "185", "47525", "3797", "44317", "21122", "15162", "61532", "55952", "43394", "46827", "26890", "71635", "70980", "61027", "53236", "67887", "30679", "43740", "22179", "51317", "35389", "72858", "68611", "70212", "105", "8155", "38789", "69500", "33676", "2770", "33826", "34364", "15002", "13543", "65530", "38799", "61781", "17677", "48792", "38099", "56148", "57668", "6291", "45078", "18576", "24297", "26255", "31855", "23240", "64954", "19219", "7278", "59858", "1757", "36810", "17347", "27019", "22545", "49654", "14852", "46880", "55381", "64393", "46220", "51446", "28002", "29879", "48391", "31308", "59680", "60782", "30928", "30661", "34662", "4390", "27823", "6191", "8553", "24446", "29206", "9527", "61008", "42041", "42059", "54867", "16790", "38943", "44619", "29137", "19337", "15620", "3496", "44559", "2101", "30139", "57112", "30296", "18888", "40070", "67076", "24693", "18883", "12360", "28330", "38602", "17716", "4649", "72955", "22591", "20307", "71847", "22596", "10364", "16481", "70765", "8627", "47275", "14713", "5320", "38256", "18002", "36627", "18944", "2515", "10276", "67199", "10288", "67433", "58070", "49055", "54879", "55557", "31436", "49084", "8331", "19889", "39830", "32831", "24529", "6458", "22386", "53390", "60552", "42650", "42663", "7784", "7185", "11061", "7504", "24541", "58185", "72577", "16863", "37072", "59812", "61627", "14909", "61173", "15784", "51422", "41260", "71808", "27614", "8179", "45388", "64479", "55993", "14992", "312", "57217", "21272", "72262", "71971", "59790", "46020", "57064", "15314", "58174", "65143", "5978", "38420", "69769", "62782", "2073", "8474", "63502", "21922", "20690", "4672", "64776", "24730", "32071", "43346", "29470", "48738", "4014", "57351", "8376", "13025", "60287", "54754", "46650", "70333", "37273", "34925", "9369", "15098", "71959", "39621", "40568", "7560", "23144", "65882", "50829", "52410", "27063", "10417", "32748", "36926", "189", "26164", "44038", "25518", "60193", "46092", "23425", "53578", "59765", "42639", "48185", "37923", "43981", "40261", "32871", "60229", "52218", "68409", "2650", "64418", "2640", "5430", "45952", "46919", "23166", "16177", "60227", "25206", "47593", "67087", "35229", "41269", "26681", "42023", "48671", "15173", "17228", "5292", "44701", "44936", "12520", "8630", "66424", "50691", "27414", "8427", "52328", "3663", "71160", "47320", "32983", "18093", "40533", "22482", "39451", "64003", "17739", "58809", "33412", "23513", "35392", "14802", "50789", "42602", "71209", "9231", "54109", "3599", "42147", "56125", "42201", "8348", "31443", "38286", "63122", "55644", "17671", "5729", "65462", "53330", "13533", "7490", "53407", "43558", "70912", "24192", "43926", "26486", "44281", "30277", "54704", "2673", "59831", "30171", "3933", "28266", "11097", "61935", "38441", "39440", "62678", "52180", "72722", "56575", "71941", "23366", "48921", "27333", "6290", "59665", "12038", "43340", "43303", "59865", "10334", "37101", "36338", "41060", "29446", "2140", "55473", "63645", "47513", "72378", "62268", "6347", "48790", "11162", "56406", "62082", "28189", "38136", "8819", "38608", "49968", "64205", "14160", "41634", "31042", "62284", "30883", "68520", "1186", "20105", "67293", "30354", "62023", "30592", "20538", "63092", "51330", "30641", "1292", "70151", "8231", "49692", "11091", "24575", "24883", "49168", "26999", "64933", "38438", "11600", "30939", "72670", "30042", "52907", "37373", "31455", "8946", "46330", "37883", "21302", "9017", "44747", "7729", "7173", "11035", "33709", "44981", "11252", "5067", "28214", "9462", "59726", "17981", "73036", "47", "34146", "19574", "37161", "28120", "10002", "34156", "67971", "63699", "46023", "23006", "54908", "37771", "47724", "21588", "22656", "57155", "65645", "47464", "37484", "70603", "29945", "68105", "26502", "36559", "17697", "7442", "30977", "72825", "19783", "43662", "65897", "6623", "20738", "56573", "67431", "10833", "43768", "6711", "55396", "49367", "19358", "68680", "71557", "60454", "15064", "56159", "57325", "49900", "8495", "61735", "70258", "28353", "18265", "60293", "59381", "22295", "2670", "12630", "67889", "20741", "40851", "42157", "23239", "44811", "66124", "59622", "71659", "44929", "17691", "20942", "30642", "72576", "16128", "19656", "4324", "17503", "35877", "10582", "19156", "71442", "44971", "38655", "12592", "12117", "49510", "14914", "16843", "61655", "24624", "15732", "20695", "51295", "18216", "63625", "59678", "53857", "65652", "21262", "58012", "25520", "10533", "54934", "33057", "63242", "19700", "58153", "48157", "63544", "33366", "11369", "37094", "61726", "52755", "52071", "53296", "20419", "67446", "2964", "20970", "6249", "19817", "3452", "64384", "37182", "57851", "41618", "16327", "22709", "47531", "24746", "5440", "32996", "18059", "52690", "32777", "67218", "67089", "62532", "46524", "25849", "18045", "30491", "3711", "65147", "51707", "49630", "44641", "20270", "19440", "7900", "44752", "63886", "69789", "63306", "68810", "20856", "69786", "29007", "45163", "27982", "41682", "18711", "49458", "41660", "15329", "8835", "60987", "41341", "23835", "5623", "1206", "25701", "64321", "56464", "3998", "22303", "35029", "37488", "29121", "32733", "46366", "53420", "55198", "34834", "8964", "19877", "24278", "32097", "55805", "47370", "10179", "34077", "24220", "26842", "18174", "19386", "14829", "70665", "70782", "4025", "55238", "2735", "66421", "42287", "61601", "58527", "25734", "23621", "64773", "37406", "9951", "70553", "59583", "45831", "66871", "40202", "15209", "49496", "6123", "17668", "51916", "47049", "60569", "50993", "66030", "28533", "71705", "14065", "36463", "53117", "9551", "4518", "70034", "10308", "25789", "8129", "65779", "32633", "35614", "36613", "13719", "42084", "11557", "59025", "47586", "40119", "17656", "48944", "14547", "38100", "28679", "71145", "64879", "23633", "38816", "1474", "65023", "9477", "52715", "46421", "12219", "17479", "39060", "55542", "21411", "22070", "50756", "62854", "27958", "18286", "22170", "13416", "68390", "50547", "64903", "35082", "25516", "51352", "1968", "59241", "22779", "69528", "20825", "71743", "64440", "42841", "40348", "33655", "5323", "28990", "63482", "33839", "61454", "26976", "24333", "25067", "22238", "3222", "29587", "23457", "64130", "12803", "36612", "24599", "45147", "48450", "27426", "41183", "1645", "30101", "24877", "34165", "28132", "45549", "46013", "65982", "24109", "66938", "32363", "67321", "27179", "40852", "18437", "6520", "37401", "67030", "18776", "17694", "6551", "2833", "49918", "71140", "31537", "69768", "41432", "8895", "5202", "51007", "61629", "32682", "24707", "9487", "55553", "65878", "10838", "13343", "26473", "25287", "24356", "36288", "51370", "5283", "7753", "23869", "9975", "57861", "542", "71466", "18758", "32470", "194", "53590", "38026", "51867", "37022", "63159", "55690", "21406", "65432", "362", "15464", "10323", "31612", "40955", "3385", "12585", "70121", "68435", "16865", "45207", "68060", "37255", "46097", "69958", "62984", "31557", "3970", "15586", "47608", "68064", "41223", "14032", "46356", "6876", "38727", "14837", "33403", "70411", "70317", "24132", "40685", "25217", "23071", "34010", "63393", "6962", "61756", "22306", "53197", "59563", "45878", "55415", "14136", "39302", "49145", "46191", "44510", "4963", "24264", "71362", "1971", "2138", "13331", "39634", "65503", "6790", "63349", "42804", "72053", "24671", "17066", "40402", "38141", "48644", "11546", "69276", "22151", "22160", "58342", "68685", "27054", "7947", "38145", "67203", "28014", "15426", "8541", "6390", "57932", "14263", "65479", "43550", "7643", "52201", "18853", "19137", "30517", "69579", "19233", "45798", "14792", "42189", "25771", "48134", "9322", "15266", "16695", "67794", "63079", "32763", "44585", "57881", "42590", "26300", "4774", "9294", "43400", "1277", "42183", "42892", "2749", "40995", "42179", "20226", "2619", "54311", "6118", "3435", "2625", "53540", "20701", "34727", "22475", "59995", "6833", "33118", "60652", "47162", "17108", "61644", "72357", "67665", "7472", "72390", "35712", "43578", "16976", "72566", "47834", "67167", "35555", "13809", "19434", "7257", "9124", "46112", "24177", "53824", "54470", "45092", "29459", "68326", "41111", "54506", "51320", "34711", "63057", "49114", "63553", "62146", "42592", "48512", "34975", "62056", "46900", "66696", "53381", "26123", "31101", "29924", "2671", "35688", "52347", "51235", "29968", "50437", "71492", "27617", "23261", "68370", "41457", "14182", "18700", "20289", "64385", "48499", "54202", "16366", "5785", "24681", "47476", "70524", "16241", "10023", "49478", "64332", "41989", "32385", "55636", "3199", "16879", "19537", "56859", "32400", "46624", "46964", "38399", "73212", "37029", "45216", "39260", "25910", "13765", "258", "70684", "60078", "60023", "18914", "6654", "67377", "26576", "48712", "10433", "49400", "66617", "43562", "37598", "56296", "23466", "65868", "65164", "34151", "4067", "64823", "422", "58872", "54462", "20658", "6634", "31779", "68665", "24915", "9594", "17113", "52123", "71702", "59809", "5993", "38502", "22520", "49507", "44697", "15435", "56626", "69997", "24722", "40949", "37822", "51785", "55299", "1826", "20440", "11194", "46198", "53818", "33302", "47250", "45706", "6670", "68020", "50008", "10684", "11994", "46674", "60138", "47485", "46970", "18275", "30668", "867", "43424", "6156", "40475", "13231", "32360", "9260", "23096", "48601", "19947", "58450", "34036", "12191", "42251", "46875", "32186", "45506", "4220", "17629", "38654", "18142", "54271", "64411", "37138", "28587", "36883", "69185", "35947", "4955", "40896", "41460", "62341", "58847", "50045", "41372", "26420", "72749", "45480", "46812", "60355", "32994", "46727", "15229", "47986", "24422", "32845", "20791", "21261", "63184", "36778", "12020", "64093", "43299", "40569", "16099", "41084", "71511", "10034", "65871", "27926", "31197", "12738", "24423", "780", "65919", "36933", "36035", "11603", "4628", "36339", "41187", "15215", "48184", "22956", "856", "46689", "29948", "43509", "26426", "14576", "33784", "18664", "65170", "65887", "9212", "32338", "40906", "828", "49466", "49750", "59729", "58553", "49567", "71155", "31627", "9811", "60842", "68863", "48577", "71232", "56289", "65442", "25363", "67906", "63610", "41037", "72764", "58550", "12701", "6194", "27265", "15944", "8183", "58685", "16196", "10858", "11144", "60760", "65553", "743", "28410", "42402", "55403", "21750", "26632", "34047", "53640", "9135", "43246", "55311", "34456", "51459", "21409", "38906", "18902", "25391", "38170", "50013", "8403", "38509", "14221", "37369", "52402", "51102", "30697", "33116", "57014", "20084", "67053", "45607", "41575", "38301", "57492", "10598", "31438", "70120", "48932", "13435", "51149", "25990", "8934", "59882", "41779", "60889", "38775", "64197", "26102", "71528", "42081", "68798", "25914", "1782", "49246", "352", "54009", "41226", "49070", "24972", "13037", "56727", "87", "3426", "40207", "5793", "2031", "19395", "66715", "23710", "30117", "30094", "72673", "18478", "3657", "64287", "17571", "31208", "58329", "49619", "48935", "14220", "19612", "57597", "22590", "38329", "15978", "72644", "20467", "58284", "67456", "66630", "41994", "6766", "43791", "2501", "9752", "66526", "29640", "3009", "63863", "10050", "35737", "65215", "70364", "44569", "42613", "28514", "67040", "65623", "69115", "16891", "32936", "61432", "24041", "39487", "9062", "19962", "58028", "60236", "67140", "46949", "1413", "5125", "27952", "64875", "25665", "54829", "44051", "13326", "42292", "40186", "27118", "39142", "67540", "20954", "65607", "60296", "24713", "31419", "27021", "29497", "64238", "51249", "11997", "63238", "65047", "20164", "58307", "41668", "72783", "11594", "37284", "31440", "8329", "48238", "31667", "11968", "40576", "50341", "64938", "47549", "27499", "27947", "38277", "34947", "51082", "27225", "47709", "28597", "1908", "21885", "26468", "12759", "60753", "10382", "27199", "5011", "20901", "23939", "43630", "56603", "60870", "33696", "16945", "27971", "12142", "36606", "7097", "20209", "67952", "66205", "12795", "30741", "19422", "59819", "33034", "63073", "15860", "61131", "14169", "39076", "27542", "11567", "31279", "69708", "24674", "11188", "5357", "13354", "68703", "46152", "2236", "18452", "10547", "29123", "53414", "20136", "1345", "32746", "70679", "44439", "71681", "49072", "131", "187", "51254", "3546", "71325", "36916", "32610", "7889", "27659", "65500", "5281", "9237", "56437", "23813", "8443", "52396", "54584", "34810", "5500", "33584", "32741", "67595", "37644", "70179", "30149", "38287", "24186", "39741", "71867", "63673", "15979", "64323", "9671", "66634", "977", "31603", "19255", "51497", "32262", "69975", "43107", "13256", "12467", "63147", "55341", "68318", "51273", "2622", "20668", "24940", "35113", "63982", "24531", "54832", "45295", "32055", "50138", "32852", "35579", "1592", "14284", "42477", "9939", "21338", "16645", "27865", "62160", "32986", "52903", "56085", "56453", "27794", "20438", "63280", "71256", "36066", "16757", "10556", "7880", "69993", "58086", "50634", "55175", "6783", "58325", "60869", "57056", "19057", "16817", "65297", "5722", "63170", "43349", "73098", "12299", "50100", "42730", "7217", "19595", "10754", "37694", "62159", "29777", "49623", "30003", "37957", "7011", "16961", "9576", "47244", "13500", "36443", "38829", "47022", "4605", "28399", "20714", "25130", "66319", "57898", "4427", "33009", "62087", "52404", "7228", "6899", "3423", "24825", "42160", "389", "54920", "53809", "58929", "55358", "8302", "32665", "6726", "28430", "27017", "8697", "47324", "68855", "68640", "46638", "24376", "27460", "11867", "24607", "2269", "68418", "32104", "65989", "29725", "11492", "46816", "23776", "44545", "47935", "58510", "27445", "69710", "64993", "12413", "54234", "25737", "8501", "31846", "63433", "53713", "1654", "10837", "50505", "4290", "22272", "54877", "23160", "44881", "59187", "63748", "58263", "30907", "59368", "44743", "23768", "61210", "25537", "40936", "73115", "30217", "48579", "38675", "44638", "30663", "57862", "63275", "9660", "65533", "36690", "18685", "32882", "62776", "69324", "59843", "68457", "47279", "43249", "30763", "25046", "58145", "57573", "35391", "1770", "53623", "13123", "42318", "11028", "61809", "43588", "31585", "51744", "56873", "62401", "42799", "23420", "2769", "7098", "52526", "15024", "69125", "23592", "52516", "62711", "67763", "11560", "11370", "2465", "8679", "24964", "53427", "34667", "52453", "15695", "58103", "39225", "51492", "13987", "58349", "68711", "8328", "26940", "26497", "51167", "11766", "70595", "10087", "23436", "70720", "35897", "49027", "9914", "9565", "53349", "52306", "7684", "61223", "31615", "52241", "66283", "47307", "26841", "32102", "23417", "66239", "67295", "38450", "42469", "29972", "66896", "10689", "13524", "65180", "54873", "17652", "66249", "5364", "32425", "20736", "55428", "71503", "18521", "4493", "66134", "27738", "10652", "71660", "29637", "54907", "31357", "18841", "38557", "64294", "22982", "44541", "31126", "61777", "33723", "1938", "72935", "61695", "47920", "43417", "66567", "56578", "66524", "53536", "67316", "64570", "29176", "61544", "5168", "23707", "60207", "58245", "12937", "47358", "36970", "4903", "38238", "65403", "44456", "2310", "275", "42948", "54", "25373", "46586", "64175", "62506", "15195", "39761", "6821", "57843", "8859", "4564", "64050", "70749", "17873", "56063", "42975", "50643", "49250", "41066", "65877", "22325", "42419", "26566", "62580", "53283", "4318", "25813", "31750", "28741", "28268", "58075", "693", "5081", "1330", "3104", "3028", "3407", "3804", "41443", "26425", "65698", "9614", "40897", "37238", "609", "60314", "56905", "14277", "12449", "30297", "17224", "40856", "37239", "62886", "4422", "61720", "61509", "40156", "61560", "57378", "42103", "41480", "22025", "14665", "25772", "53920", "51511", "38563", "15028", "68495", "3072", "25294", "71052", "9882", "19486", "29582", "21817", "24821", "51264", "20344", "44601", "12977", "27949", "71195", "28107", "68378", "44209", "50299", "45661", "11228", "50103", "42017", "41536", "10636", "68653", "69395", "43096", "43700", "33857", "72314", "35170", "28518", "35900", "70278", "36654", "46660", "24744", "1439", "907", "63115", "39536", "805", "61920", "47128", "3813", "37343", "49224", "48119", "60523", "5410", "33597", "50969", "23164", "4714", "65970", "62961", "23123", "33880", "58495", "50779", "25503", "69274", "67825", "63658", "71439", "63847", "29298", "62133", "64706", "17518", "56985", "29522", "6224", "2783", "61914", "54926", "4657", "34869", "18492", "44785", "44461", "12106", "9497", "18566", "56115", "63032", "39791", "6463", "19205", "47924", "8072", "39843", "67732", "16373", "22433", "69218", "11657", "2826", "32333", "30146", "45781", "4527", "22464", "24007", "64939", "67869", "22219", "63369", "24289", "35260", "37349", "70938", "36685", "9606", "43109", "31716", "54757", "22139", "21809", "18728", "7719", "72350", "5139", "67956", "69338", "2527", "17350", "30050", "55866", "69719", "63050", "13101", "6699", "67719", "32987", "24107", "37089", "40785", "27577", "48341", "39589", "30499", "24532", "49424", "39248", "12772", "28176", "19557", "13943", "5170", "9689", "11822", "71630", "45024", "66719", "1985", "31778", "12292", "54508", "68109", "26333", "51503", "65416", "47990", "60981", "8488", "2016", "30267", "1687", "20149", "43379", "33114", "57011", "62492", "26908", "3245", "5677", "29852", "11663", "49543", "46004", "25981", "7150", "5116", "62279", "23618", "33666", "51215", "51103", "64293", "14296", "12763", "34712", "67048", "24129", "68258", "45568", "51316", "22769", "55493", "56073", "31310", "25892", "16619", "55709", "51019", "23176", "46102", "26221", "9718", "15176", "25006", "72026", "33314", "12656", "33901", "34533", "51722", "380", "3816", "67408", "8180", "43724", "50715", "72829", "25420", "12230", "37944", "62582", "3716", "39715", "40630", "21741", "30927", "17323", "22410", "52407", "35861", "43876", "17559", "15379", "17217", "46870", "53256", "48621", "68759", "2519", "47186", "61545", "46737", "39585", "59242", "42409", "14466", "29834", "32644", "33894", "9513", "4026", "66364", "39560", "45898", "67309", "55065", "39833", "64247", "24849", "34256", "46685", "14030", "11915", "65380", "1817", "40672", "51343", "41221", "5862", "53688", "22611", "18839", "58909", "54324", "63536", "29088", "23536", "50089", "38527", "11167", "31206", "19025", "41933", "36817", "67093", "42565", "48971", "13375", "60789", "6618", "26698", "63356", "64195", "13355", "31049", "15130", "61463", "11888", "63671", "50886", "61891", "30353", "48043", "39906", "71690", "24873", "9391", "17670", "27103", "43313", "21901", "63649", "49825", "12668", "32892", "945", "56944", "65283", "6994", "28053", "2738", "59039", "31851", "52436", "37102", "36081", "9542", "16167", "4280", "38735", "64699", "31886", "22926", "27879", "64276", "64476", "26406", "68744", "48086", "5344", "50489", "67722", "2758", "46788", "30550", "54211", "26099", "52968", "3023", "64940", "33493", "52582", "11639", "60729", "6389", "14331", "68151", "19755", "67127", "48656", "45523", "20052", "48713", "67912", "69653", "701", "65977", "60667", "230", "29645", "17832", "17742", "25635", "15841", "35022", "43068", "56389", "10040", "9649", "11810", "61945", "18577", "12770", "18533", "52969", "65528", "29719", "39293", "52268", "62572", "5512", "50808", "12056", "52676", "52081", "16666", "29173", "9989", "9870", "249", "4910", "28305", "25322", "28151", "60145", "10722", "27994", "61361", "7514", "31934", "58054", "21931", "22701", "16140", "61344", "30931", "22751", "64392", "42467", "60095", "58296", "27867", "40291", "32863", "23919", "64757", "63460", "34016", "21940", "57580", "24901", "15479", "9679", "41420", "4101", "43017", "67551", "30257", "30818", "27338", "49921", "12512", "71634", "64634", "57380", "68907", "55696", "4985", "59752", "54315", "25573", "62650", "37025", "59611", "72893", "35622", "3764", "3376", "37162", "67769", "5091", "45103", "71287", "46163", "29377", "52220", "61722", "4513", "15678", "19526", "29025", "19851", "23485", "541", "6315", "15083", "69504", "160", "45123", "15666", "15574", "15299", "55039", "22574", "46227", "71636", "22269", "65221", "28880", "36426", "17905", "67773", "45692", "24899", "46226", "72942", "15344", "8790", "47702", "46504", "24728", "21320", "47962", "31546", "987", "68328", "54151", "64558", "46001", "41599", "3400", "50561", "46995", "67400", "29771", "30080", "43215", "70877", "31719", "40557", "935", "57691", "36106", "66012", "17791", "2431", "20802", "69149", "5534", "57777", "35394", "58520", "44791", "54737", "23329", "48059", "27073", "1081", "60649", "50379", "35912", "10137", "24564", "11779", "8404", "34549", "20086", "67105", "21437", "47994", "42003", "12347", "48007", "1922", "67357", "33070", "51078", "3651", "32696", "30662", "71756", "31595", "68373", "46705", "9740", "13079", "73118", "70134", "11341", "22735", "62616", "26810", "11303", "1659", "7998", "32853", "19577", "70123", "6355", "48163", "49085", "35301", "66648", "58291", "36322", "62385", "62944", "46364", "70580", "62846", "10448", "41378", "6826", "6701", "5435", "15225", "68465", "68059", "26924", "5526", "10940", "54117", "59958", "10065", "33940", "53705", "10571", "36490", "34693", "2775", "41890", "11290", "67786", "12500", "70842", "68019", "43609", "71858", "20983", "26884", "24428", "16005", "71757", "44370", "46715", "42962", "22951", "52003", "33088", "46179", "43499", "47260", "38189", "59548", "6582", "67237", "16231", "50149", "56507", "44776", "65376", "12882", "37576", "3286", "21316", "61758", "45110", "54945", "43657", "21856", "60320", "53730", "37224", "72998", "61094", "40868", "65762", "40171", "47175", "24176", "71270", "65855", "40737", "62476", "49439", "53297", "63705", "43120", "2728", "31912", "23388", "50201", "32061", "38482", "57576", "3260", "55037", "33136", "25980", "46992", "16109", "51444", "22477", "25669", "21789", "61996", "25361", "72887", "69090", "59577", "69510", "62262", "58652", "384", "9508", "71683", "43797", "65547", "57672", "8587", "9597", "5582", "10565", "21339", "67858", "31698", "58435", "3328", "33131", "60501", "49942", "63835", "14968", "35666", "69917", "17316", "72011", "24028", "58289", "4472", "9930", "29662", "30665", "39305", "68080", "62795", "28134", "41986", "17959", "8054", "5575", "47743", "53957", "16387", "25378", "28468", "64679", "34651", "54802", "14216", "37465", "9429", "51219", "24788", "58557", "23021", "36732", "52989", "42867", "11443", "33824", "42733", "56481", "57623", "41908", "43989", "50029", "51214", "16954", "60192", "55724", "6847", "67353", "6257", "16845", "17839", "16753", "2399", "44792", "65942", "67625", "61763", "3476", "50097", "70409", "62533", "2911", "31980", "26991", "8001", "65019", "59099", "56244", "27850", "32235", "16489", "14575", "34318", "60221", "32190", "22066", "69386", "72036", "72080", "9963", "38834", "18333", "32951", "3085", "11428", "21010", "70790", "9340", "61901", "63119", "62289", "34074", "21075", "21734", "50828", "17315", "12217", "10768", "7484", "54937", "67893", "65680", "44171", "48402", "26770", "61347", "62067", "56068", "32403", "2385", "2069", "29304", "61333", "37336", "5151", "34796", "42795", "22841", "10059", "57562", "55138", "69476", "12216", "31387", "19906", "67381", "11622", "13522", "62641", "73142", "69295", "49904", "39841", "24096", "4137", "38951", "41808", "24416", "35955", "70098", "38939", "5804", "52167", "17197", "67262", "19321", "18880", "28302", "9574", "58756", "60995", "26623", "24843", "9005", "28549", "25982", "34114", "22736", "63071", "53669", "13631", "1772", "32944", "28618", "40712", "17119", "62477", "9702", "33013", "27558", "18039", "28985", "2126", "11809", "67497", "14714", "28110", "1555", "26954", "64896", "59166", "12781", "28865", "17842", "18519", "45667", "55384", "56482", "56101", "39935", "49852", "23319", "62555", "62531", "6522", "3654", "12678", "43097", "55681", "65358", "50152", "5760", "68748", "69632", "49293", "56805", "46416", "53755", "72906", "48758", "36094", "21745", "20552", "54382", "57919", "16244", "42503", "31830", "42177", "19666", "21937", "9504", "45303", "43206", "30671", "34380", "858", "45837", "41383", "32906", "28465", "11359", "6305", "46843", "57488", "14186", "38555", "38402", "7728", "50032", "2024", "61508", "43683", "58326", "63416", "1425", "61648", "8077", "62728", "47467", "18073", "66529", "67063", "61378", "34227", "60279", "26220", "23810", "35995", "27244", "50900", "34009", "14585", "59380", "62904", "16000", "22045", "66656", "60829", "23531", "58248", "61970", "51008", "42007", "18244", "4960", "40607", "59605", "20220", "2189", "40874", "73238", "26532", "18032", "47797", "13728", "42205", "48389", "56942", "54162", "5028", "72653", "69688", "32069", "60477", "70796", "66297", "65566", "57980", "70872", "5021", "6261", "37151", "56364", "28696", "7906", "26564", "8766", "65182", "10782", "17015", "60378", "6", "68885", "21324", "14491", "11856", "2873", "63043", "64064", "71366", "45580", "49074", "38326", "41851", "27122", "7647", "59742", "16168", "72908", "68730", "65550", "43888", "60450", "53685", "48261", "2593", "13538", "25198", "70963", "46369", "63771", "7776", "62635", "5750", "28195", "72824", "62379", "29213", "23452", "66241", "22546", "10675", "9762", "52836", "64889", "33066", "19742", "40695", "56554", "37649", "64688", "20019", "20916", "30719", "16363", "9795", "3535", "8720", "47832", "19492", "56732", "3734", "32803", "5647", "53712", "60656", "28958", "42244", "9386", "61818", "71610", "34898", "54121", "40604", "35219", "49494", "34454", "39747", "49305", "36548", "59309", "47764", "43123", "23031", "49926", "7429", "40322", "39726", "71558", "49360", "26222", "47460", "56543", "54722", "14425", "45157", "70571", "4445", "47276", "10301", "36652", "70444", "12211", "9375", "54883", "38051", "1496", "33263", "68772", "16406", "15714", "63224", "49651", "137", "17868", "44877", "23970", "5639", "11110", "52487", "22696", "25896", "64859", "17598", "72769", "19375", "69248", "53937", "56853", "20463", "43733", "51202", "4624", "55514", "21309", "34630", "60602", "568", "70719", "43519", "59853", "53095", "31978", "46601", "60615", "34982", "32903", "31819", "17867", "11568", "17805", "41379", "34874", "37668", "18743", "40768", "46505", "23685", "43932", "50893", "7176", "5707", "9431", "32274", "62085", "57732", "35658", "34755", "33385", "24814", "71999", "13558", "227", "5457", "21564", "69838", "14549", "47755", "70717", "52456", "33560", "4890", "17936", "64191", "50044", "64736", "18316", "64318", "71370", "7500", "22254", "38835", "19630", "4144", "24880", "21679", "62114", "53161", "19370", "66813", "13835", "14051", "31080", "31244", "20844", "31106", "12575", "16228", "41953", "70135", "55684", "47849", "28178", "18769", "67960", "15637", "38811", "37733", "50745", "17796", "11798", "44179", "4882", "36166", "27777", "61527", "51578", "42096", "29970", "58250", "48936", "46454", "427", "26526", "27282", "629", "28657", "30520", "33811", "58397", "41930", "28932", "7520", "43963", "30905", "63304", "62574", "11122", "43442", "62272", "55109", "49665", "42853", "50304", "37312", "8809", "15779", "25286", "59253", "5380", "69165", "66387", "50776", "55227", "32459", "17046", "35846", "29395", "13449", "5508", "15914", "14419", "67608", "69384", "53247", "25035", "56210", "58116", "58839", "36355", "68892", "41184", "8764", "47632", "14932", "52089", "46728", "20099", "42822", "16374", "63257", "35073", "35038", "49484", "43747", "20018", "72895", "8853", "52153", "32293", "11309", "40200", "14128", "17746", "47238", "38695", "68085", "47035", "2067", "38460", "35730", "33158", "16838", "21250", "59168", "15901", "9329", "33442", "59432", "32918", "15628", "60627", "19342", "49609", "35673", "53914", "14489", "8401", "22589", "50180", "51023", "13481", "55083", "41106", "10102", "71311", "8730", "35550", "51737", "111", "30397", "3027", "22036", "9514", "29471", "32541", "14317", "66441", "5967", "44716", "39254", "3846", "57852", "29191", "42110", "32615", "27342", "1569", "42043", "2883", "12828", "23605", "17419", "54064", "13205", "36318", "10532", "2584", "43335", "67538", "15594", "38202", "59969", "34888", "59334", "10763", "6121", "1050", "718", "32340", "66394", "4747", "16803", "38928", "8976", "865", "52747", "44216", "35562", "72946", "44696", "7395", "25952", "66020", "65405", "51980", "14142", "45306", "67279", "35300", "62450", "49841", "21505", "68982", "46973", "38519", "2207", "56548", "58671", "9390", "24540", "9118", "50112", "67682", "4209", "24510", "47265", "73030", "71862", "17660", "71670", "22737", "55874", "25062", "27963", "52221", "41415", "26265", "10975", "41339", "1414", "64509", "10212", "18936", "7755", "58892", "44234", "72100", "59760", "63653", "66638", "63569", "36487", "12311", "1646", "65086", "47125", "64016", "39645", "57052", "59673", "49936", "57808", "71556", "44215", "27071", "11074", "36461", "26772", "47689", "31428", "9303", "1298", "40317", "41896", "54655", "4838", "41050", "21828", "7314", "40786", "4045", "65986", "842", "63850", "55731", "21361", "27340", "11526", "20771", "1454", "13371", "65739", "33918", "43670", "8138", "18877", "56332", "19421", "6773", "14081", "64386", "70825", "53203", "31844", "8157", "59370", "32847", "38840", "55982", "12997", "33604", "66459", "71290", "11026", "21045", "39779", "66310", "56960", "13357", "51577", "21140", "39518", "17221", "67345", "70816", "70122", "57728", "56945", "5504", "34", "55048", "67562", "22765", "65084", "21898", "5429", "28631", "65679", "39719", "56335", "49", "52718", "51299", "6796", "29979", "35510", "66087", "7318", "4920", "33942", "56133", "35954", "27060", "38508", "62534", "2533", "41044", "7765", "15171", "13598", "28590", "32957", "65144", "61791", "35233", "66624", "51369", "67235", "69048", "70555", "66340", "38763", "2141", "59268", "35211", "32321", "62630", "5826", "17062", "66481", "35976", "60147", "38669", "52976", "28426", "42681", "1951", "23059", "36021", "34268", "36280", "14893", "8310", "33632", "3966", "20261", "13470", "33540", "9163", "58072", "36056", "72589", "14767", "21598", "29288", "66542", "29391", "26892", "15456", "7360", "68522", "22484", "34600", "11357", "45605", "59992", "25372", "66788", "58681", "5083", "31599", "61737", "51971", "67808", "17554", "53066", "56475", "32506", "53684", "21854", "25732", "21522", "51635", "65876", "45481", "30509", "43040", "29387", "13285", "10991", "2763", "62671", "13", "7035", "37785", "24515", "62931", "67207", "10395", "58548", "7541", "30057", "8966", "68946", "21401", "51569", "48541", "27241", "27191", "27707", "12277", "20441", "18352", "14239", "49132", "3241", "47108", "58239", "59064", "1956", "59692", "42057", "15373", "72317", "23451", "53638", "32222", "9151", "71754", "1714", "9182", "35001", "50550", "47440", "9432", "35534", "20004", "13284", "20920", "6503", "36793", "63287", "2719", "29561", "39816", "61546", "65912", "49130", "2319", "67767", "35203", "71574", "18549", "19178", "14059", "10914", "55823", "61013", "8864", "3881", "7300", "46101", "58207", "16653", "18051", "408", "50862", "12511", "58819", "8333", "29199", "64315", "45511", "58880", "36404", "47314", "16102", "46630", "72264", "23104", "48693", "55571", "47552", "60775", "31020", "48222", "26712", "31245", "7073", "51394", "7309", "68570", "12921", "61355", "51325", "8192", "50480", "45591", "67231", "19957", "7159", "4951", "68684", "7096", "58945", "72014", "43008", "8121", "18494", "44725", "29833", "4539", "7350", "72704", "873", "33027", "34713", "6532", "30436", "52725", "42765", "46134", "52785", "24454", "55089", "7853", "19179", "58410", "2778", "56753", "30954", "30772", "13911", "67920", "25550", "1630", "71396", "1255", "31309", "59403", "51391", "49624", "30814", "34678", "28212", "6903", "40799", "64239", "46982", "47498", "71063", "11225", "16259", "67777", "65849", "55882", "38900", "1397", "62680", "2847", "22099", "30487", "52852", "36083", "57248", "26588", "29706", "40380", "25882", "70623", "48959", "48464", "41630", "17736", "40194", "18018", "40289", "6136", "60259", "14952", "11620", "64142", "1812", "59695", "51674", "18105", "26472", "5894", "18744", "53143", "56178", "18021", "25156", "53846", "52637", "63414", "68403", "31099", "59768", "11405", "72438", "68228", "69246", "30628", "55281", "54410", "18137", "33782", "38680", "11044", "53666", "12718", "45839", "35162", "21841", "6973", "48194", "28509", "59997", "12987", "35944", "10716", "18574", "9854", "34754", "40802", "18206", "58757", "61803", "46246", "40110", "71700", "70692", "40316", "33117", "17512", "47413", "34379", "31931", "44375", "70224", "51318", "15372", "2322", "5316", "54083", "15167", "5799", "56750", "15066", "72286", "61167", "51220", "60601", "72487", "59396", "17351", "58387", "13368", "37374", "22707", "29003", "19510", "52667", "47776", "7154", "42064", "43099", "69482", "64952", "7677", "57535", "63117", "5619", "44091", "28158", "38923", "73246", "72787", "47395", "47083", "20909", "46465", "85", "34935", "3925", "27320", "8621", "59434", "61452", "16023", "18460", "67521", "4893", "46520", "43964", "53189", "51884", "5062", "56570", "22241", "68035", "19711", "30291", "7942", "16385", "7134", "42748", "65584", "25873", "43223", "29188", "20108", "46521", "49907", "21838", "62823", "23157", "10568", "58", "29912", "10409", "66911", "31083", "25254", "58441", "24617", "19122", "1655", "36907", "23735", "27935", "38052", "20852", "14965", "33291", "21653", "5681", "22549", "35482", "14116", "72490", "11358", "65217", "58211", "23922", "34638", "40071", "394", "29837", "71868", "35169", "7421", "38959", "71278", "15762", "29831", "11967", "31772", "72020", "41678", "3630", "5902", "5413", "35600", "4719", "35577", "23570", "51441", "7748", "37248", "48070", "29774", "2149", "73211", "8957", "36478", "71905", "71374", "16816", "6533", "11607", "69795", "36851", "32841", "70469", "61124", "880", "27190", "31147", "68313", "14362", "5079", "31070", "57118", "68175", "58286", "15471", "34229", "38916", "14077", "48416", "67591", "67246", "61162", "43524", "61657", "44784", "36868", "48446", "31600", "38408", "29559", "36846", "53881", "46070", "36756", "56259", "56679", "19075", "29116", "45586", "38760", "42531", "38818", "29697", "54530", "64210", "30304", "68808", "38765", "13607", "10553", "26783", "68302", "72165", "28071", "57169", "34199", "52133", "2817", "56143", "29429", "11874", "467", "61612", "17557", "72729", "5189", "51985", "69293", "57970", "3042", "64520", "54681", "19981", "8711", "10014", "4578", "27980", "55714", "60838", "47077", "32547", "61703", "24675", "71469", "41193", "665", "9117", "11646", "19940", "41061", "35834", "17565", "63354", "59193", "64748", "8194", "72920", "35471", "65101", "61541", "53324", "61736", "8918", "66841", "20592", "72975", "36084", "18718", "46009", "63871", "56463", "45598", "66505", "11878", "16509", "12453", "34900", "64068", "7049", "37404", "4070", "54720", "4700", "63162", "11662", "22932", "21511", "56938", "46247", "25393", "25919", "29091", "25597", "28864", "6165", "39546", "13353", "31082", "25686", "35464", "27894", "13288", "36714", "46941", "6658", "2854", "32166", "65197", "33099", "346", "50630", "39326", "29757", "13938", "7137", "66837", "3331", "133", "13778", "4185", "4824", "34889", "4432", "47287", "37518", "18252", "51495", "7016", "38860", "71228", "43063", "58488", "1907", "18030", "55188", "68184", "52980", "13506", "27871", "49749", "26323", "10320", "33868", "1807", "15417", "49328", "47922", "55761", "18676", "18400", "16391", "37750", "54225", "64492", "40870", "17058", "27390", "54007", "43962", "22551", "16862", "29389", "30601", "11170", "43391", "24702", "6134", "32067", "3547", "41261", "463", "4962", "59292", "39411", "28156", "5501", "58201", "33041", "8801", "7891", "72256", "13510", "22954", "44105", "24113", "1584", "61932", "15446", "70678", "45317", "20779", "11464", "6960", "16918", "51734", "7839", "61123", "24828", "61257", "39194", "65252", "68876", "6604", "69046", "58998", "68596", "25584", "19078", "51925", "49929", "22738", "49387", "10082", "52117", "16747", "19076", "23880", "15712", "12011", "67522", "18592", "14620", "53065", "70922", "28257", "43019", "61503", "43497", "73012", "12912", "61917", "33056", "35302", "63826", "40370", "61656", "29376", "10338", "35050", "68597", "37991", "43023", "64804", "16926", "32947", "47383", "15710", "19890", "6307", "71955", "1335", "59734", "3758", "9070", "64080", "35224", "47944", "55411", "57965", "29817", "25844", "7499", "58719", "54726", "1697", "5225", "20258", "63209", "47968", "70303", "8876", "20316", "68972", "12629", "41903", "44312", "10365", "55123", "4503", "13367", "20975", "14167", "22632", "66490", "10699", "58196", "25384", "55632", "5469", "42246", "27792", "60437", "34897", "20219", "71307", "3047", "68547", "63567", "53625", "7347", "1882", "10019", "38667", "66035", "39725", "54876", "25791", "54631", "47147", "42301", "54529", "35078", "9057", "43207", "44827", "48796", "45594", "72537", "47075", "41024", "43401", "25853", "23856", "57493", "54072", "65529", "45337", "56916", "73094", "43283", "41772", "62579", "70375", "32101", "10656", "27341", "39955", "59905", "61438", "49828", "11393", "54195", "149", "47714", "6270", "55201", "27985", "23720", "18788", "53949", "72698", "29888", "43694", "58525", "41333", "52964", "45142", "12784", "50727", "66900", "27575", "72889", "64194", "31190", "23535", "15831", "2524", "66046", "58428", "5090", "19013", "65691", "70517", "17436", "42837", "4268", "38280", "51513", "20191", "61686", "70241", "11438", "73201", "72617", "36403", "37573", "3231", "46551", "34621", "64236", "42245", "5915", "8283", "34696", "10458", "55701", "3228", "5338", "28202", "6340", "35710", "9577", "8952", "71413", "70248", "5759", "55374", "29776", "45241", "40888", "42688", "11629", "2693", "23068", "12692", "68230", "43321", "26936", "7692", "61865", "28506", "49612", "7414", "25545", "31136", "7984", "42312", "51997", "57127", "59206", "2372", "27671", "31322", "58354", "32510", "22395", "64327", "4516", "14652", "5075", "63778", "55937", "28766", "46310", "27144", "8715", "54436", "53113", "72684", "58089", "42210", "68517", "25785", "13903", "69943", "47850", "56952", "47853", "64780", "38186", "61604", "18259", "61487", "2967", "60814", "35865", "27260", "62378", "59842", "62605", "36639", "45846", "46434", "15127", "1027", "8961", "33251", "30518", "34448", "49657", "47739", "47733", "71549", "61974", "14556", "71190", "19806", "69999", "30163", "72970", "23222", "46303", "45487", "55587", "14373", "36415", "2315", "70793", "65654", "67680", "64937", "63232", "65137", "61120", "5428", "72151", "51742", "57830", "20134", "67160", "71112", "27110", "14137", "22493", "33144", "31039", "39651", "8088", "39777", "34582", "45486", "27357", "31340", "32485", "28182", "51805", "45342", "64726", "58120", "59965", "33184", "70050", "53458", "61730", "68811", "57883", "18855", "64189", "21157", "7952", "47846", "53214", "45315", "29779", "72392", "33230", "40353", "7068", "53804", "19128", "33725", "45652", "51179", "13232", "37765", "70062", "21765", "23034", "36529", "54278", "30998", "53766", "4501", "5953", "23412", "54320", "54782", "5278", "55720", "19842", "64193", "8237", "36898", "19123", "22777", "41270", "70305", "65534", "20747", "64788", "37762", "15503", "48941", "47880", "59561", "7411", "58040", "51160", "46988", "41645", "21490", "13907", "34514", "26660", "52875", "27634", "71030", "17496", "47044", "41635", "52614", "48151", "34817", "34642", "9185", "60880", "14989", "61497", "46107", "18642", "36674", "69092", "69935", "34866", "32381", "38139", "51628", "58482", "23629", "24517", "23595", "22509", "59998", "37899", "70645", "34722", "52432", "32815", "56871", "2336", "35167", "64150", "49509", "42331", "59271", "69523", "28532", "44186", "4984", "38657", "21137", "23063", "52584", "43501", "36834", "5391", "54561", "10737", "67395", "9245", "42037", "52669", "14998", "45102", "33771", "22230", "7567", "12599", "1429", "59745", "24582", "29102", "29957", "14621", "30796", "4177", "36867", "28001", "13167", "61846", "21319", "13908", "42894", "12426", "37036", "72615", "20503", "4227", "72917", "4966", "49633", "5831", "50812", "10711", "36969", "63916", "56106", "50155", "1386", "54842", "71734", "34022", "53750", "45262", "71954", "19040", "39973", "56888", "20652", "30787", "66351", "45190", "70501", "21083", "20145", "34510", "18431", "70481", "62731", "22369", "8608", "68196", "11092", "70060", "55185", "20251", "11551", "21700", "62257", "28576", "60630", "50860", "27162", "53951", "15366", "14973", "38415", "46539", "9209", "10706", "49903", "51152", "38248", "6258", "3695", "126", "1266", "150", "48220", "23252", "17963", "5164", "46278", "12103", "12530", "23418", "39473", "24350", "72460", "69557", "44243", "16107", "11601", "8158", "18290", "9979", "5121", "19147", "49034", "14366", "6631", "55734", "26786", "27973", "41107", "29433", "30252", "40277", "62956", "61624", "45406", "30482", "34668", "22773", "11770", "9661", "55794", "45848", "36524", "53864", "27319", "4226", "69332", "40747", "1062", "20459", "18164", "51984", "60535", "14183", "34421", "51131", "760", "6608", "28234", "65603", "29368", "70760", "53454", "28350", "72186", "42722", "11454", "9518", "65079", "57943", "9013", "45151", "63743", "4497", "28475", "6384", "9403", "61140", "2387", "64049", "30686", "16566", "64914", "45432", "59002", "33575", "35308", "44453", "31482", "23874", "51851", "24139", "70663", "8429", "29438", "38542", "68451", "71189", "45279", "19240", "47338", "18371", "6392", "70218", "48909", "60142", "38201", "43778", "6892", "49626", "37006", "48271", "27779", "20653", "10732", "8049", "17514", "30411", "31016", "46122", "24215", "71206", "4591", "63498", "17955", "65250", "9099", "26533", "18570", "28215", "5706", "58996", "32230", "1304", "41056", "9804", "47631", "53086", "35173", "25831", "35441", "57441", "8149", "6541", "70764", "53558", "50799", "57679", "4346", "36936", "39598", "44259", "70733", "52066", "57021", "9318", "56162", "68406", "13148", "60217", "28659", "11960", "31041", "69006", "5087", "43195", "44289", "42997", "21506", "22864", "15129", "21404", "12802", "50620", "5112", "23253", "45859", "48801", "281", "5274", "45145", "19141", "49524", "5362", "52454", "33809", "36400", "69944", "36485", "52632", "24175", "17233", "70913", "54190", "51155", "3887", "48146", "27057", "33523", "58990", "68579", "32318", "30302", "15291", "50616", "71568", "45007", "64245", "64921", "30623", "73227", "56628", "59034", "25639", "62151", "1582", "18169", "18418", "69950", "44658", "68504", "14344", "27679", "61067", "41546", "18926", "38902", "43117", "25925", "39609", "41900", "25341", "43190", "34740", "24286", "71470", "9880", "32683", "61352", "30248", "51285", "3470", "14096", "26937", "42197", "41036", "26286", "43054", "40537", "54255", "47578", "69226", "72776", "40143", "50397", "27068", "40534", "12593", "26128", "41593", "35079", "55759", "8337", "22028", "25104", "44382", "61747", "54405", "6138", "36095", "17539", "64855", "50819", "34130", "29403", "1498", "70455", "13554", "8900", "5248", "22700", "57088", "4161", "52449", "70753", "47660", "2596", "7403", "25216", "7121", "65171", "35343", "32140", "49696", "5289", "70172", "39280", "18439", "30133", "14553", "41697", "35312", "11887", "27298", "40981", "45461", "73008", "32300", "35189", "36534", "34133", "28995", "67276", "59521", "54375", "64836", "57838", "46283", "35180", "68255", "15023", "13279", "25031", "13725", "5394", "763", "54023", "47925", "1059", "51277", "22783", "35337", "57794", "62745", "19196", "10969", "55939", "45888", "53723", "71345", "31339", "42501", "36782", "12166", "42667", "14047", "51104", "59515", "3176", "25387", "27521", "23729", "50495", "41445", "39592", "5709", "18483", "36361", "39623", "21950", "1009", "49898", "45764", "12501", "44367", "22358", "28961", "51132", "69331", "53177", "38836", "58845", "19401", "67188", "71400", "72846", "24929", "6220", "41428", "52419", "12615", "21412", "4124", "18194", "61481", "20448", "35153", "72859", "37289", "14662", "9952", "60766", "56368", "30274", "70146", "25781", "12879", "37199", "45633", "11571", "44679", "26178", "10633", "22133", "34575", "11236", "65493", "19922", "22552", "1142", "30070", "47188", "27900", "62351", "11953", "57782", "4488", "43361", "49221", "9997", "61943", "52794", "9650", "26371", "61195", "23030", "40749", "68708", "70410", "22719", "12382", "13096", "22256", "22750", "21029", "19556", "42491", "10928", "42027", "62370", "26648", "7015", "14701", "71560", "29442", "65574", "21224", "46041", "25307", "12206", "32750", "2678", "19414", "960", "48622", "59792", "64428", "57913", "6364", "71737", "71945", "15736", "58209", "69663", "63299", "65127", "17798", "54746", "7680", "27188", "26641", "5327", "54523", "7486", "54597", "61675", "8031", "22875", "22924", "70837", "33405", "36736", "59000", "56005", "50366", "69650", "55044", "70367", "34633", "49266", "35678", "1870", "26827", "53216", "1821", "62165", "48254", "63957", "70921", "55606", "55181", "64669", "65859", "25293", "71384", "5131", "21867", "35593", "11547", "36519", "36149", "45128", "47928", "65163", "37492", "63288", "22732", "61529", "54390", "67995", "40671", "64642", "57983", "31478", "9884", "1531", "59908", "40781", "63796", "3882", "493", "52296", "56074", "48236", "24151", "63156", "51150", "63217", "17491", "31607", "6293", "8267", "46336", "34747", "37144", "53575", "71940", "16423", "22896", "66533", "64192", "57743", "4619", "30081", "68860", "71122", "32183", "43336", "48880", "66603", "55156", "71391", "43644", "51463", "23902", "36366", "69973", "9617", "21839", "50600", "4795", "31580", "8197", "68681", "38269", "35284", "32008", "53455", "41893", "25315", "37712", "31199", "29840", "55261", "49029", "47916", "7060", "67455", "66471", "65114", "26262", "8512", "26224", "4257", "18094", "39356", "7297", "73000", "35008", "30973", "53137", "63958", "13183", "68704", "36057", "66158", "40412", "48594", "26388", "13308", "71021", "21028", "9959", "15077", "42441", "24386", "24019", "41088", "6207", "433", "70809", "72818", "57596", "55335", "68501", "18827", "48651", "62273", "59175", "69673", "12422", "11204", "50083", "69689", "44146", "24981", "36672", "9667", "36312", "46331", "35276", "67051", "27852", "7246", "50751", "72767", "31834", "18741", "851", "68686", "59113", "50513", "71555", "69433", "60996", "29329", "33211", "51697", "31201", "2201", "5539", "47783", "43871", "44652", "8695", "27062", "8729", "55676", "64004", "40838", "14626", "33607", "42941", "5906", "2816", "67457", "33692", "52399", "39852", "47066", "12578", "47341", "1703", "1094", "57647", "23821", "65651", "48999", "71762", "1301", "40153", "17152", "62578", "59587", "54914", "62995", "34212", "46359", "42387", "39799", "63729", "45369", "27798", "31285", "59462", "32285", "36281", "1028", "19902", "1932", "6051", "44542", "41140", "30698", "18507", "63142", "52409", "44815", "18254", "6189", "5085", "7712", "23855", "39212", "68518", "19426", "54476", "14464", "73029", "71069", "72362", "6530", "44248", "601", "58733", "8509", "67904", "50841", "3588", "31151", "44682", "14990", "22899", "39972", "48857", "66138", "19524", "35152", "56235", "52625", "59552", "46291", "58038", "22145", "31864", "3753", "71675", "44803", "10517", "30615", "28747", "52323", "65896", "37529", "44893", "50095", "25507", "10335", "52749", "8780", "11905", "19329", "15596", "34880", "58084", "12063", "68191", "29054", "33645", "68928", "48955", "45018", "50354", "63121", "50518", "2743", "66626", "23585", "70276", "68930", "60796", "11043", "16152", "50592", "56860", "47825", "39289", "19665", "60746", "3422", "15026", "72012", "32337", "61905", "35120", "13856", "41504", "6621", "27339", "51646", "38790", "8691", "33574", "19847", "1310", "4713", "23455", "18485", "11731", "52059", "64813", "23424", "41222", "7158", "54159", "17038", "3090", "3965", "36871", "72463", "67428", "5161", "46171", "4377", "64591", "50217", "46021", "70681", "32443", "21182", "30702", "6819", "59276", "51504", "52186", "58832", "33658", "24226", "37423", "12203", "23732", "25383", "60879", "45740", "29596", "58424", "63292", "57482", "13460", "57461", "3294", "38742", "6115", "71593", "19334", "31811", "30023", "3814", "7888", "52126", "58987", "3402", "29864", "36803", "26020", "70846", "13248", "20862", "4549", "11112", "55113", "1688", "62900", "63212", "18890", "25396", "66028", "26982", "7995", "46754", "15838", "48368", "63404", "14727", "43429", "26848", "27525", "14498", "66962", "46832", "2445", "3935", "26930", "35406", "37587", "56438", "54011", "18828", "39412", "68857", "37755", "42605", "47219", "63134", "12151", "33955", "28099", "38300", "21454", "59410", "40372", "33390", "62386", "53836", "23534", "51498", "37553", "51313", "36313", "66645", "30011", "61906", "3101", "41311", "5134", "2216", "46481", "56812", "67943", "4746", "35779", "33482", "63882", "61561", "62157", "25876", "60693", "26157", "60688", "64009", "63269", "38626", "69335", "49314", "6547", "8417", "17761", "54952", "64022", "4635", "52722", "20590", "64061", "8469", "72613", "18659", "11640", "38730", "12076", "61468", "47591", "1470", "357", "33105", "45090", "19011", "48189", "24719", "43478", "27443", "41506", "53094", "33543", "31601", "34419", "3524", "10362", "62303", "5110", "27914", "29663", "54197", "29672", "19527", "54002", "2257", "60499", "67829", "42856", "47486", "32123", "23656", "908", "60146", "57195", "18945", "40827", "25066", "37314", "51817", "51059", "53425", "57106", "14737", "9673", "1566", "66339", "32688", "22585", "2740", "66450", "70026", "30031", "62884", "41331", "56311", "43341", "28573", "25676", "2377", "6026", "20185", "23154", "53298", "49004", "49036", "56042", "5957", "67032", "53694", "69731", "55038", "26047", "27869", "59909", "44410", "45069", "30446", "6849", "22961", "70741", "30829", "30789", "14557", "12003", "63098", "45708", "56271", "51454", "58358", "45427", "57136", "27988", "44362", "55231", "43035", "11634", "12760", "32509", "48810", "34506", "34066", "47036", "33568", "69782", "35870", "5493", "54554", "4335", "10496", "19894", "46999", "13861", "6763", "46431", "9283", "12623", "22850", "59557", "20208", "41200", "65108", "71975", "59821", "51399", "41391", "31483", "48176", "5389", "59973", "63898", "52120", "2392", "25978", "381", "39383", "44908", "19190", "58513", "35980", "58739", "63786", "42071", "39834", "29238", "65742", "8786", "69151", "5203", "66192", "40488", "6893", "71895", "16899", "23505", "36900", "66771", "41902", "33814", "65888", "60974", "24856", "20680", "25171", "43304", "30667", "51840", "12594", "6679", "30916", "68904", "43960", "42260", "52641", "57696", "55239", "7742", "5449", "47257", "57274", "23942", "61553", "72974", "44085", "50677", "23546", "63229", "64082", "66475", "63991", "70423", "23499", "6298", "17476", "32194", "4330", "39711", "23971", "28633", "64151", "69315", "26539", "72724", "37217", "32527", "71992", "28508", "3051", "14945", "46060", "28376", "38306", "63524", "3464", "35732", "18953", "48483", "9535", "19345", "58400", "68425", "26225", "65058", "38858", "33757", "16820", "33289", "28776", "35566", "26404", "47157", "54834", "12334", "41136", "50759", "17265", "10640", "22662", "35588", "71885", "70612", "50219", "67816", "23112", "50454", "5660", "36299", "60809", "617", "8605", "22419", "45504", "31913", "46211", "57515", "4733", "47434", "27269", "29941", "69885", "71", "2958", "19", "40532", "31998", "58734", "21114", "33719", "35125", "17218", "4107", "15076", "1756", "54158", "60793", "28029", "39478", "4940", "1348", "72094", "23122", "1253", "2081", "9505", "25020", "73207", "58246", "40871", "67216", "55232", "9566", "15938", "69233", "17567", "22408", "5971", "63282", "26869", "15194", "17843", "41481", "55045", "26066", "56778", "3394", "53768", "1828", "28167", "30060", "59569", "12751", "59198", "27948", "5446", "56119", "6327", "52701", "62345", "54461", "10373", "57681", "66151", "66063", "26285", "57558", "41057", "67985", "8799", "30258", "44956", "65834", "42135", "56839", "6234", "35291", "23368", "35451", "54351", "20887", "11658", "12307", "12273", "19232", "47254", "17774", "6053", "5988", "8975", "43796", "53235", "61764", "56655", "11352", "40649", "3364", "70833", "5300", "70703", "51965", "39158", "53165", "1501", "54051", "49459", "30092", "62885", "64120", "12722", "10654", "10148", "2071", "25087", "44917", "8048", "31617", "35490", "12891", "20838", "43141", "33215", "60617", "59816", "3572", "42184", "52784", "62993", "13733", "8391", "26507", "25973", "58105", "17917", "3270", "4271", "26763", "45036", "50931", "33208", "26809", "10663", "5058", "31936", "35271", "66907", "64810", "66998", "27846", "16826", "35422", "1553", "12621", "16033", "57609", "65025", "17785", "3171", "3202", "21245", "38512", "51687", "70956", "40360", "46782", "5841", "36049", "20256", "70405", "71912", "6730", "18044", "40642", "41739", "6222", "21281", "61767", "15205", "19972", "14979", "63675", "55935", "40504", "8468", "10442", "66379", "51000", "28550", "61128", "52493", "36768", "49685", "26967", "70996", "6353", "64705", "17285", "57945", "43454", "7169", "64861", "45133", "72002", "8038", "3872", "53419", "39351", "61990", "20640", "44087", "27569", "20543", "62564", "16497", "64219", "49487", "50995", "14740", "42001", "57025", "62175", "37115", "31004", "66148", "27182", "64838", "45727", "47676", "27810", "46525", "25176", "31881", "70525", "5089", "31737", "62909", "51726", "71627", "33829", "44945", "72359", "72948", "40727", "27573", "13080", "38432", "59541", "48339", "69499", "44290", "15485", "47103", "64105", "3943", "61951", "29476", "5774", "39461", "55605", "70093", "34092", "39485", "54820", "27111", "9756", "1940", "5573", "22367", "49395", "52332", "34182", "37447", "1203", "40006", "13573", "47124", "69221", "65726", "31318", "72640", "65012", "45268", "12969", "45493", "13307", "22121", "49138", "28345", "20678", "4744", "24838", "58689", "17163", "26422", "35371", "70451", "5908", "10882", "50583", "47410", "17626", "6031", "49972", "43471", "65070", "11284", "32679", "63691", "54545", "32963", "28219", "38641", "51567", "55875", "68937", "26230", "11517", "62788", "48202", "61012", "58332", "57801", "43245", "11095", "15877", "35255", "35731", "18537", "13557", "32629", "33578", "19792", "62222", "15832", "164", "47411", "66097", "70124", "16494", "55627", "51331", "3593", "66128", "55019", "44727", "10036", "7289", "59001", "1549", "35971", "7262", "57675", "41992", "3451", "50005", "2692", "61280", "43070", "55182", "33825", "25968", "5484", "12888", "12761", "60339", "25546", "11908", "35907", "36174", "49435", "6470", "39426", "55406", "7037", "51457", "3377", "68716", "37359", "24274", "65612", "2502", "70770", "62041", "12411", "67957", "27135", "53829", "37367", "63930", "25634", "23333", "30597", "26118", "37095", "24509", "24569", "14759", "3731", "1881", "40053", "788", "54322", "67273", "44048", "14795", "23760", "7107", "33536", "72064", "1040", "19383", "46779", "53792", "19149", "3010", "56492", "28064", "53480", "5115", "29255", "56547", "3247", "53463", "28515", "20501", "14318", "4285", "40986", "71984", "7863", "67384", "18732", "11984", "11721", "9213", "11812", "16247", "16235", "46229", "64176", "48239", "9208", "46517", "55955", "19699", "25033", "14435", "65495", "28940", "21295", "35179", "36896", "2789", "19432", "18973", "20320", "63815", "19207", "28241", "13591", "21195", "51107", "18781", "35085", "16463", "22630", "7426", "13899", "29947", "50168", "29339", "58391", "42496", "31026", "70382", "44849", "70081", "72048", "45265", "56239", "28253", "48797", "46082", "6314", "15384", "53135", "50081", "18103", "66890", "28486", "63911", "58608", "63101", "24231", "55713", "26004", "19897", "11817", "46612", "1502", "28497", "17317", "43210", "21040", "33792", "72971", "29286", "5250", "48785", "34401", "48427", "71493", "6762", "62824", "40765", "10705", "59452", "62248", "15606", "53643", "35668", "17022", "41039", "36351", "10874", "51607", "62950", "71254", "69388", "65960", "8521", "60347", "15035", "55715", "61178", "26606", "28036", "34673", "37433", "25400", "28432", "52501", "38368", "36951", "13255", "68700", "60951", "3859", "60235", "4269", "66964", "12976", "35698", "59676", "50938", "71803", "14487", "61853", "5566", "37178", "13008", "18932", "12447", "37383", "1441", "21999", "26389", "8619", "65646", "1855", "1178", "9304", "2218", "24238", "15980", "57340", "9918", "44352", "50035", "72632", "46718", "32207", "60484", "72696", "22132", "62802", "4989", "18967", "67328", "29967", "24396", "55466", "15019", "65626", "19804", "39439", "6322", "5819", "46604", "15157", "8035", "48358", "58877", "31446", "41686", "9831", "1750", "50143", "15810", "38061", "40997", "56188", "43284", "67054", "28630", "32248", "24729", "21934", "50240", "67240", "30439", "15365", "72991", "2586", "34887", "37754", "15259", "3852", "316", "70512", "69028", "49595", "19021", "20655", "51919", "12137", "69844", "45802", "32891", "26956", "66668", "49338", "48095", "68774", "55068", "35376", "69322", "7550", "73250", "58073", "35722", "33321", "9181", "12064", "33849", "58260", "14581", "45276", "38172", "17475", "65134", "29517", "63682", "62722", "67371", "28083", "16525", "38945", "24236", "65314", "49934", "72367", "41736", "56799", "61959", "35633", "67850", "26434", "26358", "41052", "39393", "29766", "45861", "50189", "2544", "6534", "3466", "31121", "50556", "41017", "13820", "9835", "31069", "45553", "29374", "42768", "30903", "51353", "28034", "21108", "68045", "43607", "42745", "48519", "1656", "11473", "66174", "27009", "42959", "55047", "45313", "24292", "29357", "56243", "31275", "23558", "9532", "38762", "56882", "17169", "40980", "2467", "13967", "3386", "39170", "61518", "62446", "2406", "55784", "30993", "61660", "67311", "54762", "10514", "38391", "59485", "9361", "17485", "41151", "60686", "67596", "37085", "4279", "654", "73079", "36579", "56164", "16001", "57498", "67536", "10480", "48283", "44970", "29482", "19119", "12493", "20883", "6580", "46272", "7493", "48360", "24202", "22540", "62358", "20292", "38285", "48885", "13606", "7451", "51175", "365", "9942", "25885", "25093", "31005", "69606", "52005", "25904", "29932", "52867", "18666", "47335", "2941", "58135", "15378", "50642", "49422", "65318", "29292", "40820", "51850", "13095", "66653", "65900", "67326", "14994", "72408", "37410", "15554", "22082", "53651", "68438", "36645", "11186", "15346", "53130", "42660", "55127", "8372", "36421", "45249", "1912", "41234", "11237", "36708", "61407", "65620", "46276", "63138", "36590", "31089", "26177", "17542", "19938", "25323", "59949", "42002", "16290", "52212", "55219", "45213", "10038", "13103", "47831", "13920", "44246", "18798", "57145", "41239", "49399", "66578", "7179", "72812", "8884", "42995", "68178", "27790", "50902", "11985", "33519", "70046", "70014", "22125", "41733", "3180", "32749", "61213", "16927", "18383", "15805", "60884", "48678", "53267", "9001", "66223", "8839", "7689", "12863", "31000", "50701", "63072", "1448", "3567", "10854", "24451", "10909", "62953", "53215", "53753", "55059", "46585", "66705", "68334", "50182", "31162", "52491", "63947", "4833", "52943", "7551", "68331", "32065", "57734", "40286", "36126", "32955", "69898", "25351", "3880", "5554", "31010", "18756", "59245", "5339", "52892", "48156", "53436", "41359", "44534", "41463", "32790", "37728", "53739", "50795", "60635", "37365", "51392", "14114", "38119", "66454", "46167", "675", "35351", "37703", "29221", "60009", "72187", "32548", "19409", "65987", "37163", "65630", "2214", "23911", "47218", "23114", "7857", "35524", "10795", "65733", "13810", "66253", "33107", "9494", "1799", "66330", "22263", "47001", "64439", "60000", "47843", "30897", "69104", "29493", "40505", "30607", "31087", "35156", "34932", "15837", "19721", "22807", "9201", "5409", "43136", "222", "41870", "69360", "69928", "71771", "66774", "11870", "66999", "15572", "62270", "1080", "55756", "4262", "23193", "60911", "59129", "7934", "43658", "71219", "42042", "34325", "21907", "43636", "54585", "29428", "54983", "60558", "55097", "69012", "15562", "58479", "55706", "63426", "58690", "45826", "69522", "34117", "12296", "60610", "53108", "51164", "34749", "19658", "31229", "55733", "34997", "51898", "65540", "69841", "20205", "61", "54793", "62343", "71648", "22377", "31639", "52586", "21581", "33884", "42286", "33973", "3744", "15011", "15785", "59115", "61606", "23943", "62668", "31105", "49537", "64609", "40445", "30468", "48672", "3302", "22115", "65875", "23791", "770", "68599", "37135", "60834", "52514", "1287", "46958", "29014", "62054", "26187", "29005", "9421", "22537", "66405", "65175", "48595", "44266", "64598", "29463", "24127", "51126", "5219", "44689", "22888", "19197", "43912", "30102", "15213", "18182", "31964", "39857", "53744", "71958", "71800", "17751", "35205", "62906", "13237", "16260", "68755", "16483", "796", "42536", "61896", "53637", "11858", "50741", "17212", "2517", "4538", "44787", "42020", "52862", "57480", "49456", "61252", "72526", "46869", "63443", "492", "29673", "25045", "11455", "11801", "23244", "44633", "63056", "1191", "50510", "14612", "56246", "38381", "3372", "40403", "7335", "36024", "22217", "15616", "19347", "69998", "10979", "17327", "32200", "54269", "58414", "41194", "11453", "32068", "41786", "32351", "28588", "25815", "72368", "42240", "53527", "7293", "25850", "40683", "17960", "52952", "32502", "45041", "38520", "15420", "6159", "63401", "33866", "34564", "37656", "24947", "16936", "44551", "32672", "49672", "31590", "25223", "36726", "13659", "12403", "1942", "18160", "35956", "49996", "38386", "9913", "28686", "38505", "24922", "21479", "38977", "40514", "49267", "70342", "1193", "29991", "22022", "42819", "650", "28478", "593", "67599", "53591", "4480", "43776", "38303", "14803", "45749", "15826", "55904", "53863", "26064", "14320", "53785", "27787", "2998", "1111", "744", "1467", "7890", "5620", "61160", "72701", "73215", "57357", "51336", "56375", "67111", "12407", "10584", "5044", "40373", "2922", "31956", "355", "41424", "49165", "60160", "6774", "60774", "31915", "1794", "54030", "49867", "33521", "9435", "72982", "7144", "59320", "38590", "17847", "44263", "16316", "19803", "31569", "35369", "17596", "27353", "48534", "9228", "7860", "52693", "31573", "29849", "4729", "19254", "1725", "31582", "50976", "47058", "579", "23712", "19139", "41370", "4967", "21706", "6308", "46736", "38175", "20998", "57205", "45719", "73139", "70998", "52925", "26044", "37099", "1463", "40513", "4664", "45675", "18465", "31486", "32052", "29275", "45380", "44958", "12685", "43566", "49414", "46471", "37631", "63496", "67449", "70032", "15120", "32990", "32716", "57747", "31776", "31640", "43066", "48457", "53979", "22119", "55431", "57579", "48842", "21629", "15760", "10235", "15696", "46865", "42449", "16784", "15133", "72245", "20345", "29951", "31129", "41495", "6814", "13143", "68654", "11780", "58372", "37185", "54853", "69667", "4543", "67297", "1660", "42433", "41155", "68411", "15208", "29962", "2494", "6454", "32521", "70175", "1919", "23471", "63178", "7556", "6723", "3637", "20178", "22940", "8992", "27653", "29216", "60338", "9896", "2965", "25019", "8268", "62301", "18571", "39604", "63885", "43905", "27380", "45070", "27436", "18775", "6831", "45254", "61967", "55533", "69542", "10508", "45587", "40208", "50419", "37572", "23408", "65021", "11485", "42271", "38732", "3471", "57408", "36089", "41699", "25629", "41674", "32898", "34173", "71248", "67980", "17606", "30056", "52287", "63400", "65261", "16807", "63368", "33334", "10694", "28395", "23523", "25478", "67632", "47158", "35761", "60541", "22910", "15697", "5442", "29799", "30150", "12869", "17965", "45705", "36074", "70961", "29781", "17365", "72680", "36730", "47109", "375", "71407", "13981", "709", "4456", "10230", "68587", "19561", "32002", "51620", "9306", "41422", "49345", "72451", "14783", "64778", "97", "47594", "70859", "57406", "29384", "70110", "36737", "17129", "16694", "28966", "70746", "32971", "48351", "24434", "21652", "21042", "2037", "3448", "32813", "55477", "68100", "692", "29461", "26583", "47234", "62786", "54633", "8870", "63252", "60700", "65644", "71810", "21135", "43639", "41482", "37191", "37821", "56163", "41263", "68569", "44656", "29171", "41435", "37354", "13413", "40951", "30228", "8288", "15627", "34566", "32920", "40638", "480", "28690", "59708", "40853", "36850", "30727", "13605", "20972", "52910", "26137", "59291", "22503", "57185", "62139", "62116", "32353", "63430", "20080", "51682", "58164", "5417", "60257", "37574", "55460", "11015", "33783", "31967", "42278", "64999", "56504", "57424", "9235", "20042", "38190", "72246", "23343", "70358", "45346", "35129", "7725", "62075", "28860", "67351", "30827", "158", "11618", "65846", "67695", "21433", "68841", "14017", "52689", "55976", "32670", "6129", "44669", "33198", "49015", "28952", "28322", "60429", "67731", "65613", "24334", "61137", "71097", "45609", "43896", "62369", "45762", "63768", "38006", "25956", "16153", "46182", "1845", "70889", "59769", "12161", "8463", "29135", "68990", "11258", "52828", "60848", "14307", "70575", "67708", "15921", "63112", "52577", "54804", "37910", "60126", "28342", "40267", "15097", "36186", "54623", "42836", "72551", "24062", "62516", "33991", "24462", "34988", "32018", "5682", "12366", "71259", "17255", "44284", "24807", "14873", "70210", "42173", "31029", "8531", "27520", "27232", "22161", "3096", "24527", "16443", "33693", "54764", "5903", "71919", "68540", "65202", "52591", "56044", "46298", "69747", "31448", "37114", "179", "39756", "71301", "4297", "4676", "72171", "7445", "57817", "30618", "16557", "23139", "37486", "43296", "2560", "21886", "61566", "31093", "47045", "13483", "48555", "41122", "36031", "12356", "22785", "44094", "41964", "73136", "32616", "71676", "68844", "57222", "57023", "68651", "5864", "52592", "26817", "18681", "8508", "23012", "31283", "64899", "44698", "64582", "41295", "32901", "24033", "14709", "54374", "21669", "17669", "33459", "25424", "61542", "20772", "925", "34604", "55548", "65966", "30424", "25123", "7406", "7067", "4683", "25621", "36561", "29487", "17902", "13679", "52329", "16814", "45055", "54981", "31207", "377", "31011", "63846", "17179", "34455", "73025", "42224", "21968", "15184", "38345", "45510", "51340", "2479", "38926", "72078", "40650", "16597", "59208", "26090", "27233", "12588", "69771", "15873", "2579", "36980", "65761", "3319", "6911", "53109", "44341", "10187", "53009", "34396", "29299", "53275", "63781", "33353", "929", "45246", "61562", "57455", "47384", "7481", "51014", "543", "64314", "73016", "36961", "36277", "73130", "68754", "37252", "47160", "43278", "27316", "47024", "62562", "57940", "26474", "62673", "64834", "68138", "59273", "53142", "55801", "29990", "11921", "1493", "35480", "15119", "26920", "21342", "29261", "38449", "69525", "23444", "30933", "33324", "59893", "50156", "59504", "21086", "1899", "61083", "8380", "31957", "38089", "37078", "13178", "1575", "127", "18140", "55307", "18652", "55581", "55168", "50099", "38967", "27645", "26933", "17178", "8883", "31175", "33203", "33979", "5473", "21502", "62016", "64986", "10529", "58355", "35355", "19268", "10844", "27467", "59756", "20468", "35118", "47757", "50861", "12711", "11265", "11977", "26939", "5163", "26968", "58682", "33873", "46054", "53466", "72747", "25794", "40501", "26969", "37240", "63572", "28650", "24469", "23600", "70119", "50837", "72732", "36948", "9230", "5530", "14879", "5443", "36189", "43289", "31379", "8707", "38437", "55988", "18010", "46667", "50192", "44244", "49539", "28466", "32649", "48268", "18471", "54430", "17560", "22664", "41450", "11558", "61370", "47861", "24637", "48816", "70056", "2903", "33896", "21240", "2208", "2923", "25522", "48439", "26642", "16613", "6627", "4038", "19374", "47511", "27536", "44530", "40564", "20536", "9618", "23175", "51053", "71594", "43726", "25753", "3239", "22710", "5767", "38405", "19392", "471", "20710", "50917", "59189", "21801", "39434", "44566", "50588", "20111", "68909", "70440", "71480", "18341", "55722", "23672", "42692", "53645", "32261", "71394", "6919", "15881", "33717", "8907", "8452", "44546", "33451", "13710", "42076", "1681", "13994", "39455", "57552", "70369", "26312", "52163", "43015", "42229", "42400", "64383", "63712", "33398", "1893", "1796", "38340", "48507", "18963", "33214", "41782", "26571", "33858", "48496", "31799", "27649", "15396", "34008", "8460", "13485", "61493", "49454", "42823", "22412", "38515", "261", "12716", "36874", "64758", "1167", "43767", "43514", "8622", "47042", "37073", "34059", "15468", "17376", "10888", "51893", "50504", "201", "69886", "40518", "30937", "44554", "25636", "37851", "23892", "57180", "59911", "21423", "30334", "51218", "72900", "52558", "56422", "35797", "55422", "47546", "60704", "23243", "7244", "22069", "20073", "62201", "60327", "44153", "40384", "15286", "6027", "62353", "34522", "12209", "25297", "6739", "65083", "49137", "72158", "60466", "39196", "31255", "35637", "46991", "70662", "25410", "46003", "34962", "9842", "66093", "64448", "15086", "47251", "15407", "35790", "30032", "66984", "55439", "21722", "12683", "36566", "48372", "751", "54377", "48064", "38033", "29687", "48556", "53991", "49574", "44333", "5063", "37143", "5033", "57953", "57583", "73155", "33592", "9338", "12070", "30726", "36731", "32970", "50661", "26635", "30941", "8850", "66393", "3102", "11828", "11220", "55620", "60139", "5119", "47642", "40971", "20577", "34645", "65981", "67755", "45626", "65853", "35711", "42901", "24121", "46314", "69363", "11964", "41041", "33961", "27626", "69270", "570", "64307", "48370", "55118", "40901", "65426", "66982", "31754", "25453", "40314", "7433", "38039", "13480", "62772", "44925", "28718", "49634", "28951", "22253", "55510", "71603", "26033", "927", "9965", "62207", "53167", "30966", "46909", "53740", "5929", "8206", "56030", "27403", "10984", "70658", "45980", "2461", "38800", "70000", "28694", "15448", "47193", "62342", "25651", "54810", "66488", "40283", "16547", "256", "42637", "71763", "25002", "16323", "40072", "72275", "72046", "48992", "42471", "19861", "13327", "62727", "29017", "53563", "58443", "63794", "58610", "71315", "60148", "68931", "35452", "12696", "51212", "32278", "37242", "58194", "51449", "41909", "29132", "14202", "58828", "71860", "667", "28970", "1138", "8205", "47934", "7388", "39728", "29578", "55643", "23190", "38133", "34106", "15080", "44504", "7023", "70042", "8867", "20619", "69723", "33883", "20839", "34785", "20734", "39397", "35370", "37330", "63750", "16668", "13340", "19739", "67972", "39863", "8252", "58649", "58676", "47337", "12302", "11939", "72672", "18996", "63051", "13277", "26867", "39018", "60200", "68693", "7295", "3997", "47793", "9003", "1987", "72885", "30176", "16579", "55981", "54796", "58695", "39131", "46956", "58165", "921", "10027", "31037", "67771", "36499", "22042", "24200", "65905", "52633", "62211", "52010", "55215", "25349", "17747", "31214", "19236", "49142", "7883", "34175", "26046", "50087", "12034", "38337", "16842", "17287", "72022", "45193", "24199", "9521", "62445", "50953", "18714", "70498", "1777", "16991", "51975", "57561", "25235", "25388", "54199", "18367", "46164", "18778", "70180", "52762", "61477", "9315", "23830", "29667", "64735", "15755", "64648", "67757", "53886", "38990", "28594", "33331", "22653", "18133", "53717", "12935", "12876", "25155", "12851", "51639", "71977", "2119", "3812", "52421", "63656", "67822", "7701", "23073", "52477", "40940", "3942", "31423", "8413", "16123", "45689", "49712", "6910", "17786", "1823", "36082", "25251", "13552", "67337", "60167", "53995", "40356", "16479", "41266", "67872", "1099", "73191", "7693", "31043", "266", "67164", "65062", "22541", "14929", "47114", "9520", "7168", "6020", "44922", "73028", "65071", "70682", "7062", "35738", "67493", "10758", "17109", "64460", "7879", "14388", "60333", "23477", "10719", "13076", "61269", "37992", "42261", "5066", "47782", "25190", "44500", "53078", "65438", "11963", "12725", "6338", "4576", "31604", "48742", "34891", "47767", "16305", "1943", "14661", "15885", "43323", "39040", "21252", "52941", "9525", "31391", "69775", "51268", "65308", "45651", "18319", "28080", "16734", "6583", "8575", "44948", "52040", "28221", "23647", "52031", "22519", "55196", "25878", "25252", "38487", "34322", "58559", "68040", "44035", "3230", "48743", "72364", "37004", "47589", "40903", "33264", "28457", "65524", "60249", "62888", "25966", "65809", "10398", "38312", "6949", "42668", "43754", "54758", "72690", "37088", "55205", "26419", "9823", "54565", "71141", "70654", "30067", "20676", "54184", "57619", "54056", "41205", "8779", "4689", "62073", "22798", "54076", "9619", "5352", "34041", "70294", "61672", "64764", "8664", "27193", "25751", "55111", "54783", "39516", "20613", "50778", "17112", "60359", "22645", "7937", "67143", "31727", "23433", "3929", "57663", "2504", "50975", "42840", "27120", "7717", "69286", "53360", "64435", "21451", "64562", "30126", "31566", "46422", "24017", "66772", "31321", "12014", "3205", "71447", "30051", "51836", "24507", "24312", "62260", "60708", "61169", "29380", "56549", "45459", "2251", "47882", "55765", "37729", "50714", "21079", "61619", "4540", "53026", "20417", "4938", "63497", "3145", "17788", "58707", "48636", "55825", "46073", "30997", "18374", "24792", "48276", "49320", "49583", "18488", "50555", "8496", "48557", "687", "4529", "29558", "35378", "61812", "15862", "5496", "41293", "70024", "6922", "41963", "69400", "32021", "26054", "48423", "54237", "66352", "44942", "47266", "10307", "9234", "41532", "2932", "595", "55747", "49889", "56336", "72933", "19734", "39151", "64467", "11764", "71415", "65309", "8499", "57572", "23612", "28726", "24846", "3782", "10387", "27883", "13720", "12537", "44214", "5039", "30938", "54185", "15152", "68431", "39232", "59684", "2205", "39008", "22761", "38712", "8020", "18750", "23369", "1511", "47816", "44425", "54421", "6690", "1789", "60266", "52691", "46354", "2102", "14034", "42870", "44489", "36564", "16886", "6357", "521", "58369", "19022", "69548", "43734", "53408", "6603", "5244", "49906", "68758", "53486", "17413", "32773", "28379", "2947", "30463", "627", "32880", "4881", "64702", "33181", "24622", "869", "63672", "39257", "55348", "42785", "53008", "54074", "23360", "19627", "59869", "58447", "46536", "23878", "11210", "53999", "14524", "4125", "35920", "5529", "26354", "39959", "19459", "16569", "53901", "17685", "16553", "10319", "56767", "58308", "68029", "26430", "66377", "13064", "8421", "47672", "4830", "62676", "42776", "30392", "18505", "68483", "9139", "58799", "42223", "21767", "19602", "33678", "44469", "37714", "7657", "39616", "60518", "40165", "2022", "8710", "26814", "61424", "68623", "41799", "24664", "72086", "65145", "54475", "72413", "4931", "16794", "68343", "26089", "54222", "68673", "70748", "33356", "8772", "59157", "39154", "17041", "54096", "63389", "51974", "44307", "27359", "26994", "41806", "69621", "54880", "27277", "17897", "876", "38271", "65335", "48705", "37300", "52611", "52879", "6399", "68355", "3824", "65158", "17931", "8978", "6141", "30677", "53384", "52610", "4859", "11392", "38092", "31824", "29385", "20754", "69148", "33406", "30958", "1499", "23695", "54579", "54805", "59191", "66313", "7052", "18971", "39774", "41262", "15874", "33999", "6947", "40020", "24968", "69668", "64576", "53671", "49417", "23103", "47667", "62092", "15561", "40014", "59204", "17471", "65841", "66212", "15267", "22582", "64391", "18100", "41352", "20000", "41228", "2196", "61549", "19689", "49980", "47544", "11461", "62566", "26235", "46631", "20919", "1563", "36975", "43455", "16771", "37335", "1755", "2617", "29821", "66768", "34771", "65835", "53426", "54167", "16871", "67469", "9371", "38649", "26790", "65565", "28504", "56705", "30972", "67901", "15156", "589", "29630", "38809", "58844", "70910", "25178", "41562", "28842", "9113", "58379", "32861", "41434", "35612", "65772", "62883", "39300", "40101", "25671", "59987", "40077", "607", "21988", "61112", "29654", "59114", "15907", "50896", "33843", "6613", "64083", "42069", "19801", "57557", "55146", "36241", "4066", "24058", "66733", "39273", "2786", "2237", "34577", "30725", "10587", "30614", "16377", "63624", "6904", "4300", "12469", "66899", "68713", "23172", "64786", "9499", "15165", "9498", "59683", "8338", "22529", "13384", "33338", "18241", "14508", "54429", "14440", "15815", "54105", "65159", "43295", "50353", "10672", "42493", "38222", "4436", "35083", "15746", "66736", "8414", "57250", "35013", "37263", "58944", "729", "43961", "17701", "27270", "393", "69348", "4382", "41358", "69479", "22102", "1604", "38693", "65270", "46443", "21755", "18162", "66883", "13750", "73033", "13968", "19903", "17977", "57193", "52705", "39423", "31761", "54626", "50981", "57738", "63584", "1962", "62170", "9832", "38324", "69167", "49193", "32714", "14021", "621", "53315", "38150", "50255", "43586", "44655", "59400", "66398", "68209", "52511", "16723", "4702", "30159", "54600", "68216", "46482", "10746", "71682", "21965", "63398", "14710", "26709", "10642", "70259", "34587", "3937", "5887", "26218", "36087", "29235", "32315", "68164", "34050", "52061", "35194", "67229", "26629", "30330", "10761", "2867", "30729", "37766", "46322", "28593", "64862", "42352", "71751", "32", "18222", "7291", "64742", "55223", "13634", "25374", "54479", "10950", "48698", "26387", "64858", "44124", "70867", "32419", "45527", "69715", "367", "67903", "39818", "1100", "54480", "20210", "71245", "55629", "45517", "23750", "30545", "61136", "3806", "40679", "51140", "26806", "17076", "64114", "19377", "49807", "57467", "17077", "56580", "45815", "48673", "22095", "58781", "30327", "42537", "56466", "58241", "3541", "37209", "8520", "22675", "2176", "50786", "37439", "48245", "38376", "56493", "37682", "56823", "13893", "12482", "27465", "46580", "39540", "28583", "16222", "32647", "37152", "54153", "50933", "39851", "62757", "13370", "67978", "32706", "38645", "39963", "38338", "20212", "13528", "66995", "43188", "24465", "15030", "45822", "46873", "7837", "51843", "19174", "1783", "42035", "20787", "8091", "12671", "55139", "43861", "40904", "39288", "33260", "12622", "41608", "27979", "50287", "63902", "21156", "69358", "6687", "70624", "33240", "10623", "66600", "7700", "931", "66972", "69420", "1469", "520", "22730", "53343", "16585", "11566", "6785", "70407", "44322", "25050", "49816", "39694", "53282", "18115", "53571", "13753", "47974", "32979", "67507", "71677", "12280", "33724", "14138", "44433", "46378", "70070", "34573", "5491", "23467", "49777", "50984", "47902", "62052", "3754", "52619", "34330", "6754", "64985", "68519", "13997", "42645", "50441", "72925", "2874", "51118", "29277", "58152", "61582", "65220", "27946", "3373", "34588", "38561", "38412", "2610", "69746", "49563", "51188", "49930", "43748", "58287", "67186", "6427", "1400", "5187", "55306", "905", "19266", "31316", "3564", "58506", "13309", "31060", "62962", "50696", "19569", "6656", "40892", "24862", "13651", "321", "41594", "14729", "69623", "46776", "15730", "56183", "18938", "32372", "17517", "35674", "45864", "68630", "49202", "8596", "3323", "34652", "13222", "57329", "14850", "8454", "41713", "54448", "29862", "63083", "41126", "63373", "50374", "46800", "34310", "3558", "10349", "27511", "7069", "51415", "72788", "19649", "63866", "37940", "34126", "25071", "57608", "13477", "2503", "55371", "66207", "45803", "10660", "39586", "18342", "1710", "17123", "28319", "12113", "66208", "3349", "10209", "17014", "15664", "49668", "8970", "18246", "57204", "32512", "1232", "58958", "54439", "23811", "61779", "39539", "68506", "3598", "57584", "26824", "33382", "10630", "41672", "25776", "44972", "57305", "1554", "47570", "50198", "33531", "9258", "17065", "1842", "14413", "32658", "7811", "51672", "62853", "20449", "42861", "47010", "52981", "72606", "9243", "39053", "39978", "33147", "45929", "66679", "65149", "6177", "30538", "49693", "35802", "27052", "61305", "55211", "30489", "66956", "71882", "66756", "29963", "61146", "41207", "61690", "66560", "70448", "8162", "379", "68600", "62171", "23997", "17470", "36489", "2219", "40352", "48168", "45091", "70402", "3098", "42475", "31848", "65700", "1022", "55748", "52472", "44678", "61859", "47008", "14730", "4458", "28538", "64123", "32473", "31693", "13877", "71257", "31368", "43113", "32039", "20253", "36592", "70952", "35794", "41244", "9224", "14520", "6865", "3672", "37961", "48860", "24260", "22323", "66301", "9707", "897", "9347", "9558", "1061", "27094", "53243", "64001", "18232", "7754", "58904", "38838", "47641", "25404", "51721", "41150", "63910", "13845", "50334", "72155", "61457", "52374", "63425", "25983", "29062", "57787", "36530", "23490", "6630", "56242", "10142", "71943", "17257", "41807", "38637", "6889", "165", "5651", "30143", "61673", "59279", "45601", "70255", "50082", "26281", "17522", "34303", "13512", "39095", "5325", "26441", "46046", "20830", "46969", "73199", "60511", "12030", "50086", "33852", "47788", "18405", "3896", "56280", "54210", "9894", "18555", "6410", "43389", "60975", "18745", "36026", "1512", "26081", "46106", "12887", "34521", "27004", "4665", "6987", "1804", "10267", "11851", "67865", "68987", "42679", "10938", "16506", "22913", "4316", "39722", "42625", "48000", "55770", "19736", "5958", "45492", "3450", "11171", "9599", "57677", "64388", "7870", "47483", "1798", "28474", "53477", "19749", "20821", "37683", "7547", "56523", "4764", "34144", "60565", "54765", "51631", "42346", "69281", "53386", "51591", "17326", "35450", "62761", "16958", "41732", "10964", "72224", "20451", "39308", "23560", "69016", "64569", "26911", "19647", "1952", "9103", "53263", "21100", "51585", "2975", "34940", "72969", "30909", "10596", "43396", "506", "14842", "15850", "18981", "58840", "53387", "58833", "9798", "10287", "10659", "23159", "25691", "25542", "13886", "35077", "4486", "49734", "31243", "26191", "26852", "456", "12140", "4492", "57759", "71426", "47781", "36374", "72534", "33659", "46025", "9503", "11038", "37848", "17226", "61122", "36767", "28242", "51357", "13509", "35571", "26861", "65507", "46104", "36586", "72458", "67511", "30962", "62917", "41389", "57111", "35062", "14276", "69304", "3748", "56363", "67292", "27131", "13126", "72737", "64822", "34163", "53788", "42666", "56919", "1724", "35915", "69285", "27412", "20139", "36719", "56320", "22226", "18077", "37914", "23188", "22657", "2796", "45180", "21491", "54058", "49604", "19206", "10026", "5794", "64227", "32106", "63076", "15269", "21538", "50137", "26760", "8024", "53222", "69539", "61279", "20721", "25566", "28337", "36743", "16596", "10951", "54760", "28469", "17947", "17625", "1284", "48273", "27726", "36539", "55054", "2045", "58448", "52621", "14956", "68381", "28443", "12180", "22506", "16743", "64072", "18872", "14938", "37049", "23576", "36762", "56204", "71680", "71117", "27029", "10330", "14333", "4877", "15998", "21291", "28625", "17709", "42832", "47845", "49097", "49143", "11523", "49174", "16347", "18882", "67941", "43042", "60351", "10724", "63165", "28062", "7494", "44231", "47930", "27908", "40510", "6033", "42680", "28180", "18831", "67215", "66022", "44924", "72480", "66303", "1752", "72658", "33508", "44767", "22", "11126", "19779", "30866", "6279", "58926", "10408", "52397", "38795", "53234", "71090", "31598", "15813", "63055", "22839", "20755", "25398", "40821", "35232", "38676", "23748", "40966", "54547", "67480", "7625", "11449", "37041", "22030", "71518", "58122", "69448", "39682", "18127", "44419", "37333", "32757", "42951", "49760", "39644", "11230", "66576", "49601", "31427", "55333", "13313", "38751", "6394", "72611", "65854", "46169", "17427", "63708", "4208", "53337", "44818", "58554", "37441", "46679", "42718", "38315", "51483", "35734", "2135", "59402", "28621", "41400", "17259", "28762", "51383", "42896", "66967", "24740", "17286", "28239", "6810", "65838", "9445", "35845", "6682", "2078", "17439", "24377", "19686", "49842", "57918", "30026", "20980", "14715", "54611", "10005", "50968", "44398", "16689", "20096", "64553", "30828", "58270", "8153", "33410", "5868", "3312", "69651", "68534", "69126", "40803", "23321", "46348", "58399", "6543", "7734", "62545", "13171", "35767", "58179", "50269", "71106", "51061", "19900", "37398", "12825", "42546", "23857", "22182", "9690", "6864", "37724", "5216", "72090", "41883", "12405", "61623", "63573", "36709", "6101", "55781", "48315", "4200", "41011", "65731", "50939", "8370", "29551", "16248", "54433", "13695", "56828", "53926", "42516", "48730", "63542", "41824", "63511", "33152", "24294", "28666", "2858", "44630", "27581", "69772", "54560", "4240", "34665", "13883", "19418", "18922", "37093", "60685", "69652", "1200", "20078", "69217", "11004", "43072", "55167", "8399", "21895", "35942", "69851", "68976", "31472", "68474", "72222", "24267", "16376", "21802", "63933", "55529", "30460", "53323", "70131", "25194", "43745", "56206", "55699", "63981", "14067", "15197", "8661", "71679", "39221", "12769", "34326", "12812", "56849", "53322", "35488", "43531", "12085", "37585", "68081", "26414", "9011", "43788", "61552", "13802", "35092", "58076", "59754", "18237", "8607", "69483", "60035", "70657", "62597", "41369", "66445", "54330", "61889", "31297", "24909", "32155", "17958", "26779", "32651", "12574", "68521", "11042", "51523", "4085", "538", "13403", "4808", "35246", "46363", "71457", "28290", "60803", "69300", "8996", "63431", "68119", "7642", "49115", "17988", "1176", "6243", "44763", "57022", "34124", "5378", "16450", "23017", "733", "11185", "30541", "61837", "43428", "33079", "23569", "65938", "25300", "34861", "24826", "21935", "52874", "30220", "10741", "14630", "7656", "44381", "67917", "51804", "11545", "59581", "45730", "51588", "20858", "68917", "65287", "21134", "66357", "16989", "12762", "56170", "59392", "19388", "41446", "24095", "51953", "46755", "70515", "37763", "65332", "70861", "57273", "38319", "25010", "48198", "33550", "43412", "6744", "3094", "63271", "52098", "750", "55512", "18615", "25875", "16706", "34911", "24216", "50375", "36754", "3718", "33006", "37831", "24197", "52885", "70884", "17545", "388", "37778", "7618", "48474", "67193", "65035", "45784", "29732", "26661", "13283", "27561", "73171", "31708", "17328", "4517", "32560", "57822", "64258", "62695", "17133", "16662", "3081", "63296", "51162", "31079", "53786", "53954", "58477", "5962", "23891", "1915", "56327", "43736", "39897", "39633", "68305", "60390", "10022", "11489", "40288", "35949", "2010", "31940", "8435", "55572", "61293", "45370", "43080", "36079", "22502", "29264", "4500", "27119", "55476", "27989", "68505", "72118", "38182", "60395", "139", "57284", "65213", "20391", "15723", "40817", "64703", "23999", "63842", "3766", "24370", "31113", "26928", "30479", "17206", "37564", "68428", "57613", "47957", "55534", "48998", "28785", "40908", "35589", "70531", "25661", "47326", "7438", "23101", "45787", "25055", "42576", "53806", "48591", "40560", "40910", "64969", "11286", "30174", "9325", "17056", "32263", "27121", "46093", "1606", "12067", "19993", "63294", "15141", "55594", "34835", "58643", "4049", "36496", "58453", "44286", "5741", "62473", "51826", "45916", "36114", "68736", "26813", "57029", "23393", "40107", "24229", "963", "34826", "34208", "38514", "72456", "26962", "15412", "54748", "25411", "72728", "53433", "30666", "44023", "24219", "19832", "7862", "36746", "67086", "27481", "12857", "17674", "60019", "43259", "6517", "4409", "21675", "64108", "62329", "4758", "69892", "33059", "19657", "26902", "27829", "69581", "33760", "58264", "54993", "26436", "52859", "4852", "72799", "40252", "55985", "11090", "12984", "54323", "55389", "35032", "66157", "45610", "25077", "67645", "62410", "43167", "21345", "9548", "16002", "59946", "387", "37713", "33167", "6997", "19253", "42585", "48928", "48908", "72122", "49016", "49648", "69547", "27205", "10154", "38230", "29425", "66101", "64528", "32974", "13680", "29058", "40725", "52424", "40192", "25976", "50520", "69301", "67910", "34906", "3830", "31862", "34159", "37457", "37130", "31810", "57459", "38403", "20876", "39998", "33242", "67263", "68958", "46575", "33623", "10851", "66716", "61385", "34689", "34266", "51890", "11927", "37894", "35597", "27928", "22404", "20273", "10634", "5626", "26079", "49908", "31949", "8309", "614", "22307", "60557", "20311", "67169", "54496", "44319", "62838", "12927", "58363", "8893", "68311", "35384", "49020", "25000", "13993", "55186", "30456", "69209", "61010", "52618", "43650", "73108", "17758", "52458", "34105", "62932", "52572", "24659", "42551", "34741", "66298", "50285", "68955", "53769", "15783", "40293", "67159", "54416", "65827", "53993", "15583", "16721", "18822", "19461", "63495", "40425", "51159", "66543", "2464", "41633", "32505", "22103", "26529", "45081", "42006", "66686", "27177", "68088", "12956", "8624", "72816", "16957", "9295", "71034", "55995", "18768", "2561", "55532", "45716", "19874", "64161", "67746", "68979", "66392", "65913", "69637", "28893", "70302", "10628", "7227", "45396", "28307", "46494", "24270", "68165", "60858", "36020", "58080", "26511", "54492", "16059", "3446", "10429", "24353", "10113", "68360", "27351", "55272", "71065", "65928", "703", "45373", "44122", "46608", "48691", "22944", "66415", "16543", "17592", "18925", "32931", "15982", "63665", "33803", "47311", "33108", "35733", "57241", "5946", "3467", "68392", "32312", "30386", "34452", "28453", "27975", "26274", "7982", "46932", "25967", "29099", "16172", "35019", "47617", "17911", "71586", "8357", "11496", "36749", "16602", "36680", "26715", "7833", "13612", "67870", "48747", "45277", "16205", "62852", "13336", "54844", "29330", "56047", "68895", "21936", "69089", "11120", "66222", "52887", "29016", "39410", "38332", "14937", "7886", "50986", "16970", "3271", "44568", "587", "70887", "14028", "40090", "16006", "55092", "22820", "1545", "51479", "26781", "28441", "36662", "40798", "68261", "24926", "35285", "3038", "43270", "41626", "26175", "13007", "52320", "62144", "20264", "48763", "59007", "63791", "71405", "35182", "55274", "59879", "62412", "37717", "72815", "23405", "29820", "67019", "72503", "14198", "49891", "24379", "43387", "24553", "46961", "65818", "72176", "18313", "5863", "59047", "48649", "35522", "63029", "43445", "28429", "30946", "8304", "18417", "45838", "41777", "45721", "13317", "13254", "28608", "41639", "72778", "58742", "25479", "27720", "72395", "53879", "22568", "39375", "3915", "1642", "18121", "66990", "508", "68342", "53149", "68143", "63295", "62914", "65427", "17583", "19670", "21242", "42788", "31137", "17063", "39315", "56922", "71873", "599", "2608", "46506", "47837", "11436", "26688", "53270", "22298", "128", "35565", "5321", "8578", "34037", "21634", "9725", "53543", "47155", "16010", "51678", "50884", "6085", "9796", "16677", "16795", "16965", "55071", "66510", "16380", "21467", "21346", "71233", "48775", "3380", "9879", "47734", "20808", "28507", "72316", "46438", "63457", "45891", "45983", "31035", "70136", "31451", "70166", "14141", "16312", "71087", "16018", "70787", "26415", "40887", "46959", "12549", "28912", "36806", "3788", "39559", "9449", "20294", "63858", "38849", "55441", "9164", "15625", "21008", "21408", "46267", "62435", "34987", "32504", "39004", "14757", "44646", "31899", "36570", "8699", "71180", "25484", "29505", "61222", "31238", "47996", "2798", "5976", "11459", "5659", "31485", "28327", "46824", "60881", "10009", "17281", "73216", "1433", "30347", "8271", "3719", "26341", "29866", "55007", "43420", "42398", "22060", "60420", "50342", "40699", "20717", "42670", "65450", "53079", "37868", "28535", "49371", "26723", "39960", "66045", "35098", "2695", "27376", "66403", "10868", "70083", "27522", "10673", "59614", "57116", "21910", "17682", "9254", "3662", "16697", "11730", "54749", "60241", "2623", "13584", "44841", "23941", "28297", "37863", "20031", "1049", "273", "56973", "70103", "18202", "38350", "9785", "5897", "21221", "23223", "55626", "5024", "10610", "57524", "21757", "12050", "43122", "35099", "51083", "25406", "54498", "72867", "24394", "55967", "24522", "10619", "6402", "36160", "20753", "43611", "51993", "36439", "17428", "11190", "8910", "38394", "9651", "59009", "22128", "27305", "17278", "37522", "42868", "48598", "21419", "18923", "53642", "54576", "54777", "58710", "45589", "13502", "63696", "66803", "49512", "4423", "26011", "37935", "12340", "39169", "12195", "26096", "5132", "66557", "41605", "56449", "12505", "24346", "39811", "17050", "34977", "11605", "1721", "1751", "71783", "48280", "31346", "69656", "12400", "49813", "29473", "73078", "37594", "53104", "41977", "14397", "16390", "4926", "5006", "62505", "31833", "59027", "4457", "63471", "14927", "32441", "32542", "55881", "69984", "50042", "8118", "16663", "49940", "72034", "37019", "1384", "50238", "58716", "71546", "28795", "34904", "28046", "55857", "73218", "16331", "8106", "31400", "38881", "20999", "2285", "56019", "37654", "30722", "1289", "2931", "22524", "31336", "40884", "5635", "70568", "69905", "21312", "26695", "49723", "48572", "50356", "44129", "63592", "35149", "68778", "37249", "47675", "64876", "26665", "28314", "62510", "7075", "19372", "55523", "63520", "10434", "47252", "42404", "21923", "33135", "13905", "41275", "71828", "28551", "16310", "45503", "271", "63693", "27745", "14691", "26801", "47685", "63634", "46026", "46722", "46172", "3660", "20119", "16897", "14185", "4496", "66795", "68193", "32305", "25281", "13428", "19941", "21519", "15996", "35921", "61248", "59207", "51870", "38582", "49803", "5608", "47152", "51991", "46935", "40606", "35796", "18898", "582", "24066", "41600", "73007", "53895", "10891", "44390", "19793", "33209", "58023", "66913", "56958", "72506", "48574", "18461", "70109", "58365", "20555", "44503", "68352", "40762", "8566", "49517", "32226", "30452", "51296", "6710", "56088", "6951", "3994", "32905", "53230", "28181", "37782", "13415", "13915", "12481", "13249", "34300", "17106", "13070", "14101", "29233", "59421", "12706", "50452", "61108", "39419", "71402", "70561", "14724", "2302", "48121", "47750", "60085", "38879", "65554", "51250", "69544", "36204", "46528", "69826", "42344", "16565", "1533", "9848", "65873", "50671", "17374", "52982", "10594", "27689", "57992", "34955", "13571", "66184", "33499", "67876", "28456", "57707", "62245", "19289", "68981", "68835", "37173", "49181", "32456", "68335", "39005", "71032", "53832", "59546", "40167", "27466", "39650", "54488", "70712", "10042", "17030", "66406", "51598", "51967", "17138", "3206", "28359", "18610", "22959", "46487", "16449", "41364", "67968", "4959", "19568", "59054", "1662", "57958", "72150", "12515", "220", "27092", "16237", "18854", "171", "2448", "2210", "45377", "41606", "13169", "54349", "67743", "73051", "10853", "7501", "18512", "46596", "47664", "52520", "21064", "42328", "51677", "40937", "67500", "415", "63674", "68234", "972", "40387", "49585", "10823", "61992", "17025", "21421", "32515", "22912", "19260", "54637", "16986", "37458", "58006", "72228", "36444", "72149", "48486", "62908", "62084", "5335", "24891", "15974", "30888", "3517", "24437", "49555", "58295", "10998", "15295", "14483", "56918", "61402", "50923", "38390", "42095", "48404", "60224", "64491", "69911", "66183", "20813", "52807", "22731", "48475", "30854", "30385", "52097", "3060", "42504", "48639", "11641", "4019", "71356", "39458", "27574", "25276", "71620", "41206", "22450", "66004", "67495", "69037", "61459", "39544", "59685", "57142", "7741", "61049", "55766", "43194", "57567", "26594", "56458", "47987", "56723", "65735", "21593", "10822", "62411", "19067", "45256", "54759", "57797", "45196", "1183", "19697", "36196", "50816", "14686", "708", "43621", "47144", "56256", "45141", "29521", "49264", "27837", "16370", "19064", "59928", "13761", "34336", "60855", "32277", "55753", "10295", "11456", "4035", "10220", "145", "22705", "68345", "25476", "37620", "8570", "31560", "69801", "62102", "11399", "68787", "28003", "45032", "25787", "42393", "55235", "180", "38869", "9175", "64698", "39176", "49592", "5178", "49044", "49056", "21466", "5147", "37901", "38349", "6198", "58514", "30236", "37028", "40152", "19433", "948", "64870", "10412", "874", "22429", "30599", "32095", "48586", "16911", "23842", "67688", "51430", "26918", "31484", "1813", "33668", "5916", "27954", "58731", "19986", "39311", "7537", "45660", "55552", "38367", "70194", "52797", "24279", "64204", "40787", "1017", "67083", "29027", "3189", "29076", "25605", "38115", "8776", "30988", "40722", "24942", "36733", "32563", "225", "54215", "56299", "68731", "50347", "25765", "49212", "683", "2801", "23094", "46419", "4159", "17081", "59301", "50906", "61154", "63481", "10176", "18712", "71965", "735", "67097", "50535", "30024", "42215", "39331", "40977", "71637", "26958", "42165", "28554", "54364", "41889", "35161", "14698", "6042", "54021", "28127", "68589", "46344", "24908", "66946", "68367", "37494", "26830", "58769", "45946", "23347", "51365", "41292", "17793", "12097", "29589", "70193", "53798", "52279", "22037", "8023", "67930", "44376", "13158", "5966", "65292", "34067", "67527", "34752", "6137", "69337", "59565", "23668", "51692", "6425", "6757", "6196", "5924", "16218", "65744", "34108", "3592", "50231", "6232", "25907", "19473", "33752", "21386", "19211", "4888", "46889", "68923", "72528", "17967", "69976", "32233", "15247", "9799", "42299", "16571", "59317", "73052", "8132", "28123", "26334", "32677", "18739", "47627", "4832", "13507", "56358", "43755", "49506", "2178", "18159", "25540", "5286", "4304", "33671", "45656", "23964", "35528", "46450", "4388", "14453", "11999", "8202", "65734", "26198", "71693", "51203", "54089", "31550", "1408", "54831", "16461", "19004", "65768", "14401", "56496", "21019", "8704", "53415", "21219", "7000", "34412", "71543", "21530", "22744", "26731", "12082", "32916", "3826", "8442", "34405", "47101", "53373", "69622", "6200", "62185", "1008", "50197", "11595", "6882", "54521", "56714", "51275", "5672", "48930", "50025", "37686", "72047", "63888", "43081", "50368", "56436", "64934", "18211", "19540", "28367", "42633", "70424", "70634", "39409", "1305", "35228", "28782", "40235", "62310", "41005", "412", "14597", "34998", "8899", "65116", "68042", "44262", "60042", "50177", "25997", "58821", "64309", "67463", "36978", "35644", "3152", "51358", "41721", "46178", "53049", "64791", "3086", "64666", "25803", "6351", "27811", "45901", "6972", "33346", "30571", "63444", "6761", "23979", "15931", "41629", "56724", "2320", "47367", "41471", "50945", "2175", "12425", "63065", "3127", "1211", "38485", "67372", "35624", "44073", "39687", "6830", "58217", "49599", "59128", "9122", "7670", "15252", "2184", "260", "19799", "52050", "47076", "13145", "39866", "51920", "64670", "50404", "66950", "32291", "54348", "67928", "36323", "51533", "24927", "68077", "28287", "16083", "67976", "21438", "31558", "32275", "3298", "32687", "46643", "20343", "38614", "43741", "57181", "19914", "27781", "33319", "9910", "27997", "58856", "40240", "52764", "26831", "51225", "23383", "13513", "45713", "37875", "20796", "10411", "20631", "39950", "15852", "52414", "37402", "9014", "52924", "28269", "59451", "10825", "63646", "60859", "71261", "56270", "65338", "66878", "13709", "35016", "2635", "3198", "20562", "50220", "1675", "21061", "62789", "42604", "69418", "30556", "57369", "60752", "19463", "24374", "15672", "38683", "65075", "51461", "21807", "25583", "55481", "19061", "60755", "57650", "40392", "22331", "47007", "57571", "36517", "9468", "48972", "13322", "16578", "49919", "20742", "11365", "9955", "16785", "62455", "15734", "17011", "11671", "32807", "26385", "16400", "31664", "17307", "49064", "33967", "2015", "32574", "19763", "8988", "52154", "60109", "57259", "46400", "68724", "46735", "45127", "10531", "72965", "17227", "41211", "49180", "27176", "29378", "62952", "59031", "1129", "51356", "43409", "55321", "49423", "39304", "60553", "50214", "70344", "60067", "12617", "15602", "3699", "35447", "24568", "35275", "26051", "34097", "63554", "13832", "7785", "58282", "5265", "29247", "24052", "32011", "6106", "22799", "37338", "27969", "9067", "1648", "51182", "55309", "3321", "7604", "21800", "51385", "21427", "35777", "39235", "68647", "32991", "33616", "57473", "13965", "62945", "6376", "72583", "53986", "41248", "29100", "57445", "12327", "37820", "45800", "39164", "55771", "48592", "6104", "8191", "58646", "3807", "29196", "58605", "45906", "53747", "22166", "8069", "48710", "49670", "9860", "51862", "59098", "31290", "7895", "8878", "63720", "12747", "58587", "49857", "15680", "39133", "45563", "39570", "43540", "58345", "15110", "8803", "49006", "7172", "35358", "69925", "52890", "42294", "13811", "40824", "69108", "53496", "17663", "50210", "62785", "22459", "61156", "53594", "54645", "58624", "48301", "45101", "19217", "29708", "3795", "24466", "62864", "13029", "5658", "71696", "9839", "54111", "4981", "41875", "28091", "47674", "55852", "55550", "12820", "17120", "1570", "18364", "30701", "56644", "65457", "16570", "55390", "66271", "56768", "50446", "44072", "24015", "47519", "55773", "14327", "35200", "27701", "2066", "924", "20020", "18295", "24122", "7322", "13708", "38912", "31562", "63385", "62982", "71099", "59409", "25821", "14389", "53781", "28942", "16304", "4724", "32497", "48327", "24078", "57436", "4093", "16437", "25529", "23688", "11169", "28490", "29653", "56776", "29770", "14423", "6201", "3235", "57595", "52408", "10639", "2558", "57731", "28404", "56650", "19694", "22638", "37596", "11982", "33418", "63341", "38714", "52307", "48440", "20352", "37962", "39876", "6472", "35660", "31271", "66370", "27366", "57314", "70290", "56632", "3739", "60549", "69614", "72916", "46376", "48803", "25222", "56086", "3842", "42024", "1849", "15578", "3675", "37853", "32427", "21639", "72883", "14907", "29008", "30312", "25954", "29342", "26531", "69903", "68252", "3419", "25321", "57019", "71773", "25724", "2896", "28251", "21416", "19797", "64066", "57778", "16809", "22953", "43003", "5061", "60187", "12847", "33220", "18281", "63448", "10361", "22651", "28104", "51282", "2412", "48740", "30568", "23125", "6348", "3776", "52915", "25898", "69047", "47715", "69249", "4993", "39466", "60891", "52368", "17454", "50763", "17084", "38062", "20151", "63938", "70508", "19815", "47866", "72114", "15143", "2575", "55136", "955", "27587", "37075", "71422", "39228", "56103", "16230", "56053", "64337", "2253", "70438", "72548", "6135", "35363", "10064", "44130", "72902", "26456", "53745", "45080", "72752", "34781", "51740", "4435", "36801", "72040", "28676", "31386", "72229", "29175", "33171", "47048", "17992", "44960", "13976", "14305", "22425", "53098", "45040", "64306", "36259", "55871", "5359", "1921", "39527", "71318", "54956", "44183", "32136", "69832", "38914", "30886", "13785", "34497", "67699", "11816", "50894", "8753", "52530", "56488", "69644", "44213", "55498", "30578", "39692", "45625", "54512", "34623", "56230", "34288", "14886", "28165", "66065", "9915", "29868", "36937", "5784", "51191", "33278", "55625", "53114", "13153", "29953", "14000", "7206", "37635", "41", "40764", "63069", "66270", "5035", "62518", "8299", "5459", "46519", "57928", "45302", "21758", "58121", "7258", "21572", "6453", "11376", "27736", "66742", "64387", "39252", "2192", "53357", "39340", "18703", "3449", "6812", "62512", "16375", "24456", "8540", "39402", "33935", "52054", "21998", "40916", "30782", "61705", "66066", "69327", "50517", "27348", "18311", "52443", "8977", "62049", "51382", "52171", "35814", "24440", "860", "69203", "3557", "57570", "40496", "36271", "58259", "19973", "50300", "10564", "5646", "12908", "11115", "35476", "22248", "54150", "65724", "754", "14171", "18597", "49957", "8958", "18639", "55502", "23547", "47040", "49419", "39048", "72407", "69982", "68052", "5603", "9546", "36497", "36873", "4094", "18738", "19553", "18550", "4139", "64356", "45761", "48303", "17889", "27847", "19472", "55051", "16612", "66704", "13671", "21127", "49763", "21265", "49548", "2275", "8146", "41345", "42950", "28662", "41967", "36142", "53462", "18339", "18435", "7299", "43938", "18786", "68628", "43333", "23577", "63458", "5596", "6561", "2235", "33878", "47738", "16253", "17523", "59930", "49330", "55987", "48617", "36544", "737", "31762", "18916", "58810", "33457", "72334", "13466", "45478", "54676", "10", "16587", "49575", "44440", "28299", "37157", "53743", "17274", "36508", "52060", "24614", "68851", "56621", "3256", "8550", "11751", "14163", "64128", "59458", "59860", "40157", "54544", "24393", "56043", "45843", "6735", "23327", "59333", "4007", "52758", "32451", "54071", "32366", "58318", "36947", "21590", "37139", "3948", "347", "50442", "13462", "63666", "67569", "31418", "15001", "7552", "56480", "2976", "24071", "68670", "18719", "53598", "7592", "27411", "56354", "28171", "3738", "39990", "55491", "65346", "217", "52623", "65207", "2534", "2304", "50263", "35693", "67344", "5847", "48172", "66889", "65296", "46844", "31502", "1608", "51685", "64645", "24316", "21784", "8944", "72293", "49814", "29702", "40047", "14702", "40329", "39825", "6011", "30207", "43951", "10872", "16487", "3921", "61047", "14420", "18545", "26160", "52121", "10488", "31432", "4661", "20353", "40391", "48320", "25719", "65765", "41319", "29454", "37034", "21168", "36122", "47414", "67842", "23653", "52351", "61939", "45963", "13013", "62935", "34890", "16379", "44326", "32744", "15296", "978", "40462", "17185", "26027", "41927", "39612", "67310", "17525", "37187", "38673", "65348", "28147", "41988", "31459", "49309", "63244", "42642", "67441", "22526", "63522", "52575", "50135", "29916", "70932", "70673", "37259", "27309", "25202", "53388", "4594", "49396", "572", "26094", "21636", "37325", "21773", "9897", "58814", "14899", "58249", "48347", "69241", "16522", "24125", "10745", "39778", "40005", "54497", "20663", "37158", "11983", "16671", "11159", "66136", "35399", "65122", "16716", "6931", "56822", "44342", "67369", "60570", "22822", "43950", "56058", "22633", "2698", "8754", "4263", "17844", "46940", "43593", "67296", "57368", "24952", "67506", "11785", "68845", "27266", "55555", "36085", "23516", "28871", "32770", "47397", "42572", "39966", "51671", "27686", "1744", "26416", "63684", "65962", "52540", "25441", "64487", "53302", "10788", "67622", "55451", "51954", "43699", "69717", "22167", "21712", "70522", "45170", "24978", "64961", "52009", "72206", "15006", "55300", "51763", "61296", "23993", "46305", "44734", "22207", "50396", "3076", "54335", "47509", "18502", "21350", "38870", "8791", "55585", "31033", "32276", "15818", "62742", "60660", "64575", "58255", "70804", "34529", "3504", "45023", "20674", "43983", "46030", "56933", "72402", "56568", "70178", "57735", "48873", "6915", "19878", "40678", "63168", "59514", "11118", "63061", "2660", "9211", "36430", "11773", "53344", "40716", "51054", "39804", "69783", "50284", "35508", "61505", "51713", "45883", "36631", "66660", "17060", "6331", "58917", "14437", "34269", "40182", "68641", "43490", "34750", "69038", "52380", "56931", "6124", "29167", "57018", "65710", "51597", "20507", "63449", "45732", "40481", "62199", "48834", "9777", "28024", "10096", "65778", "38452", "69745", "65516", "32596", "45902", "12939", "15571", "23409", "16027", "39624", "55489", "34553", "17208", "7166", "4120", "38856", "55524", "15495", "30812", "4843", "61543", "39898", "31023", "69915", "30645", "8932", "68171", "66361", "19150", "70228", "32304", "52682", "38366", "123", "72952", "5886", "10900", "10558", "63313", "65081", "5671", "51419", "63249", "64880", "50590", "44167", "43640", "35121", "46915", "50321", "48352", "28144", "32655", "15923", "981", "2934", "28981", "11158", "13578", "1399", "65582", "54943", "9180", "69495", "838", "49095", "3497", "54472", "40559", "30025", "13531", "58364", "71320", "44344", "50770", "58566", "44229", "48143", "26615", "36236", "53584", "12391", "6933", "51590", "70697", "50865", "56972", "40869", "24709", "69194", "59152", "25203", "6516", "72570", "5757", "39226", "18050", "29333", "65157", "20829", "73071", "68749", "5634", "31067", "70464", "43203", "30209", "2104", "29710", "53941", "32181", "35344", "47914", "20956", "60595", "21222", "14642", "53803", "67602", "55977", "16338", "12071", "37784", "72569", "8594", "56034", "44331", "24668", "47056", "3641", "49430", "26393", "55204", "22901", "53452", "11734", "17252", "36248", "11319", "10273", "20240", "51480", "37108", "25608", "37031", "64203", "17593", "44438", "51321", "59915", "6765", "19502", "41113", "18983", "61227", "37740", "23661", "31351", "61687", "14204", "378", "23992", "41625", "66836", "9220", "69198", "44235", "66815", "10195", "58125", "68783", "22727", "26201", "11446", "68500", "15451", "12317", "53209", "53375", "46842", "18830", "38174", "421", "29481", "49144", "44749", "3678", "47282", "2933", "32478", "15301", "48306", "25245", "51669", "50187", "62241", "23731", "49606", "62679", "5249", "20167", "3623", "57335", "30536", "4460", "66918", "46583", "67388", "48963", "27018", "71312", "39640", "56961", "23224", "43038", "56009", "68879", "36536", "63716", "68156", "55042", "45290", "53813", "52001", "10109", "61551", "66534", "25620", "38407", "37570", "30000", "58951", "62714", "53014", "49249", "3126", "49878", "42972", "55519", "7956", "59955", "69970", "24230", "54961", "29695", "70950", "65440", "15187", "37657", "55407", "65635", "56360", "41251", "5645", "20383", "6954", "45074", "51465", "48165", "21960", "22175", "39917", "24703", "17416", "55783", "63435", "45825", "62156", "71143", "12131", "63363", "20511", "25334", "43965", "29712", "59188", "44522", "14127", "6068", "26929", "43426", "9255", "44580", "48124", "14288", "4685", "72250", "66221", "18436", "68752", "1381", "14680", "23208", "64181", "58871", "140", "12817", "71314", "12278", "35817", "64256", "37254", "13556", "32886", "48311", "68887", "44166", "1065", "20375", "64027", "64659", "14874", "33085", "2249", "19413", "12813", "48147", "58912", "48850", "52937", "47301", "48089", "42107", "8556", "56788", "35397", "46221", "67233", "43369", "58784", "61880", "36143", "56181", "67532", "67689", "5546", "22487", "32249", "7256", "4884", "62844", "60856", "65129", "38027", "52569", "58386", "42735", "5332", "24160", "57692", "20948", "12967", "15945", "38725", "36445", "54884", "34492", "38291", "6448", "46374", "19072", "62167", "24203", "8527", "67512", "40760", "4855", "24748", "51774", "3545", "21536", "35045", "64465", "19244", "65317", "16698", "22193", "4697", "44834", "75", "35607", "11379", "14592", "42349", "36636", "13419", "26610", "10579", "69626", "48794", "2396", "52182", "66606", "6479", "64687", "47735", "24537", "16912", "57903", "60413", "27974", "8843", "10470", "12058", "37904", "70202", "11317", "50435", "69686", "43065", "65481", "30530", "34814", "40324", "38385", "68921", "66033", "5955", "50468", "34191", "15942", "13790", "26725", "60663", "69931", "65785", "20262", "8449", "16850", "12610", "57343", "69529", "25211", "41677", "22259", "66914", "43535", "52649", "1541", "13688", "62035", "3756", "22902", "28831", "30449", "15816", "4338", "38050", "38434", "34728", "72318", "51773", "47146", "60177", "3834", "16178", "2751", "52190", "57367", "49511", "52134", "34789", "3073", "19887", "64059", "17849", "5434", "29851", "27482", "23825", "68213", "43681", "69716", "44833", "15925", "9101", "3601", "8369", "53128", "10399", "31224", "55376", "65632", "8530", "25689", "15289", "25342", "46864", "58803", "62821", "49091", "8423", "10527", "31686", "17013", "43467", "63822", "34637", "59803", "35136", "15206", "71796", "42940", "21528", "25036", "21660", "55623", "23350", "22416", "29457", "13421", "21383", "32221", "64973", "22698", "33273", "61113", "22320", "63529", "8739", "11647", "17590", "3841", "68056", "27584", "71715", "32367", "16302", "42621", "14972", "57544", "61371", "55290", "36015", "55255", "43598", "29163", "28052", "3684", "51561", "51747", "55328", "34442", "15457", "21977", "40025", "67718", "59620", "6653", "36860", "45390", "50485", "72424", "19155", "30368", "21246", "1210", "43852", "40054", "30971", "25809", "38815", "11368", "25692", "45177", "70427", "40031", "39184", "12074", "19393", "608", "7736", "47942", "73219", "37707", "34119", "7013", "43570", "21271", "60273", "41561", "71653", "31471", "21556", "64902", "65588", "43047", "18260", "35503", "62150", "65", "36635", "46294", "53770", "1281", "41429", "1702", "17401", "51261", "31840", "3214", "32771", "25714", "71579", "5337", "64532", "64241", "5754", "72554", "68485", "37430", "24546", "43516", "40628", "61760", "26171", "24983", "69111", "61641", "34348", "25038", "60144", "8761", "32091", "42774", "72743", "17329", "54666", "15493", "53870", "69346", "59996", "49208", "42378", "11630", "40416", "64572", "52164", "14818", "41711", "49167", "52202", "30130", "58314", "7313", "36157", "38871", "9280", "35021", "53522", "23822", "63454", "69298", "24725", "10617", "57982", "30913", "21472", "55905", "33143", "70990", "65531", "48049", "26528", "44884", "26433", "21893", "13848", "5950", "26821", "64047", "13792", "48094", "26543", "26159", "662", "64932", "32395", "58590", "67370", "43468", "10647", "15069", "26636", "73121", "2435", "4020", "30447", "30311", "9290", "11756", "53499", "41448", "19539", "46407", "20385", "15473", "30680", "14173", "29012", "62334", "14348", "45182", "72872", "4452", "30678", "12881", "27740", "43668", "43715", "2520", "29356", "30801", "34239", "13714", "48923", "7549", "43556", "39313", "52176", "5627", "49671", "20241", "60930", "46916", "10930", "30579", "36684", "13464", "29856", "39752", "36292", "6206", "22297", "61063", "38648", "9345", "10435", "2912", "55357", "28616", "11373", "64886", "10733", "26266", "31331", "16504", "33855", "10575", "22550", "36258", "25941", "4210", "23916", "42231", "71812", "2704", "40195", "48340", "19583", "49931", "72208", "19404", "33970", "5145", "58216", "29873", "56136", "26555", "36367", "55444", "18842", "59084", "21299", "55859", "28724", "59411", "1543", "57307", "16444", "54833", "11030", "29713", "8470", "59513", "62692", "69810", "56722", "45294", "36437", "69866", "10345", "15875", "1069", "42307", "18052", "43198", "48120", "47318", "32864", "13879", "66649", "20355", "33639", "69899", "7578", "28804", "3326", "32035", "38318", "38758", "48744", "55619", "72751", "46765", "62957", "20032", "9778", "20519", "43230", "37956", "30724", "4301", "44544", "30410", "31576", "41110", "50584", "21976", "43676", "28519", "153", "4582", "59711", "15113", "38822", "42406", "29861", "14975", "25301", "9773", "10921", "38013", "65602", "69616", "36113", "44354", "16840", "54850", "35677", "14074", "45441", "53056", "56605", "48472", "35608", "21956", "17288", "7855", "14298", "69705", "31180", "24758", "39775", "7136", "15507", "35699", "33736", "6535", "571", "12355", "58859", "26680", "1593", "13673", "43911", "70926", "72683", "48082", "54481", "59493", "42497", "8750", "67541", "65836", "44473", "12743", "11655", "71744", "22991", "28059", "6545", "37294", "1966", "36702", "10707", "9682", "70600", "17648", "50064", "16636", "29467", "23423", "73236", "38065", "39889", "43737", "34017", "68650", "53876", "49213", "26452", "3883", "29568", "47655", "26849", "38025", "60028", "38935", "20713", "67630", "45195", "10879", "63923", "40661", "1356", "61021", "64444", "12546", "60802", "61731", "21964", "1914", "62428", "39919", "35895", "67894", "1601", "6850", "59468", "24993", "35869", "54770", "14663", "42728", "39149", "30437", "46527", "57246", "59520", "4629", "56066", "5848", "6412", "71745", "34596", "22870", "11500", "41716", "29624", "29557", "39723", "33162", "61048", "63548", "60303", "10003", "2700", "18886", "68969", "47705", "65532", "32817", "6748", "803", "35815", "10889", "39111", "40160", "64262", "70412", "27870", "28980", "31794", "6280", "27321", "15839", "59697", "66328", "30873", "13704", "1109", "46825", "28447", "64223", "31163", "16484", "11045", "34160", "66152", "20944", "54015", "69263", "71866", "60736", "63215", "56683", "27049", "4004", "62138", "34519", "56455", "39814", "48035", "1426", "37804", "73244", "12423", "15754", "40393", "23606", "42133", "15624", "13737", "36071", "70072", "29051", "10170", "51", "67450", "36061", "55365", "43949", "32017", "19523", "35901", "51088", "64058", "31470", "33810", "49976", "55953", "8692", "37528", "351", "26303", "46123", "31447", "4312", "58458", "32520", "66894", "70841", "21599", "57785", "58869", "60155", "48407", "26488", "56194", "10942", "8120", "53196", "33859", "6407", "53940", "56597", "3136", "34188", "48504", "58940", "60410", "42754", "69569", "12089", "49442", "27972", "12611", "32561", "40253", "50868", "20792", "13667", "51664", "58472", "13496", "5913", "67325", "72365", "10130", "72898", "35051", "52816", "71891", "46072", "46185", "66478", "59850", "42067", "62125", "45160", "46637", "33610", "6725", "49052", "33355", "63685", "8406", "68760", "41996", "72836", "9200", "42906", "27993", "31063", "48364", "57909", "9745", "69253", "41178", "34878", "1691", "62313", "68301", "26003", "13195", "15591", "31624", "11708", "26659", "53704", "29153", "52489", "14895", "49997", "41191", "29510", "18609", "57077", "63499", "65478", "27910", "21013", "35864", "71872", "27650", "53624", "41664", "59125", "19821", "29032", "41147", "59989", "53488", "36509", "61129", "58311", "67849", "70813", "55347", "16223", "15331", "46100", "8498", "43622", "1235", "10422", "65860", "9397", "9348", "440", "12096", "27763", "10787", "22601", "9747", "55221", "57060", "13202", "59322", "44386", "69075", "72328", "31237", "72298", "65167", "16404", "31692", "55003", "61913", "11394", "60101", "31962", "21682", "60171", "42123", "44306", "12455", "37519", "62112", "7199", "39479", "2703", "19085", "65656", "14360", "24360", "43520", "57952", "13429", "8240", "52568", "35080", "6727", "45814", "46230", "57612", "44156", "54790", "29154", "52222", "59295", "41722", "1764", "20496", "58404", "22094", "71614", "63681", "34150", "19442", "13752", "6506", "24347", "38627", "62825", "22871", "54395", "52716", "66232", "72017", "44637", "41753", "43437", "35808", "68244", "42966", "4918", "41175", "13777", "67679", "28364", "13924", "36231", "20153", "51311", "56314", "48615", "5498", "19833", "55088", "62549", "34025", "28792", "37947", "69192", "27559", "31707", "24483", "47801", "32581", "54897", "64295", "13197", "16647", "20644", "3014", "7864", "63110", "9988", "55001", "48772", "33593", "15288", "68767", "20407", "52990", "57869", "38219", "64618", "63667", "67368", "61904", "70414", "25891", "20013", "55959", "19845", "63008", "26627", "23305", "3284", "24794", "14439", "13596", "1416", "26295", "60079", "24753", "69114", "18370", "38169", "32973", "60098", "7198", "67077", "25007", "44962", "40259", "71201", "14671", "28383", "47648", "41447", "24438", "59951", "23213", "18887", "10219", "34176", "25860", "7303", "7899", "62716", "67407", "49640", "60036", "49427", "53695", "15790", "65421", "20997", "59653", "24967", "1558", "1054", "28196", "6695", "62831", "53797", "18328", "39982", "31937", "46805", "45445", "25829", "67242", "58444", "66701", "28817", "3732", "44258", "24025", "33551", "51378", "32130", "7978", "29755", "52717", "35580", "61427", "3628", "32802", "57634", "40343", "33001", "61500", "11301", "17801", "71076", "30275", "63725", "44404", "37059", "32953", "4287", "34218", "30608", "36371", "9936", "12045", "49026", "64601", "72061", "46289", "8585", "36365", "57724", "11598", "48697", "65280", "37491", "64911", "43060", "59095", "50370", "60017", "34775", "22845", "37926", "69538", "49883", "14593", "29509", "23377", "2437", "31889", "26748", "3178", "48100", "7607", "44356", "48371", "43995", "15163", "1998", "38191", "17352", "64267", "62990", "41144", "64341", "48726", "1723", "35591", "12582", "70558", "169", "41083", "22321", "3350", "31265", "23183", "39639", "7043", "44490", "47909", "12283", "23283", "9", "26161", "71297", "66176", "37047", "11765", "16203", "61034", "14192", "14640", "24871", "68658", "2963", "31241", "66457", "66149", "2111", "3975", "23197", "67753", "20406", "67044", "28276", "69000", "47002", "25843", "40408", "66224", "38171", "28300", "15123", "37228", "35141", "11727", "48116", "2438", "5181", "5423", "13189", "1515", "69713", "11757", "42142", "63038", "47821", "57775", "37422", "39105", "23162", "20050", "46929", "38517", "46324", "25654", "64336", "48560", "53391", "63905", "52830", "24305", "8980", "34558", "13387", "26696", "19750", "58147", "35613", "70491", "37561", "64905", "46996", "13581", "16108", "9864", "39408", "48112", "32543", "50761", "9495", "22573", "19229", "43156", "19017", "13682", "34850", "50752", "24301", "15016", "4873", "55015", "62249", "54602", "27515", "2916", "1679", "22287", "12864", "46621", "44275", "70752", "54061", "21294", "64373", "4925", "9453", "22767", "36340", "59808", "44492", "49185", "68750", "48259", "42862", "41067", "59306", "37897", "61716", "1358", "21501", "47431", "26674", "7808", "36110", "25682", "62593", "24774", "34995", "39013", "49096", "44457", "33455", "61558", "30643", "41662", "38054", "44274", "20007", "4731", "48516", "424", "70576", "21382", "71797", "16069", "9985", "57133", "44379", "15741", "20089", "6226", "23519", "44789", "61281", "62266", "16130", "66616", "66912", "44227", "855", "33864", "5510", "995", "54515", "6982", "71200", "25733", "23233", "8045", "29574", "17441", "60006", "70783", "47518", "24536", "46219", "70619", "57223", "43135", "42574", "30120", "57220", "71613", "25336", "34195", "40624", "33501", "72127", "30994", "55637", "14765", "27028", "4976", "26006", "53417", "65349", "11096", "55956", "60188", "54294", "15089", "12541", "29272", "35653", "5004", "49008", "45611", "7448", "6401", "32266", "36453", "17166", "13361", "57304", "20248", "58348", "21512", "44501", "62255", "48395", "9954", "41320", "11909", "15934", "14098", "32139", "46306", "59785", "27490", "49751", "46402", "31338", "57120", "62307", "71861", "62258", "52131", "25146", "11480", "35751", "71732", "4728", "31280", "39087", "21521", "23733", "11411", "67074", "60419", "20509", "55667", "5562", "70267", "8004", "53172", "36593", "21306", "33062", "61775", "151", "29350", "14659", "50091", "26868", "71477", "60063", "24252", "67831", "31352", "71241", "69161", "61961", "34278", "17762", "69379", "72372", "60821", "55206", "51937", "50253", "42011", "6373", "50330", "36326", "22815", "11823", "16335", "41886", "28807", "48531", "59526", "34920", "13036", "64069", "9921", "11592", "67150", "60966", "47551", "29485", "49247", "39201", "3138", "24038", "24223", "10895", "1616", "58188", "57876", "9191", "33950", "50337", "8160", "1256", "1192", "27287", "66607", "7402", "28179", "63638", "17222", "6077", "62611", "605", "5815", "16738", "40313", "59121", "44901", "57290", "24602", "60149", "1831", "46002", "19159", "42412", "3184", "6938", "64499", "64116", "25704", "22173", "17767", "26961", "22960", "49793", "7829", "1774", "23014", "24802", "14278", "32408", "50389", "62669", "52137", "46036", "40381", "57137", "60161", "66196", "17357", "39264", "39193", "26854", "30586", "2276", "29390", "48968", "42337", "70835", "60367", "51604", "24380", "58762", "41694", "1005", "359", "60025", "38583", "13294", "67504", "8795", "43863", "31303", "49781", "41436", "54308", "21194", "34804", "7028", "61216", "38553", "55194", "28665", "20687", "11143", "65823", "24114", "198", "41724", "425", "33905", "10461", "40238", "3213", "57338", "51520", "72973", "17469", "38426", "22942", "6212", "10756", "32042", "11745", "15880", "10200", "61287", "758", "29803", "39376", "8924", "29069", "70447", "68825", "19365", "51337", "34703", "15430", "64317", "18844", "51877", "69376", "39422", "60733", "22087", "67987", "72860", "13565", "41053", "8759", "39712", "27751", "20340", "10166", "57093", "55648", "21477", "40166", "50680", "36670", "29072", "48559", "1415", "51759", "46568", "55693", "2063", "63015", "55669", "9583", "45593", "11912", "20604", "6048", "24391", "55687", "56843", "9138", "66575", "28150", "26578", "21606", "9737", "58753", "56151", "13837", "67911", "10880", "2877", "446", "186", "44654", "51464", "270", "33244", "68973", "68831", "18476", "32855", "61356", "19221", "14697", "17238", "36638", "61492", "68097", "44989", "34952", "52970", "19299", "67982", "36537", "14395", "1995", "50640", "53783", "15864", "43104", "42066", "51987", "53012", "66639", "31132", "16464", "5745", "69235", "9771", "44021", "22040", "8135", "14472", "46645", "4526", "51481", "13910", "36124", "16688", "8291", "17699", "72375", "41928", "8464", "7040", "66683", "5032", "9416", "64797", "12057", "27506", "51787", "2627", "58095", "31109", "39071", "54658", "43886", "18900", "54563", "44029", "12689", "62173", "48746", "54630", "34240", "44980", "49899", "69496", "52288", "40619", "34523", "22396", "59553", "6661", "23345", "16493", "20312", "64638", "40039", "25640", "16206", "28442", "45835", "48325", "13567", "36092", "11000", "28496", "26517", "46333", "7078", "62216", "69860", "26753", "27651", "55125", "45851", "43677", "42113", "26368", "46472", "72432", "55806", "57456", "19945", "42364", "24559", "7564", "2885", "14062", "48170", "36267", "61226", "69312", "48398", "1668", "7911", "31408", "15691", "5711", "39520", "1185", "61854", "71583", "23798", "33891", "51869", "70529", "7517", "32440", "31122", "31143", "17360", "67187", "50904", "14392", "484", "39347", "19464", "19573", "60001", "30282", "56135", "65123", "56114", "61692", "18513", "58794", "26135", "45107", "63731", "11834", "27956", "47806", "44065", "51571", "37112", "21582", "56347", "62062", "9998", "7661", "28675", "33715", "65360", "68837", "7750", "19688", "14526", "40613", "61488", "16482", "42214", "39576", "39702", "890", "57213", "4523", "42468", "39631", "41412", "33654", "7964", "16261", "10077", "43988", "58947", "35672", "47563", "12192", "10135", "47796", "62968", "7413", "61766", "14822", "58866", "24714", "66926", "3092", "10585", "20058", "57974", "57086", "6306", "1653", "37462", "30774", "27830", "66165", "6431", "42934", "16145", "57401", "555", "68818", "58381", "15111", "27504", "42581", "35155", "6975", "22604", "37603", "21449", "66539", "29989", "17838", "10747", "37625", "18906", "50153", "12255", "53644", "42361", "135", "31150", "54135", "57055", "50710", "54612", "38847", "2342", "70260", "67487", "6650", "31572", "40811", "5898", "54213", "65940", "8513", "6382", "69478", "49618", "7202", "3645", "3251", "57172", "37020", "24934", "62585", "2748", "1290", "2202", "35821", "24658", "44558", "55087", "25998", "40219", "31123", "20647", "53250", "58337", "29217", "30136", "31836", "40715", "49148", "59793", "28789", "1188", "14980", "39050", "35687", "5150", "16098", "30799", "18934", "56663", "34346", "32359", "39500", "41346", "35785", "3941", "22033", "18594", "29094", "49075", "28721", "5865", "58104", "19056", "44933", "62703", "7976", "40251", "22445", "44141", "14218", "32474", "6310", "39561", "40376", "23115", "11586", "48489", "70149", "54583", "68125", "21799", "37003", "17854", "55892", "22763", "50257", "37051", "13546", "65575", "18309", "31594", "13447", "54063", "66968", "39783", "20485", "12873", "15526", "3315", "28033", "33595", "14622", "29398", "69936", "44343", "60465", "30114", "64734", "50549", "67327", "46468", "9461", "33067", "32420", "43353", "44175", "59423", "56341", "13844", "14894", "27725", "34260", "33969", "33060", "25017", "66007", "64298", "11176", "15385", "37091", "9054", "663", "48716", "47363", "4839", "42414", "53975", "109", "36495", "10677", "202", "45200", "37179", "20912", "28682", "37942", "34584", "35342", "32247", "35256", "66741", "930", "44536", "33103", "3724", "70406", "35", "32837", "47769", "28224", "49529", "71365", "6971", "14479", "33821", "60616", "22975", "8051", "21072", "56033", "152", "28950", "453", "32154", "12009", "32452", "62439", "24171", "57784", "25144", "46986", "30754", "60544", "18867", "44704", "63531", "7722", "33608", "8056", "6712", "19723", "7362", "67037", "68003", "68619", "20488", "8825", "42395", "31544", "42360", "31739", "53245", "2587", "37640", "66281", "1843", "36616", "70181", "24409", "45166", "25672", "13992", "66609", "6926", "13601", "38176", "57834", "40924", "71818", "26499", "72915", "64165", "47967", "23528", "2228", "39784", "69612", "48996", "61984", "23432", "9663", "19958", "12686", "8665", "69118", "71969", "44135", "5996", "15360", "16859", "44560", "65901", "32236", "71016", "64577", "21085", "30271", "20336", "2590", "34829", "19891", "72427", "48066", "2375", "50926", "43651", "28527", "44940", "34862", "32961", "62350", "48564", "41535", "13933", "12126", "70381", "39859", "69985", "63102", "21060", "55535", "10184", "52106", "26173", "37086", "33646", "10860", "45954", "41738", "55830", "32175", "69800", "56559", "32296", "2836", "20744", "26851", "711", "1451", "31453", "56319", "63983", "51134", "45267", "10207", "68594", "36130", "30484", "42070", "30369", "37355", "54894", "67535", "64691", "39883", "67144", "39345", "23056", "1153", "2756", "47120", "42054", "27800", "25888", "34200", "11075", "65032", "17283", "34139", "34184", "19808", "42314", "30349", "50994", "8995", "49806", "20107", "54798", "38075", "55455", "72814", "59649", "37315", "13456", "26210", "20548", "47489", "71902", "49232", "69803", "25317", "21952", "45941", "46971", "68359", "43205", "7002", "36247", "48990", "7850", "20535", "26738", "21084", "72148", "20573", "56692", "47082", "31763", "64344", "11282", "54661", "36246", "26042", "56097", "52343", "41279", "15716", "40004", "41025", "19978", "7521", "46248", "56208", "43216", "60496", "58826", "57324", "67348", "40599", "43837", "44796", "22829", "18166", "50587", "22863", "6176", "35990", "42601", "26002", "32642", "64953", "27044", "16189", "72630", "58743", "64109", "16311", "49109", "47241", "26332", "5979", "47804", "28495", "40164", "56305", "43142", "47271", "27821", "51502", "46447", "62760", "3036", "60469", "70831", "35529", "43106", "48021", "57280", "44408", "41938", "41710", "26690", "37924", "35837", "62220", "27449", "2305", "55537", "30564", "32809", "14599", "66879", "51760", "16185", "40248", "8787", "47323", "63844", "32066", "66875", "49384", "51782", "6017", "44555", "11444", "31823", "41780", "46768", "55848", "21631", "56713", "52425", "60722", "23684", "31929", "59358", "32786", "21149", "43025", "19306", "55591", "57820", "60603", "36839", "43114", "29289", "2760", "21753", "28472", "52023", "40590", "4135", "21055", "48724", "56901", "70052", "70546", "73203", "64816", "26684", "45844", "23581", "33761", "42702", "47321", "22583", "53726", "61683", "1378", "33652", "67733", "21666", "23935", "63078", "42389", "46936", "70043", "43363", "3774", "22576", "19055", "35242", "48606", "32430", "6953", "56717", "64521", "55442", "58935", "66527", "4857", "52074", "21159", "50295", "65201", "37064", "2146", "33218", "44855", "55401", "35544", "50444", "38143", "5192", "47892", "40745", "61354", "5798", "66599", "6184", "24324", "26550", "69818", "31877", "31529", "18974", "37544", "11064", "16292", "67189", "32328", "46217", "13734", "33534", "1613", "49237", "33092", "27986", "26021", "27876", "20688", "32600", "59229", "45925", "28372", "20872", "36605", "1871", "68248", "61196", "12130", "10771", "53520", "62384", "34392", "39642", "61710", "20112", "10262", "24201", "53446", "34262", "34345", "66237", "14166", "57652", "15180", "34820", "11199", "31119", "10199", "55700", "28815", "47426", "67015", "57350", "25151", "48684", "20157", "14884", "5322", "31072", "12259", "49282", "62326", "34021", "35978", "47484", "20815", "51510", "31645", "58732", "39322", "66230", "66514", "51543", "55327", "43229", "14758", "41096", "19991", "66798", "49110", "15551", "68159", "21909", "13738", "48832", "27432", "48138", "7093", "64731", "29572", "56639", "214", "40834", "37503", "47416", "22003", "69766", "15380", "55424", "26853", "5714", "12492", "25623", "36181", "48447", "2382", "36314", "11992", "36230", "46429", "67411", "30340", "44831", "52883", "65722", "63193", "29251", "17154", "1427", "44586", "46613", "26082", "55861", "17675", "11553", "17461", "37170", "17296", "29129", "2945", "33473", "2510", "10597", "31497", "51849", "70714", "14172", "33104", "46582", "35654", "36268", "39864", "67132", "61834", "946", "28629", "69617", "7038", "64214", "6585", "6304", "30097", "49066", "993", "13681", "11123", "51125", "71826", "39224", "5172", "56513", "71260", "21340", "57183", "34190", "43827", "31415", "9784", "65371", "6827", "9759", "67607", "49368", "61741", "4078", "23925", "16907", "22900", "51852", "34209", "3540", "38956", "702", "57582", "69078", "46480", "34471", "53260", "39496", "72446", "35500", "55041", "71527", "42641", "38825", "38355", "20534", "37701", "72112", "18913", "65811", "40035", "28043", "33000", "69666", "46476", "31222", "70", "42047", "47121", "72871", "57933", "11541", "70127", "18528", "53816", "58746", "8805", "8174", "29451", "1901", "61201", "10885", "30524", "72247", "66231", "23191", "22076", "55027", "29458", "51326", "47270", "54665", "44430", "40589", "54698", "19834", "19705", "38068", "8972", "67349", "59847", "12494", "64041", "36376", "37116", "68493", "2540", "22908", "5740", "65235", "1872", "5673", "43717", "18948", "34478", "33971", "65535", "55797", "34430", "19198", "70516", "32686", "66827", "20550", "3196", "68061", "35408", "38226", "26609", "25273", "61238", "15046", "56840", "28868", "42937", "65580", "67008", "16251", "59598", "7491", "8641", "40636", "54710", "61907", "55124", "47366", "34001", "46068", "28081", "72995", "51066", "4914", "73127", "576", "64402", "32899", "18665", "18019", "51033", "54236", "58765", "37460", "32825", "41238", "53567", "33795", "52371", "47680", "62860", "15514", "13109", "58849", "43534", "48570", "48396", "37207", "24036", "15565", "2289", "19117", "70164", "33813", "7507", "32457", "5783", "33445", "37583", "55285", "68066", "36350", "71379", "11245", "58701", "58789", "45223", "20672", "44003", "45111", "24445", "69900", "46300", "20940", "12393", "64556", "20386", "9382", "20177", "23158", "60710", "57045", "19410", "358", "58455", "46750", "24943", "65843", "22465", "12577", "58822", "46913", "2955", "67739", "37357", "7909", "29280", "66473", "46014", "17824", "4437", "35031", "35598", "40295", "63733", "39296", "9840", "3905", "7465", "5217", "40865", "39593", "20602", "46380", "11991", "28243", "24596", "33195", "51757", "48868", "68140", "34086", "20553", "29939", "46902", "67171", "59151", "63739", "33097", "39662", "38720", "8868", "23472", "48302", "13735", "22432", "11033", "19708", "12149", "26605", "60163", "66531", "15532", "22177", "68608", "14970", "9217", "27240", "11086", "64781", "68001", "7735", "36811", "6082", "38746", "332", "20047", "72495", "2168", "45252", "67618", "51197", "34669", "64621", "20745", "71332", "23489", "42428", "48733", "2677", "54928", "65771", "2429", "9941", "35701", "25646", "8598", "13028", "32029", "23540", "40956", "23793", "67950", "50498", "68068", "4221", "971", "53445", "3898", "29038", "50173", "19496", "11898", "51770", "59126", "53610", "61864", "32487", "27059", "7731", "61313", "1370", "11733", "36550", "33204", "7932", "62137", "48588", "54533", "37599", "42913", "22644", "49895", "65073", "56539", "23378", "13869", "40273", "5030", "56920", "66261", "27872", "21844", "3845", "58622", "11754", "2982", "63922", "24167", "18420", "47272", "13030", "36940", "41691", "28400", "61531", "44616", "49741", "38019", "63718", "68372", "6729", "52628", "68436", "23150", "35774", "39006", "20743", "44089", "27804", "42772", "54200", "33126", "32673", "11954", "51944", "40718", "53223", "33310", "28237", "17181", "56237", "72964", "29784", "30985", "8956", "35525", "7568", "46511", "50716", "56703", "42290", "57485", "59321", "13323", "4654", "47633", "63906", "18066", "42878", "3253", "17855", "53405", "23248", "68174", "54241", "72242", "71360", "34535", "43573", "26728", "10685", "22903", "15737", "5813", "34219", "35557", "9430", "47897", "27683", "71976", "22229", "6084", "596", "23169", "71498", "36583", "67478", "17729", "3502", "69740", "56743", "19684", "32870", "19987", "18517", "36831", "11434", "23980", "14777", "24997", "61345", "22090", "68488", "31299", "41487", "61050", "29760", "66021", "52630", "67734", "55323", "17937", "22811", "65578", "6345", "4483", "13703", "49591", "67821", "36396", "71230", "32138", "994", "50221", "6696", "43472", "6052", "25021", "59547", "59556", "9096", "24526", "54784", "7996", "56710", "68443", "56690", "25001", "3381", "27125", "29824", "50364", "29291", "62687", "41719", "45086", "30975", "21603", "11719", "2120", "63700", "38114", "44202", "72531", "72782", "18318", "13402", "57306", "21065", "52636", "59185", "8362", "13872", "45374", "2973", "49896", "45671", "64929", "21074", "19544", "65567", "1395", "21258", "64328", "71524", "25116", "55014", "55872", "45810", "42805", "13275", "37748", "21878", "8485", "23927", "32785", "17269", "50768", "22595", "64507", "7613", "24932", "19859", "66706", "50666", "47361", "52966", "64628", "1244", "53376", "8693", "73125", "43544", "62402", "45483", "48211", "16289", "26219", "50067", "69232", "39807", "2613", "60761", "61718", "62302", "29639", "33962", "63095", "9867", "45518", "39831", "71306", "2048", "1953", "72405", "2825", "55004", "13971", "21685", "20549", "37307", "4302", "34397", "61078", "43263", "27173", "5160", "66228", "46215", "1742", "35603", "60788", "19964", "2676", "55645", "26678", "35906", "19862", "53418", "44208", "6440", "22478", "40361", "59635", "69995", "7943", "2549", "3137", "40027", "5983", "72992", "69677", "20291", "36342", "23580", "21747", "11518", "10161", "6073", "35553", "63236", "22234", "73085", "65055", "17601", "11154", "64959", "60240", "28440", "36477", "43172", "66133", "45987", "14375", "28956", "5839", "52079", "59176", "8003", "34270", "48781", "49386", "22588", "50878", "34666", "39358", "4712", "31191", "48596", "40842", "22038", "15442", "3333", "48127", "6871", "26070", "30693", "1663", "10591", "35184", "39341", "24592", "65681", "56376", "30694", "38969", "65637", "1108", "68287", "62624", "49146", "11795", "20982", "62701", "7076", "8652", "34221", "3565", "8524", "39108", "66690", "26454", "19068", "44862", "11391", "46543", "44512", "42757", "42912", "2391", "47378", "24806", "44147", "27996", "47027", "30690", "44315", "27030", "14473", "56615", "58640", "18777", "70999", "49433", "7555", "10046", "51552", "5005", "45546", "31284", "49366", "50157", "48414", "69107", "10927", "47292", "65366", "33528", "22326", "67801", "41115", "25657", "69175", "32766", "37758", "21641", "72195", "35935", "61575", "38093", "46541", "71382", "17393", "13319", "284", "41486", "47951", "37066", "18980", "37838", "50080", "68516", "60121", "4154", "41551", "60598", "20783", "17245", "47795", "69264", "3909", "62095", "8757", "7515", "64910", "31215", "53848", "66727", "3597", "38698", "49646", "48711", "40797", "37919", "15661", "1560", "4557", "23008", "26955", "15248", "70017", "50481", "16801", "46006", "51458", "54687", "25837", "466", "57589", "12653", "47185", "51559", "60795", "511", "52812", "21997", "50988", "57000", "35479", "66401", "68572", "44854", "3550", "52590", "58726", "24425", "11364", "24987", "29898", "18618"]], "valid": ["int", ["19077", "7881", "39793", "10218", "8741", "19548", "26067", "28100", "54917", "1550", "745", "68884", "35383", "6639", "33487", "22921", "67750", "786", "12794", "51517", "40197", "12239", "25902", "6087", "20403", "9469", "44143", "13264", "18479", "36541", "27212", "71854", "26965", "63978", "366", "59640", "70039", "58886", "53639", "52112", "51625", "68229", "22485", "32712", "55147", "68629", "50932", "32875", "17647", "24888", "71703", "22986", "19846", "4376", "46439", "51122", "19248", "43537", "67555", "40016", "61574", "53917", "45899", "39120", "54445", "989", "22491", "15534", "10764", "26986", "32820", "42350", "10910", "63415", "32608", "55786", "52580", "39989", "11682", "27079", "13716", "70292", "19065", "35783", "63329", "17190", "27082", "13049", "48169", "20525", "28704", "71497", "7237", "65541", "51466", "56982", "16993", "10047", "3480", "29104", "34069", "4252", "71427", "32801", "69243", "38029", "57130", "54527", "53862", "56858", "24304", "24697", "39605", "13772", "24359", "20298", "42595", "15008", "55437", "58617", "60893", "32215", "28939", "48896", "63590", "33627", "70252", "67573", "45488", "23299", "28162", "16084", "34844", "69924", "827", "32552", "64096", "17945", "33032", "19653", "18943", "16050", "53307", "32782", "24042", "40807", "68027", "42345", "45186", "66122", "72320", "16898", "41179", "53091", "67405", "52445", "62543", "52731", "18958", "31865", "68968", "39880", "34852", "24554", "55867", "8702", "12281", "28759", "38129", "68172", "49184", "60104", "30560", "42139", "47768", "8203", "17242", "24173", "14525", "25557", "20014", "64783", "62405", "50603", "1152", "21329", "42543", "66206", "66015", "52376", "56820", "48825", "37698", "22950", "49681", "5009", "44291", "42025", "22361", "9716", "40528", "62014", "60623", "3259", "53866", "26882", "67184", "36692", "14560", "22985", "48750", "63180", "48752", "29897", "42438", "33831", "40401", "18520", "63445", "22597", "16359", "64513", "15029", "7051", "26894", "32926", "48376", "2900", "15570", "58312", "11094", "14779", "37087", "54171", "18662", "51914", "64280", "47878", "50236", "19564", "50934", "63912", "71532", "48282", "43600", "40500", "52984", "2580", "20200", "682", "22997", "46953", "40028", "31574", "66565", "59298", "30731", "23583", "56746", "49951", "18740", "51530", "11066", "24686", "41014", "16015", "10185", "23131", "57986", "24161", "64942", "30071", "14833", "61471", "31086", "12943", "48206", "47064", "72477", "68575", "71656", "35983", "51253", "56367", "69743", "30396", "71813", "59687", "40591", "47621", "8629", "37292", "23681", "44372", "11343", "45076", "45710", "14058", "64920", "25846", "1602", "24088", "60052", "42526", "61318", "11350", "9051", "25154", "70992", "12897", "25446", "71809", "32330", "42845", "52250", "20500", "30215", "66307", "30647", "33329", "21464", "39941", "28238", "54848", "27439", "56070", "33019", "25884", "50954", "42172", "38483", "33051", "70677", "72131", "14964", "36410", "38705", "56930", "6714", "41726", "14756", "47374", "45326", "6043", "71452", "47006", "2114", "15146", "47331", "24329", "21560", "60075", "14951", "39246", "59868", "56691", "66102", "45640", "51474", "42707", "63719", "72753", "14611", "48614", "63040", "32949", "40766", "30234", "62393", "16630", "21944", "29929", "27627", "61462", "21164", "45608", "32577", "12903", "14830", "65935", "23163", "29317", "8082", "18546", "42530", "36432", "1810", "24021", "31630", "40551", "71473", "71048", "33553", "72181", "51234", "20461", "50487", "55440", "12102", "38574", "5407", "6514", "20933", "55631", "65799", "67213", "46206", "12934", "38124", "1352", "13494", "27005", "23047", "23001", "40503", "62141", "27697", "20347", "31662", "25247", "50320", "65001", "19633", "50098", "51615", "40509", "24141", "13000", "18715", "14100", "31521", "57042", "56153", "47151", "31944", "69158", "338", "65430", "45520", "7311", "11769", "72826", "43297", "19008", "8503", "37304", "37799", "68652", "72199", "33206", "61712", "26906", "32237", "22276", "33848", "21637", "62204", "41940", "5051", "29764", "40814", "47380", "51223", "32806", "70676", "37825", "309", "17993", "51996", "14429", "20329", "56317", "39985", "41008", "72185", "67106", "30664", "38868", "60409", "11826", "71147", "68344", "68041", "61153", "61090", "51176", "10843", "68709", "63876", "33927", "33025", "52234", "47089", "56104", "71719", "40848", "48379", "25385", "34189", "37118", "20431", "41453", "32085", "45686", "72834", "54531", "33384", "25163", "54487", "29914", "45319", "1850", "55345", "49528", "41224", "17585", "7605", "70246", "50827", "41588", "56315", "22803", "3980", "51403", "25911", "54695", "72343", "24969", "12122", "8009", "40989", "23078", "2187", "12931", "44134", "65210", "38428", "13670", "21823", "1534", "31605", "38807", "50463", "3934", "41466", "71601", "58782", "66217", "67635", "15053", "8973", "30285", "70718", "53475", "58157", "66011", "1978", "474", "72009", "69257", "29619", "1711", "62589", "15747", "67425", "58580", "30287", "39494", "51011", "45202", "21115", "19731", "5878", "8386", "8909", "29364", "36200", "49486", "13541", "63663", "60614", "18687", "44041", "31393", "4077", "20520", "44676", "62653", "49053", "56120", "44755", "55999", "35569", "72251", "4167", "56780", "36232", "749", "9008", "66532", "69459", "64622", "28883", "2823", "16628", "27886", "29679", "38442", "25368", "65276", "65914", "649", "63327", "37669", "48123", "752", "11024", "41501", "59967", "62021", "56997", "58585", "48602", "67473", "25632", "12145", "43306", "36065", "64503", "14241", "43592", "14178", "21379", "58720", "37601", "45275", "17610", "61342", "40309", "34402", "5517", "3750", "10581", "19237", "5585", "53752", "32449", "71545", "56606", "27507", "27882", "31362", "13873", "23038", "72482", "34931", "10484", "52484", "27766", "36744", "7706", "11919", "48997", "33096", "59401", "47682", "19153", "5427", "61401", "11441", "59173", "14781", "38254", "41767", "4237", "1778", "25157", "50824", "23651", "29236", "29273", "4875", "4936", "17362", "43459", "56331", "28601", "67693", "27335", "38321", "5497", "29254", "22421", "17406", "64354", "23663", "21533", "72864", "45647", "12596", "5046", "19270", "6211", "4607", "20890", "71389", "34579", "5936", "44606", "6782", "43064", "60354", "7958", "52452", "18679", "43795", "71694", "16805", "53814", "69511", "60047", "23307", "36305", "70647", "48953", "6528", "59588", "60190", "21678", "36599", "60896", "3246", "1216", "5499", "34178", "56006", "11116", "10149", "38465", "28858", "61017", "37972", "69449", "60519", "41882", "42108", "65196", "54709", "31417", "37705", "47206", "36185", "47026", "51395", "55085", "25124", "15806", "7156", "67514", "33626", "58655", "31952", "31", "5390", "70667", "26586", "48545", "63541", "4982", "23129", "50689", "65390", "9972", "36315", "30464", "10898", "36576", "64664", "1911", "28236", "25823", "52025", "57432", "40074", "18688", "30027", "24900", "27902", "27089", "68129", "19586", "22883", "25834", "3499", "28988", "44160", "30831", "16925", "61713", "69155", "29740", "42907", "45215", "65316", "33801", "42122", "62850", "12707", "56202", "60383", "39261", "54101", "34214", "68420", "38845", "45149", "29300", "48441", "14673", "34622", "31444", "14521", "31344", "12745", "22567", "27334", "1844", "58215", "44150", "53501", "19767", "58064", "20252", "38984", "12111", "55018", "15757", "13727", "44387", "18197", "36199", "28008", "6064", "39171", "991", "38908", "20725", "57182", "20639", "11109", "49905", "38342", "52207", "69697", "41572", "16657", "18626", "32984", "51345", "29579", "50961", "39748", "29694", "59787", "61662", "3790", "38427", "25458", "73018", "44898", "4641", "60018", "11132", "54519", "38581", "35341", "10115", "69596", "67715", "5832", "4485", "36319", "18816", "38305", "37495", "11079", "29928", "16085", "3432", "58973", "38187", "11830", "5586", "26287", "23330", "2179", "7467", "6694", "6704", "25565", "52872", "33148", "49460", "59801", "13082", "13635", "17543", "18621", "36394", "22054", "18463", "70898", "64355", "41449", "45908", "25501", "2902", "29550", "4408", "47736", "253", "28736", "60103", "22939", "73021", "7371", "67962", "65238", "52314", "49866", "25065", "38017", "49488", "55246", "39767", "64168", "35938", "46385", "65536", "50943", "5276", "8436", "44444", "69971", "46203", "41584", "61299", "54825", "51163", "28284", "64178", "8931", "69879", "14001", "54918", "15536", "28102", "10189", "33446", "40769", "52823", "1071", "4115", "34063", "47242", "328", "62524", "29773", "46158", "64304", "30762", "54108", "9405", "26313", "533", "20068", "43530", "61285", "64805", "53828", "56175", "35237", "37796", "19452", "5441", "59662", "44583", "53601", "23100", "40579", "1690", "99", "9417", "27983", "21732", "46258", "21181", "41794", "46113", "69927", "46618", "50425", "11625", "52355", "58570", "39573", "23192", "106", "27647", "5238", "36664", "61158", "63736", "59038", "52423", "34832", "30470", "19837", "67138", "24101", "34358", "48688", "15367", "21917", "22530", "50798", "14982", "52542", "43261", "23119", "60777", "23065", "3398", "26018", "4092", "32639", "22791", "53602", "19870", "11774", "888", "60007", "31264", "9523", "15513", "51081", "52002", "32851", "26587", "17825", "50802", "17277", "14669", "12662", "28217", "67068", "69639", "30887", "3211", "60860", "46108", "58137", "1972", "27189", "46647", "17894", "46702", "46011", "70655", "36641", "36328", "66932", "72708", "73187", "53400", "2418", "6491", "36107", "8746", "20192", "25100", "63504", "19200", "11583", "72454", "1525", "5424", "21674", "72029", "30889", "49643", "14295", "38722", "11616", "62155", "22068", "22258", "12367", "69045", "31628", "24916", "71833", "62395", "26585", "57950", "24590", "68706", "45349", "34238", "39241", "46264", "40668", "54555", "65253", "65648", "42988", "8794", "29381", "39446", "32157", "50667", "45812", "64434", "22015", "42586", "12633", "53950", "73055", "10225", "56881", "30440", "46150", "52384", "20374", "4256", "67881", "29786", "64090", "69570", "15640", "62856", "71573", "21321", "40857", "40962", "4276", "68190", "29265", "17103", "53977", "37858", "38360", "62851", "62311", "25836", "18724", "37280", "2454", "17012", "69878", "67664", "13166", "4871", "5615", "62363", "2927", "50655", "12470", "13034", "5403", "42580", "55032", "43039", "47775", "32656", "32580", "13989", "12576", "55869", "3441", "20195", "9412", "20010", "12744", "27160", "5453", "6352", "17424", "36608", "10759", "70554", "43842", "56065", "27123", "30423", "6786", "18139", "39875", "70476", "22240", "2683", "69661", "68910", "47727", "31632", "4502", "47459", "1611", "24484", "69814", "10815", "11825", "56795", "9820", "44609", "63052", "34688", "55834", "69701", "21099", "40493", "39977", "71450", "30638", "35434", "21455", "20996", "14106", "23061", "54662", "43485", "16009", "63137", "30744", "10336", "66535", "70929", "25739", "21290", "49000", "71893", "23696", "1295", "5069", "12811", "7292", "45968", "411", "64749", "69582", "18195", "45237", "72428", "63747", "54711", "1450", "1104", "57255", "64417", "61900", "27113", "23885", "12529", "29495", "58811", "17632", "54125", "56377", "45643", "9125", "65864", "44798", "35330", "34745", "19774", "33246", "2785", "71184", "8093", "7164", "60665", "58198", "5720", "48540", "11958", "47104", "65920", "50855", "12176", "31586", "20861", "25683", "54911", "38375", "3279", "42636", "71127", "55331", "24117", "52554", "59884", "41518", "11006", "13230", "66367", "30590", "59783", "5951", "39146", "25215", "45281", "40436", "53681", "15892", "49988", "42717", "10085", "8939", "37523", "41980", "6539", "58062", "19825", "25523", "12178", "61983", "3182", "16620", "48200", "49885", "38451", "15872", "14200", "38339", "58820", "35028", "27609", "10903", "39603", "14328", "6943", "6456", "64890", "35405", "2443", "22998", "69990", "31781", "63467", "65417", "2727", "56029", "69804", "24008", "68842", "51195", "753", "41318", "54049", "71414", "44096", "46670", "54616", "5734", "43799", "23713", "53511", "55598", "47154", "24534", "9291", "2741", "38633", "60131", "34753", "60683", "67274", "18426", "43209", "50559", "50530", "28610", "24839", "4765", "46236", "54569", "32945", "70871", "50637", "3694", "36815", "17871", "9448", "35218", "54250", "54721", "233", "48603", "71784", "53583", "65284", "73167", "18905", "4002", "55190", "2117", "12675", "57931", "68048", "27218", "15185", "55518", "4762", "67026", "10035", "21721", "11313", "32895", "52273", "59999", "57428", "57216", "57973", "31487", "4339", "71645", "26966", "43990", "39684", "47600", "65443", "72544", "33707", "66042", "31991", "70336", "37648", "63741", "48107", "26654", "68666", "3631", "54189", "33065", "52082", "27769", "20724", "40247", "35227", "24136", "41237", "40657", "31706", "5211", "7721", "6086", "44615", "14545", "30898", "59122", "23823", "42535", "38419", "63676", "6361", "44092", "61902", "20371", "25138", "34591", "59699", "21994", "17396", "12552", "30890", "43074", "7683", "56611", "62693", "4463", "37545", "43073", "38595", "64958", "24097", "38473", "1887", "57786", "59394", "70710", "59227", "32890", "22471", "10629", "22400", "43603", "51026", "6182", "50654", "340", "3710", "50664", "65100", "34942", "24962", "56431", "19880", "8813", "23278", "21736", "16858", "29608", "50232", "3676", "70461", "68695", "63374", "4248", "22035", "48332", "24487", "36062", "27146", "44537", "72085", "70545", "60539", "1327", "9588", "47779", "66762", "65054", "21418", "30559", "45271", "28821", "51738", "60910", "31066", "30256", "2191", "9628", "72988", "66571", "36573", "5551", "54318", "55243", "52527", "11146", "37939", "49054", "67217", "56071", "72505", "59101", "72913", "12272", "40676", "36103", "37628", "65521", "35840", "61520", "34213", "5568", "29419", "52654", "63914", "70918", "53456", "9889", "55944", "55429", "19408", "17248", "29844", "2413", "31891", "60961", "47704", "72039", "61291", "26429", "2757", "4561", "59891", "35881", "70594", "70696", "62819", "17051", "18805", "13068", "57462", "72295", "72361", "44330", "34203", "51955", "5073", "10360", "34863", "7875", "25078", "32752", "33817", "44413", "49691", "9422", "17191", "15766", "1715", "70287", "26495", "47891", "32414", "65333", "5550", "41571", "55413", "23003", "28450", "14933", "73174", "34081", "62631", "34202", "22843", "38872", "59846", "32193", "34254", "7085", "43529", "29880", "8733", "20277", "46269", "41186", "64928", "29148", "34824", "19309", "34104", "11207", "69454", "57189", "59049", "35636", "9022", "1612", "34028", "39713", "66775", "69005", "42812", "66826", "510", "9813", "17978", "28529", "38440", "63103", "45450", "3494", "10819", "72104", "25445", "22401", "16462", "7948", "45718", "9195", "10260", "68096", "48436", "50292", "44125", "7", "38875", "48916", "42599", "54573", "72153", "5436", "63812", "32783", "64818", "51716", "67523", "5045", "9845", "60071", "44392", "72031", "41765", "69182", "13452", "41658", "49376", "11181", "20817", "4216", "22185", "25935", "35775", "34294", "746", "62166", "57877", "32618", "35297", "34905", "17097", "55763", "29402", "71522", "2897", "56725", "35068", "36741", "52309", "13218", "46887", "47198", "69319", "38317", "67148", "29399", "50453", "8025", "25135", "35872", "57802", "68927", "46771", "65216", "42925", "63688", "58371", "42295", "41065", "33008", "49042", "67289", "33071", "71484", "31247", "50983", "50398", "51828", "25841", "10593", "18826", "9157", "23375", "69590", "33978", "35959", "21279", "34172", "22383", "29394", "21569", "57272", "53630", "66485", "69796", "55878", "14692", "56395", "49106", "19152", "24322", "22134", "56751", "30406", "30458", "1487", "36407", "15823", "24148", "16984", "50283", "49740", "12190", "68541", "27347", "32842", "46346", "19120", "64820", "26446", "39463", "53497", "57956", "42764", "25427", "59739", "6457", "35416", "57243", "32893", "68407", "5042", "859", "44774", "52080", "40397", "23787", "1790", "49870", "16101", "49600", "25592", "38888", "7697", "60474", "31469", "59365", "40302", "17052", "6976", "64980", "52956", "32566", "43004", "6428", "26451", "35792", "30552", "24496", "51879", "5358", "22120", "60522", "17519", "1583", "65506", "1895", "72853", "70297", "46326", "43227", "28425", "13427", "23260", "31157", "5963", "66633", "58798", "14254", "35874", "29553", "69378", "57178", "16291", "20401", "26302", "68152", "24185", "15453", "581", "53676", "56501", "19696", "33729", "47683", "53124", "52909", "72098", "32255", "64720", "2163", "35516", "52765", "42132", "5479", "67585", "59579", "10566", "53285", "26885", "10809", "61028", "15953", "43625", "17589", "1120", "46716", "18312", "28084", "10122", "51617", "29504", "41069", "63097", "55183", "60939", "731", "7571", "48965", "38157", "55841", "207", "6163", "66953", "23700", "7849", "18443", "39041", "3725", "59067", "27563", "7608", "52363", "50430", "47192", "36197", "14600", "20138", "63657", "7148", "9670", "4046", "23077", "58350", "49886", "23758", "51259", "2797", "23385", "65787", "48604", "58542", "54037", "6240", "43707", "29486", "23203", "37012", "36226", "16507", "68089", "32404", "49473", "66048", "34006", "17335", "58478", "24643", "56252", "31721", "45072", "68552", "32922", "7123", "12001", "18269", "10709", "36375", "16414", "6496", "49775", "12284", "7113", "56216", "64718", "62333", "42276", "54334", "41015", "1993", "52088", "59914", "38768", "50278", "73232", "12043", "5527", "28627", "7041", "6857", "31314", "35762", "22222", "36710", "5018", "52451", "48506", "45115", "69857", "44429", "30999", "41095", "49298", "70154", "34387", "63801", "38697", "20425", "59469", "14002", "12337", "69828", "21510", "12941", "30361", "71355", "26032", "56897", "21179", "68212", "12680", "5825", "58718", "73109", "44485", "43538", "62214", "40905", "48767", "24061", "5242", "23381", "38372", "32223", "27416", "46610", "65289", "49680", "50350", "59629", "59066", "54927", "42068", "60845", "37378", "8718", "71001", "61330", "34377", "2371", "35948", "64234", "50443", "7980", "44670", "3212", "54642", "33619", "54182", "64014", "9409", "67565", "13865", "72277", "40181", "35132", "62229", "61752", "70670", "28753", "27715", "1068", "23584", "73064", "22885", "71049", "28673", "7455", "69469", "24966", "62192", "55671", "979", "27578", "33932", "60888", "55008", "53186", "65459", "55774", "9015", "37716", "50190", "11442", "25129", "72712", "53133", "29468", "49562", "42073", "40879", "42976", "60380", "49455", "1143", "66942", "33649", "50597", "6693", "47500", "18731", "46251", "41500", "38737", "61565", "7389", "7938", "20225", "53464", "33681", "738", "35204", "7074", "59895", "14042", "36853", "950", "8320", "15421", "67578", "11512", "43368", "33432", "19362", "21710", "67951", "65706", "6214", "13176", "69339", "50258", "47609", "7476", "33591", "64728", "72182", "48060", "33349", "54259", "12991", "51686", "42813", "42696", "62750", "44366", "58176", "64982", "30653", "20280", "65649", "29960", "28892", "36729", "13842", "31928", "47351", "60673", "44686", "40952", "33231", "49255", "6544", "18540", "34449", "47613", "22752", "9098", "60379", "49045", "57996", "31564", "56067", "35234", "8612", "13926", "20882", "16314", "26993", "23010", "59192", "1282", "50022", "58097", "43547", "30002", "35251", "7003", "34964", "2333", "70925", "32113", "67324", "39107", "29987", "9934", "69406", "17772", "65168", "18440", "52333", "20063", "37026", "26670", "36158", "21743", "41544", "41488", "51136", "57857", "28048", "49378", "56264", "25722", "12892", "46822", "26310", "40438", "20667", "59280", "64359", "3167", "60728", "59311", "49635", "13853", "35707", "36099", "42564", "47118", "67448", "57704", "58607", "60716", "65454", "38087", "41340", "13152", "833", "25182", "67282", "46620", "70550", "18149", "34996", "23634", "18308", "16405", "53205", "16020", "66247", "27100", "9947", "47172", "9611", "63856", "44937", "71640", "55641", "30061", "20934", "29847", "8043", "14571", "8740", "20810", "44288", "3472", "39628", "2632", "11696", "69964", "12539", "25022", "58503", "35648", "47770", "62276", "22314", "7797", "4236", "46194", "13346", "64252", "49699", "18915", "62738", "62280", "11697", "39312", "7351", "62790", "31323", "22536", "36967", "66056", "1096", "53841", "4836", "66831", "54447", "21554", "3018", "22713", "65409", "43277", "60563", "3368", "26386", "66363", "58409", "21981", "45131", "39290", "30570", "14916", "47456", "67900", "66257", "16949", "32562", "40255", "24834", "6139", "26460", "73112", "17203", "44236", "49373", "28847", "67451", "51723", "19747", "6317", "57892", "48974", "61260", "69464", "10887", "44513", "4170", "56760", "4958", "16751", "15676", "29133", "50361", "58539", "37113", "68093", "57310", "57415", "58860", "59959", "61460", "13618", "38736", "25121", "64213", "28777", "31907", "35744", "23416", "16884", "52270", "59833", "38516", "26638", "57421", "49848", "72614", "72147", "40369", "19293", "34674", "14208", "65502", "1369", "60073", "774", "64635", "42010", "30320", "71005", "23317", "43510", "2177", "23004", "52356", "15876", "64682", "37857", "31425", "16675", "17408", "55241", "10954", "10143", "52972", "31746", "31260", "16046", "57385", "35207", "136", "44388", "27552", "53959", "67252", "5505", "45559", "2710", "64826", "51860", "22242", "69027", "54839", "3468", "10840", "12753", "44320", "34907", "27061", "64362", "48149", "51882", "59670", "7126", "39346", "49881", "21559", "25505", "63408", "11971", "65003", "66770", "60202", "39112", "30506", "32016", "58915", "59426", "21848", "20803", "64987", "64851", "64730", "25562", "63618", "17649", "52109", "26742", "67772", "37930", "22906", "35625", "23013", "47288", "48934", "44203", "24137", "72401", "16937", "62219", "55968", "64271", "12101", "9645", "3728", "67835", "28872", "57646", "37931", "22634", "26732", "32424", "46837", "33851", "17176", "9608", "22376", "20104", "12523", "1596", "51798", "61346", "6556", "39283", "71713", "72139", "69707", "41730", "54723", "62365", "41557", "48543", "13909", "57944", "71187", "60801", "21625", "37810", "1530", "26603", "1800", "42227", "18807", "62499", "4230", "64668", "14274", "25695", "31156", "19515", "66651", "53776", "68571", "48152", "66306", "62195", "42322", "68913", "31501", "6906", "24024", "50704", "69524", "63787", "51013", "21414", "7795", "33714", "8744", "15904", "57015", "56832", "22473", "7419", "9471", "60368", "10181", "35627", "6005", "11596", "21169", "21332", "63559", "48665", "10069", "59506", "43798", "54536", "70314", "42801", "12269", "27498", "57449", "38880", "43419", "10806", "42410", "27210", "45558", "88", "58395", "40032", "49227", "60497", "3287", "43475", "44016", "62502", "64158", "49515", "71725", "27813", "43709", "24921", "63323", "3605", "6272", "16900", "17612", "8308", "30029", "43703", "23936", "11490", "52668", "115", "27704", "15188", "39173", "14387", "10174", "58483", "32830", "49581", "2956", "17266", "20318", "13250", "22890", "69", "68827", "14126", "37898", "138", "39482", "36467", "17551", "38640", "55497", "30391", "14845", "28318", "70035", "28436", "42274", "42694", "18696", "32319", "11414", "43201", "71494", "16917", "47883", "44806", "16350", "10943", "15449", "68854", "67031", "2583", "62464", "47820", "45940", "61261", "38761", "69171", "50899", "59952", "60119", "21303", "4992", "24518", "41485", "15052", "68964", "68054", "21552", "57165", "41835", "43613", "40382", "44939", "46048", "24701", "71789", "65911", "8955", "26572", "19327", "72795", "20878", "71795", "19735", "7343", "15592", "44764", "63263", "29677", "56936", "24246", "34438", "30760", "52658", "70416", "20880", "10208", "49578", "25995", "50859", "30978", "35238", "18910", "36655", "15618", "17604", "34557", "53904", "39827", "64233", "15153", "31757", "35060", "66272", "28660", "13508", "57224", "46109", "27046", "64364", "10177", "67496", "38843", "12098", "54725", "25023", "52870", "68288", "69094", "63321", "8984", "49188", "68317", "60965", "70076", "47435", "34311", "19928", "41188", "26521", "25820", "33420", "45903", "64898", "36966", "50140", "32578", "27237", "33274", "5114", "45017", "15713", "46430", "46522", "14984", "39695", "60004", "70769", "68067", "30414", "23376", "45885", "16590", "2005", "25086", "33926", "41844", "39555", "63239", "29544", "40934", "29279", "3240", "24616", "66311", "48707", "46228", "5937", "50907", "73128", "1231", "17870", "14165", "41254", "11714", "19277", "2915", "53102", "43517", "49565", "26563", "38643", "55496", "42495", "65832", "67973", "2654", "39259", "7042", "52535", "46951", "54368", "35499", "58005", "67393", "5123", "25220", "24383", "17658", "42252", "18891", "50228", "54903", "38685", "11608", "58760", "28485", "37035", "15103", "1808", "13083", "42371", "58228", "65096", "73105", "43133", "32678", "38642", "59258", "11914", "63928", "51201", "62863", "36634", "24083", "63545", "31792", "27024", "19621", "58090", "25962", "40210", "7965", "37258", "7186", "26108", "61642", "66640", "71352", "31842", "31050", "13957", "37363", "71552", "36225", "57166", "9133", "61368", "46193", "39664", "8559", "2723", "11772", "10522", "13409", "3458", "53312", "62471", "51846", "5142", "5977", "23590", "11694", "63586", "38997", "482", "60278", "71990", "52884", "73152", "31454", "13441", "64693", "69392", "21523", "49002", "45094", "63721", "11548", "52648", "23178", "61360", "6161", "67681", "6484", "36038", "60901", "66029", "52042", "49182", "26347", "72733", "26745", "10751", "48721", "35164", "3191", "63197", "53617", "32988", "53153", "1862", "35281", "35025", "42978", "54008", "21187", "40474", "26952", "9540", "69856", "37998", "59254", "50787", "6955", "33320", "3995", "2626", "3759", "30767", "50810", "61419", "44915", "17048", "69042", "17356", "2977", "72706", "54866", "61068", "67891", "44576", "60771", "64478", "21364", "28037", "64683", "61187", "32202", "21450", "67336", "9192", "14194", "1658", "14339", "2231", "20526", "21097", "5589", "28614", "72841", "102", "67549", "41774", "20567", "69912", "26063", "49519", "17582", "23652", "67714", "12017", "9907", "12687", "3978", "17467", "44984", "9083", "45584", "2846", "59599", "70004", "19646", "39230", "72309", "53709", "4158", "66023", "54082", "47747", "70308", "10045", "33877", "60115", "55325", "9861", "20244", "26364", "60008", "17213", "17164", "15528", "16996", "25140", "42711", "11248", "17070", "37145", "7376", "68154", "37818", "11296", "248", "43372", "34568", "38446", "60636", "8248", "62746", "2594", "12901", "66113", "48544", "37606", "14723", "16774", "15569", "58666", "17092", "33294", "35107", "34593", "15621", "47501", "41085", "36406", "58611", "63300", "36935", "66305", "61416", "61845", "14229", "23284", "67902", "54502", "30034", "42115", "47112", "6968", "63874", "29875", "12352", "21611", "20889", "56815", "26237", "57588", "72116", "60787", "47938", "25259", "70006", "40575", "48073", "44465", "47913", "17382", "17110", "9246", "61919", "23715", "20618", "29241", "18623", "42342", "56574", "64216", "7298", "1614", "20546", "66343", "27476", "40647", "38525", "38476", "27624", "67081", "32126", "1264", "55353", "5698", "58463", "35036", "10303", "70698", "19041", "48162", "31520", "22399", "14976", "30469", "11152", "25158", "71978", "2351", "62020", "40423", "12162", "47520", "21225", "35057", "6010", "47668", "57912", "25658", "65240", "66927", "57706", "65953", "69784", "15983", "58735", "71831", "70823", "23429", "7232", "56285", "49018", "36059", "70832", "50395", "48585", "13321", "44030", "30527", "60899", "33170", "18645", "18846", "20834", "56403", "23724", "4755", "8159", "21385", "1785", "72775", "21948", "26008", "68320", "34429", "36901", "67705", "41675", "33362", "52047", "30205", "54955", "39172", "13530", "46803", "49308", "23116", "51540", "64829", "10374", "25390", "17760", "55310", "26294", "45028", "48216", "10939", "57414", "55036", "35968", "49534", "10649", "45753", "2526", "21688", "9624", "71687", "36902", "41092", "8962", "31253", "10105", "37271", "62651", "42106", "2894", "36870", "44664", "14340", "27012", "41586", "61491", "29622", "47359", "51434", "60232", "16501", "8721", "47352", "43390", "12198", "33750", "72597", "23434", "45165", "26212", "44745", "14880", "30415", "54727", "35267", "15388", "60764", "44221", "54473", "72905", "50817", "36402", "17806", "3034", "2318", "72968", "7915", "59559", "36711", "34166", "23083", "41406", "60254", "43921", "34879", "42710", "45628", "35604", "28148", "60619", "12509", "34413", "48255", "34813", "25900", "71376", "53288", "6905", "33235", "71886", "64681", "16513", "55226", "62966", "58487", "2173", "5567", "16103", "47102", "67035", "29516", "24576", "17237", "31850", "71541", "12840", "10128", "29872", "45124", "59082", "1741", "50355", "14496", "62520", "56654", "38384", "15887", "60650", "37023", "23934", "31112", "1245", "16473", "22343", "49824", "67099", "9916", "69240", "19292", "56610", "31439", "10962", "18814", "63455", "12163", "58659", "51949", "68974", "38296", "41666", "10049", "55649", "62955", "33641", "41203", "47284", "55746", "26702", "64580", "63091", "40150", "56980", "12308", "58986", "11025", "34537", "8389", "34249", "24140", "50038", "31534", "10114", "12412", "22681", "71697", "61119", "14462", "28135", "51950", "72461", "69520", "28511", "38982", "54489", "18273", "14365", "55822", "46415", "18784", "31854", "516", "53258", "58906", "31986", "63996", "6343", "3912", "6770", "35119", "19591", "27157", "17721", "53331", "69672", "63320", "22907", "31167", "40098", "28240", "50733", "56424", "52228", "38644", "48363", "4032", "22725", "39266", "59146", "48906", "61694", "36242", "3190", "27698", "34966", "11585", "47490", "70689", "58703", "27610", "5343", "65353", "26544", "38110", "38012", "46040", "31717", "73101", "65954", "9233", "11402", "52227", "14840", "31373", "45842", "64975", "41521", "19698", "42200", "67563", "54649", "37131", "46611", "4100", "42555", "4585", "29850", "33269", "68254", "31040", "57621", "2585", "58778", "12639", "30857", "41995", "57313", "2940", "9134", "21", "52795", "45077", "36330", "57154", "40923", "71252", "72398", "2042", "67139", "50783", "30076", "20582", "70514", "15902", "29009", "59118", "54550", "53679", "11323", "39177", "70671", "18017", "20786", "38293", "67846", "59714", "31826", "53935", "13952", "43169", "13912", "15671", "35661", "38492", "22232", "24311", "50116", "55289", "62535", "48011", "29334", "1678", "73126", "63028", "55077", "43307", "62858", "18450", "34656", "13401", "64088", "68265", "17153", "38993", "12232", "62225", "9082", "21875", "13814", "55842", "17079", "52191", "8919", "51304", "4000", "52105", "21050", "17481", "49887", "70791", "62628", "64613", "70319", "6823", "30442", "64125", "40199", "22048", "49067", "40076", "69753", "10274", "38239", "11300", "2848", "10487", "62699", "67924", "12433", "41519", "53915", "19576", "4490", "49724", "69130", "49156", "7182", "39321", "47296", "47818", "12197", "3229", "9081", "41619", "41727", "35331", "15539", "29609", "7616", "49856", "46761", "28369", "65109", "21983", "8218", "13786", "7378", "44182", "25409", "73039", "33643", "35515", "41045", "15094", "46469", "47274", "48690", "50847", "41899", "59580", "4225", "40244", "41419", "11050", "36069", "58755", "7218", "52230", "14596", "65218", "64177", "34362", "24820", "10739", "24485", "54403", "47624", "46866", "39339", "4406", "66720", "36369", "50070", "10749", "61417", "6435", "23588", "70041", "17310", "70273", "8280", "42864", "34838", "47309", "21091", "51453", "42549", "71242", "51292", "30085", "67386", "5041", "62705", "6863", "64546", "32079", "6920", "58278", "43010", "27267", "2820", "10407", "37321", "27273", "11046", "19135", "53240", "2787", "22531", "27425", "46413", "24910", "28032", "63395", "42018", "21613", "33225", "7766", "29052", "5540", "8071", "2552", "68098", "14856", "50085", "38070", "69873", "10826", "30454", "19741", "39667", "65141", "23957", "61358", "48952", "48053", "56160", "44462", "48717", "16676", "43691", "38457", "29882", "55609", "59325", "58741", "23846", "55430", "62122", "13408", "70439", "46343", "72194", "60107", "11010", "56383", "22795", "3415", "41253", "34161", "25500", "22312", "39332", "22566", "52810", "27394", "669", "7338", "9479", "39279", "13700", "49735", "56497", "70628", "72146", "23777", "17803", "45672", "62294", "2667", "35333", "56055", "52936", "30583", "26509", "3050", "3210", "64657", "29649", "46204", "29569", "31007", "69056", "36333", "19511", "69415", "62115", "53028", "40230", "24433", "40136", "56951", "18042", "7143", "54590", "56094", "22106", "13505", "56855", "23379", "23498", "49973", "18274", "36557", "72066", "28360", "22329", "38158", "16968", "62198", "47523", "13788", "57237", "19185", "57508", "6487", "26324", "57024", "68182", "26875", "37342", "34018", "4592", "4282", "3403", "57848", "16775", "23530", "48701", "34934", "48550", "47857", "43183", "35332", "56265", "30498", "37643", "39089", "51699", "33399", "61860", "33061", "12053", "25331", "39513", "34548", "5607", "27693", "60404", "17246", "54441", "49808", "50571", "54621", "54193", "61548", "72638", "19514", "36725", "35726", "3070", "49385", "32429", "54032", "32826", "23996", "13119", "61637", "24765", "36100", "21458", "27375", "52360", "68558", "27263", "40396", "3219", "63195", "49048", "18935", "27742", "22370", "6686", "23215", "32933", "14312", "57200", "43380", "11316", "37920", "57353", "51913", "2701", "1041", "1959", "63253", "12826", "12318", "54672", "16682", "19094", "62282", "57260", "236", "57676", "25288", "50782", "37236", "7282", "48886", "40640", "27941", "27406", "41462", "21808", "41461", "13045", "16179", "295", "65796", "54033", "52919", "39740", "60310", "46818", "21683", "8674", "5892", "30906", "12486", "19470", "50567", "55622", "16186", "55355", "1520", "680", "24089", "60226", "30037", "58486", "20314", "11771", "14089", "29065", "33699", "34807", "65036", "72107", "31095", "25957", "19795", "49428", "23372", "20232", "544", "55603", "2314", "7079", "31999", "10641", "24991", "64719", "68838", "17345", "59558", "68300", "55851", "65678", "72065", "64908", "25619", "15593", "53448", "68123", "72960", "8725", "54104", "21819", "12855", "72888", "5290", "36434", "32597", "8564", "67703", "29266", "14878", "21478", "11273", "70200", "65933", "24272", "22194", "64604", "31724", "9601", "45873", "56147", "12398", "9754", "25460", "67828", "57972", "54835", "62328", "59624", "31494", "56712", "23339", "46157", "63439", "1226", "56892", "71055", "54396", "70745", "28371", "799", "6081", "70271", "6688", "61246", "54010", "19499", "60321", "46851", "36452", "12646", "54272", "33149", "42063", "34192", "18836", "69836", "39579", "61023", "70343", "11032", "16025", "50734", "11514", "12764", "15987", "62796", "53614", "57464", "40482", "59243", "53903", "46997", "10498", "68183", "67447", "58036", "18678", "51936", "50693", "1057", "5691", "54625", "7598", "55838", "59839", "71476", "27636", "16217", "33741", "33546", "46391", "45004", "57624", "847", "42384", "71118", "42981", "2829", "64138", "33124", "59164", "21622", "50318", "39855", "5223", "31679", "4662", "56965", "41567", "36423", "53748", "48822", "54264", "35717", "28155", "7440", "12325", "45875", "56801", "1151", "11579", "4722", "44478", "47505", "12844", "51500", "5324", "48838", "70082", "10988", "72389", "22176", "37063", "17547", "70169", "38404", "5291", "9753", "72283", "32604", "56730", "49326", "2599", "35423", "26013", "25164", "27977", "49189", "5464", "25512", "51089", "37887", "29788", "36424", "66193", "48864", "71351", "287", "72980", "41564", "72136", "36681", "11344", "36078", "28362", "67354", "23106", "60692", "13593", "21190", "49892", "57655", "29136", "15335", "61639", "25652", "1548", "55515", "42529", "63153", "49587", "12049", "31988", "36450", "9160", "46684", "46680", "45566", "37908", "11027", "60441", "5533", "40428", "17962", "16703", "20563", "54841", "44992", "12551", "18868", "4626", "20337", "42855", "70433", "10231", "8141", "46390", "19786", "58771", "51404", "21972", "1946", "11426", "41743", "63773", "69099", "70005", "24150", "69631", "13115", "2612", "56277", "63864", "72412", "68361", "62748", "22555", "14811", "5823", "55078", "24560", "42118", "59962", "61410", "11385", "8444", "25805", "19036", "70891", "20223", "13921", "56374", "10803", "60230", "12491", "27754", "60096", "21153", "20572", "61340", "43115", "20064", "69792", "44953", "2618", "31783", "1879", "37752", "66677", "31809", "20217", "5415", "33318", "69607", "50412", "62623", "3919", "36419", "45082", "44703", "11312", "2309", "70896", "17131", "68385", "66509", "42480", "35407", "21069", "8451", "32388", "50876", "13278", "2761", "66161", "23449", "3884", "36836", "30684", "27041", "16420", "37117", "71265", "71250", "21268", "66666", "43384", "12246", "57149", "62031", "16810", "47329", "15221", "64118", "52775", "62184", "37165", "16258", "1315", "42539", "6046", "808", "8534", "46554", "53246", "21835", "33820", "28389", "65352", "18508", "12658", "1162", "79", "20649", "35187", "49569", "34134", "70159", "47953", "55732", "70820", "15387", "60322", "65460", "47507", "59464", "53352", "70808", "65729", "37819", "16158", "50202", "40544", "41540", "50303", "28520", "11928", "19910", "32699", "56763", "17707", "44364", "66515", "34342", "24585", "58641", "5481", "20720", "15631", "64819", "42490", "17781", "68124", "11636", "8483", "41731", "35656", "31200", "21132", "7813", "57504", "44385", "12700", "11337", "6256", "20394", "22443", "41055", "9987", "32242", "19238", "17778", "12757", "13841", "19642", "48388", "49447", "42623", "51144", "29498", "5222", "40974", "6581", "45206", "33602", "72013", "57720", "26400", "41162", "38166", "61274", "681", "14117", "61009", "46829", "45744", "50908", "21746", "43345", "5808", "9609", "13646", "32929", "14565", "8186", "35446", "50428", "5461", "7160", "1641", "50674", "40617", "7029", "40832", "31135", "37514", "66414", "50628", "6879", "1618", "62928", "10216", "58237", "15556", "46270", "42818", "1746", "30121", "22031", "56695", "43728", "35626", "42525", "27911", "61484", "24288", "21967", "48566", "849", "21786", "53674", "108", "31565", "56791", "37632", "49397", "26834", "68828", "46742", "20627", "19911", "63757", "1997", "5141", "48734", "18643", "8246", "70065", "14898", "70435", "54813", "32513", "31528", "56532", "7579", "62940", "22578", "60882", "16048", "49332", "71855", "69168", "36750", "655", "52600", "44702", "12790", "13896", "59134", "9903", "59203", "65782", "23517", "2681", "39842", "636", "1677", "1077", "66479", "65671", "20025", "26243", "25401", "56262", "19999", "30250", "54830", "18800", "55117", "15249", "14132", "47225", "30987", "43757", "66949", "9480", "46058", "11845", "56454", "71776", "3410", "41966", "36542", "23697", "26340", "56792", "50649", "54232", "2928", "50658", "41104", "20297", "53763", "21865", "55986", "8103", "58271", "52386", "48384", "19180", "34099", "53795", "59346", "29500", "5860", "13963", "49019", "70157", "49503", "11124", "9286", "54025", "25403", "54053", "18181", "59387", "44820", "29548", "56365", "33243", "58783", "16511", "13324", "3055", "10414", "59377", "63064", "60352", "72174", "33485", "41431", "15074", "37638", "64867", "11814", "39925", "43100", "15455", "30247", "58744", "12157", "35784", "51894", "26765", "44951", "48164", "2871", "70807", "20367", "29057", "46428", "71986", "54678", "71223", "35439", "25541", "44923", "62558", "57840", "31139", "26273", "6197", "35505", "25498", "39991", "33205", "20100", "491", "54504", "23089", "27323", "19681", "22023", "69314", "13273", "45465", "22505", "20751", "64470", "62081", "39066", "18644", "30921", "60520", "72580", "50378", "6859", "10432", "41566", "19849", "29823", "61374", "41009", "55158", "18939", "34648", "10528", "55305", "67652", "43643", "50755", "45919", "23899", "24905", "43279", "39753", "57740", "39319", "19618", "47774", "32735", "13754", "45790", "2021", "38663", "64251", "56096", "69730", "53225", "46381", "8259", "59375", "63909", "10380", "39750", "26091", "62449", "66500", "13457", "12059", "4980", "28390", "59974", "66439", "67133", "12465", "69868", "3424", "31226", "22935", "34024", "65106", "5960", "69957", "24387", "43915", "44722", "61682", "29410", "17526", "32100", "3289", "57725", "21343", "58045", "45869", "19872", "42279", "38887", "67426", "30483", "11573", "18852", "37440", "19084", "31127", "68951", "6266", "32924", "34708", "42838", "59236", "52877", "59682", "20502", "66164", "26005", "26524", "11803", "3855", "60679", "61365", "40485", "39599", "33539", "67686", "52904", "8013", "1218", "51715", "40078", "8234", "58699", "24511", "44020", "919", "35365", "47264", "65411", "22345", "58393", "30264", "13140", "33250", "67383", "29353", "44786", "64884", "8600", "58535", "39583", "21145", "25359", "51284", "4636", "44353", "15890", "59503", "33439", "3944", "67803", "9567", "14844", "15486", "22655", "40301", "58843", "14107", "42547", "40422", "28648", "1577", "28968", "209", "30846", "31177", "41887", "22580", "64033", "39805", "43840", "70773", "13882", "36397", "36216", "40003", "44603", "43900", "59435", "33577", "50608", "825", "32077", "7711", "43433", "65279", "21078", "46690", "53132", "62895", "4333", "66632", "31921", "40878", "4653", "31465", "27149", "68994", "38701", "29583", "25412", "23458", "9179", "52108", "63201", "41074", "62212", "25826", "8374", "28855", "56062", "68644", "8912", "60003", "16932", "71433", "40129", "67206", "21874", "37557", "47216", "42293", "43624", "5874", "52942", "69190", "47961", "33466", "1366", "14607", "29306", "28027", "8065", "53467", "50131", "2988", "8169", "10227", "14306", "5664", "853", "36005", "63221", "9570", "27557", "50386", "62942", "58864", "2100", "2364", "71852", "57105", "37922", "40663", "45238", "55689", "63534", "53351", "67603", "54541", "59310", "45950", "9850", "29620", "32037", "66922", "7815", "35690", "31831", "69084", "69773", "53468", "57179", "21151", "8552", "16813", "53944", "31307", "39697", "11139", "58107", "56345", "3513", "37660", "30375", "23487", "61867", "35329", "904", "1254", "32208", "11559", "8335", "22897", "39652", "50749", "38567", "27421", "5938", "15261", "16325", "44741", "13129", "30086", "26277", "56057", "27250", "56646", "71420", "30652", "39124", "3244", "48919", "22836", "11589", "30562", "9337", "10501", "22666", "44136", "44142", "69923", "24408", "25218", "42021", "5038", "28763", "67554", "3156", "13536", "48431", "30431", "30474", "72502", "11755", "30408", "49003", "70186", "20457", "5140", "18901", "57068", "36602", "28464", "72325", "64978", "20945", "757", "18086", "4617", "24373", "49805", "30208", "18029", "47926", "16786", "9802", "34101", "3428", "20008", "32793", "1676", "31075", "32218", "38233", "34284", "45336", "42800", "57362", "50750", "70563", "13376", "46261", "18683", "46833", "66903", "3957", "57270", "64511", "10867", "70855", "60913", "59585", "46088", "33908", "6168", "62064", "63152", "46334", "5576", "27249", "21056", "39558", "10010", "40502", "72668", "6675", "71672", "64674", "65741", "52724", "39145", "23596", "20729", "48856", "41649", "11515", "49105", "54795", "26496", "42111", "29422", "49094", "44735", "1044", "28632", "45242", "34759", "48895", "8274", "13797", "17609", "5186", "43579", "37852", "58512", "9781", "62937", "69711", "41062", "10688", "61980", "22715", "1343", "9402", "38251", "33844", "1486", "53033", "52906", "21860", "48573", "23937", "38666", "65657", "29332", "52826", "30917", "34301", "8644", "2939", "18993", "68391", "17711", "61217", "27652", "15302", "51883", "51583", "15707", "13668", "611", "68839", "71234", "11382", "25208", "10645", "62423", "32048", "55394", "56536", "7177", "16137", "49924", "10967", "60857", "37412", "59878", "23836", "21620", "47848", "67723", "27222", "35097", "23862", "36147", "61899", "21282", "15845", "23217", "8672", "41029", "23930", "18847", "17003", "17769", "7478", "203", "60585", "69938", "25344", "39739", "36476", "25454", "43204", "50809", "29325", "44170", "40886", "66049", "26547", "7238", "7280", "20156", "24961", "18287", "64019", "32943", "48844", "13360", "6245", "7600", "59109", "73217", "34385", "33037", "72473", "7648", "12093", "22627", "42353", "52661", "71752", "13270", "3352", "15523", "63683", "56734", "63068", "63798", "39744", "48846", "35193", "1197", "27000", "46532", "11463", "47963", "13393", "30218", "57376", "32998", "48412", "44974", "44074", "41612", "50647", "42315", "24539", "72360", "45273", "16837", "19933", "64270", "61539", "27337", "9350", "56110", "36587", "73032", "58969", "45564", "52339", "60726", "36488", "50920", "14409", "67642", "23974", "64363", "47113", "30672", "22135", "52293", "46796", "72356", "37126", "68634", "64124", "66503", "28754", "26478", "37813", "70322", "50892", "12184", "29607", "53131", "51915", "6126", "42310", "35924", "22979", "34023", "6019", "72934", "41515", "69655", "9266", "30099", "61326", "5122", "65952", "40994", "32045", "37451", "72041", "36837", "25267", "47153", "63783", "32537", "19538", "51148", "52820", "67056", "29600", "45515", "9339", "56064", "2085", "30276", "23915", "31363", "3609", "67879", "62339", "29615", "32908", "25059", "40507", "48118", "71000", "5361", "31450", "17128", "48522", "66723", "11195", "53424", "68512", "37202", "58976", "73073", "6881", "22556", "33708", "6193", "50742", "6368", "17034", "58658", "1482", "28617", "8177", "34719", "3161", "14870", "4095", "41548", "42341", "70445", "5053", "38668", "71116", "51764", "15600", "1965", "3705", "19822", "6294", "38144", "16670", "19677", "8842", "28931", "19518", "66502", "58118", "24919", "52057", "22286", "49901", "31570", "49022", "35927", "6058", "45685", "64063", "36877", "40804", "30513", "21205", "55793", "18128", "18969", "23035", "61377", "28011", "67029", "54606", "43376", "43225", "11702", "6776", "11260", "25779", "31777", "53072", "32533", "9075", "26666", "56978", "19506", "861", "63937", "61751", "21058", "2327", "44204", "67993", "48652", "3726", "31142", "2771", "57321", "58593", "65190", "35941", "42953", "19680", "41427", "69519", "19984", "69532", "23118", "67510", "14917", "18907", "13404", "47291", "5157", "11793", "18585", "40323", "5341", "50280", "63220", "38817", "68891", "6599", "51545", "48289", "55966", "49163", "8507", "65675", "19850", "65066", "38946", "3823", "7393", "60446", "16008", "63037", "23602", "46664", "33012", "12569", "42579", "58709", "2576", "72786", "4317", "58651", "49353", "22199", "53020", "24718", "6559", "25653", "25932", "9384", "10994", "57463", "6772", "55821", "53831", "29085", "45113", "45857", "51499", "55804", "54933", "38478", "46121", "50484", "23981", "606", "60955", "56817", "26508", "43997", "34552", "11501", "40204", "15376", "35643", "65655", "57551", "22678", "39745", "22322", "38785", "43048", "42810", "34002", "2737", "31873", "4694", "16792", "26461", "40224", "17301", "55293", "70920", "18730", "4904", "23832", "67158", "66277", "47975", "416", "60767", "22052", "30696", "39106", "20806", "17080", "26604", "39528", "51260", "43193", "52583", "13691", "42958", "15949", "33915", "1542", "29586", "27877", "45678", "52789", "63105", "69600", "56624", "34436", "70191", "12729", "17964", "69565", "67498", "35109", "59764", "35472", "71043", "18506", "34386", "36623", "67175", "51509", "38228", "45760", "42674", "2421", "40057", "58074", "26505", "19026", "56257", "28791", "47165", "46609", "42234", "61130", "50117", "13561", "56518", "15952", "23858", "3862", "70022", "48201", "65538", "40701", "45637", "28448", "29901", "9516", "10935", "3266", "7609", "71354", "61878", "67918", "2428", "70468", "22560", "11649", "19230", "55938", "25490", "62470", "60134", "35754", "35957", "43958", "41035", "41864", "17209", "29832", "2260", "28717", "17162", "29488", "17095", "39238", "13878", "24872", "21627", "4510", "57719", "52578", "63984", "17644", "18500", "26382", "27184", "35146", "47237", "47364", "67690", "40116", "7908", "32781", "58304", "25310", "46458", "34353", "32993", "2261", "13559", "2154", "53891", "65636", "64447", "45666", "49253", "59931", "970", "65415", "50672", "29738", "60492", "48964", "65619", "65600", "61255", "23604", "40024", "67113", "31623", "72563", "31609", "28548", "66466", "50174", "3555", "41530", "42298", "6098", "29511", "10878", "63770", "30737", "27893", "44907", "40885", "63473", "58737", "49243", "55474", "33898", "37543", "11799", "53603", "25304", "70291", "11247", "73193", "18903", "2109", "29434", "14372", "12121", "45976", "41103", "22848", "54353", "3462", "8971", "10468", "73014", "63339", "47889", "12535", "3128", "4999", "72479", "50987", "67387", "35561", "45381", "17899", "58974", "59127", "35826", "2144", "12604", "36105", "70829", "25867", "39681", "54065", "39678", "13611", "18233", "52760", "11203", "15833", "11934", "52372", "27048", "31917", "52378", "29195", "60923", "55777", "48853", "48609", "51526", "71841", "8804", "39802", "35645", "7559", "48875", "4438", "41388", "53199", "51101", "24413", "62484", "26170", "7583", "27529", "23995", "48841", "15717", "54683", "58502", "64807", "58925", "67304", "18972", "20818", "48736", "5180", "34327", "3488", "44155", "32082", "34257", "22029", "14393", "30453", "29408", "55592", "64684", "11101", "65748", "6438", "69385", "9912", "16600", "21488", "45712", "61956", "57537", "46028", "38267", "54679", "17950", "11493", "48322", "49218", "40999", "63291", "899", "16939", "69968", "19046", "65817", "716", "21441", "7153", "27255", "56477", "72966", "52045", "53777", "14209", "8260", "21106", "46417", "40774", "35069", "43627", "18663", "8685", "24669", "61312", "41842", "64947", "64671", "18348", "50692", "33159", "21840", "26996", "50106", "35385", "37230", "12170", "61289", "62012", "31648", "58243", "50358", "4509", "43395", "18892", "72445", "1488", "46475", "46509", "18680", "52017", "67654", "8352", "60027", "17605", "22797", "64737", "1233", "29379", "13732", "43310", "6980", "16827", "5208", "36580", "52513", "58934", "64714", "10955", "39763", "15967", "20245", "14708", "55521", "44268", "31883", "58491", "68607", "22327", "30435", "13749", "54490", "62943", "21525", "53305", "40964", "1835", "38635", "57403", "3346", "56842", "36357", "14527", "53269", "37597", "47245", "53735", "42258", "69494", "64936", "51389", "7590", "1070", "12458", "38106", "40531", "42834", "31661", "70989", "24263", "20187", "28946", "13627", "49244", "59218", "20282", "54060", "18706", "39327", "31642", "39632", "36304", "14321", "58127", "1719", "5071", "2280", "23507", "44108", "37953", "34699", "15609", "40221", "2589", "61986", "83", "14920", "21030", "18636", "36165", "61506", "61104", "15062", "26466", "9605", "10901", "5846", "17643", "52955", "71633", "2781", "41012", "64675", "34349", "71339", "57364", "13207", "40522", "22865", "45895", "1866", "20664", "16638", "4831", "21890", "50003", "8547", "18708", "47059", "52589", "23646", "12276", "43551", "168", "9442", "53906", "52387", "68032", "24975", "62977", "1285", "46474", "13482", "66472", "7513", "2380", "21371", "51424", "28598", "13953", "4211", "65763", "40442", "22301", "20685", "48160", "59250", "33073", "72180", "46493", "71444", "31281", "58154", "36934", "21919", "2277", "36276", "37987", "23587", "52851", "56816", "28930", "48052", "8828", "15337", "66997", "35706", "29908", "30203", "64578", "52538", "66650", "68645", "70362", "14716", "22726", "2777", "42556", "66812", "41982", "69353", "10326", "27592", "59492", "12463", "46263", "4991", "60642", "14025", "23055", "36950", "34758", "19405", "14741", "24994", "55150", "28566", "21111", "5629", "48954", "70419", "4861", "41888", "42781", "30212", "24054", "70706", "18596", "38023", "61446", "43655", "44643", "15245", "18429", "30383", "67107", "50760", "17451", "58320", "7105", "69690", "72241", "14950", "22000", "22430", "1445", "57833", "65702", "52559", "70856", "43470", "1457", "1269", "38855", "42569", "46626", "49040", "35168", "29074", "47966", "68610", "28348", "42706", "21600", "33909", "66586", "66797", "10798", "66730", "13359", "50020", "59361", "13160", "26883", "52478", "32049", "68966", "834", "40601", "64079", "50870", "25302", "55154", "5557", "22499", "21668", "18497", "17312", "34271", "27678", "65807", "28897", "1237", "16485", "49792", "33249", "55114", "26314", "46035", "24235", "71284", "72810", "71417", "12871", "65112", "41465", "59074", "48578", "38121", "48884", "8921", "48083", "36905", "26559", "32273", "29059", "12175", "25473", "38884", "6130", "63802", "1180", "1595", "30477", "67855", "4028", "53605", "43924", "10692", "13075", "39845", "49625", "62590", "5782", "14103", "48793", "58011", "51328", "71403", "10141", "26355", "48473", "66497", "53530", "53778", "51105", "51553", "60678", "10557", "71900", "4535", "57526", "38159", "33155", "45387", "50612", "53476", "26662", "49120", "39874", "68010", "66955", "5065", "21658", "63636", "16519", "24988", "67329", "28911", "7662", "34468", "33913", "18399", "46710", "24893", "34422", "18860", "48393", "30407", "2059", "4711", "20259", "67346", "1215", "18931", "17353", "68538", "12724", "252", "15659", "4241", "58616", "48518", "16545", "8913", "67948", "5827", "49254", "14390", "50509", "41704", "32904", "24076", "52962", "21322", "21484", "67861", "35587", "3124", "11254", "46489", "7226", "64450", "66789", "69867", "25106", "55313", "16540", "21422", "27345", "64663", "50471", "71830", "23662", "5195", "29707", "51619", "48552", "15018", "56773", "34446", "17142", "22682", "29361", "55853", "28867", "6852", "48024", "9207", "956", "65464", "43976", "24902", "36694", "50261", "3087", "40131", "32952", "13917", "41305", "39607", "32348", "67403", "10653", "54228", "49092", "56558", "34280", "36822", "52959", "28838", "17363", "28703", "46200", "62336", "51121", "50178", "24063", "67104", "13298", "61569", "53566", "7658", "30810", "63897", "15769", "3825", "35749", "9324", "68163", "6659", "49012", "32958", "58051", "59778", "30443", "1795", "24273", "2125", "36254", "37438", "61842", "59634", "35048", "55216", "14860", "62474", "38383", "66887", "58180", "22780", "48905", "52128", "46329", "38310", "42235", "64190", "7428", "40763", "61494", "6289", "63108", "12925", "49788", "10497", "26683", "39850", "39035", "70415", "62621", "70982", "57143", "43697", "2852", "35899", "47233", "72990", "63106", "19384", "37370", "4520", "71179", "67007", "63570", "65429", "47107", "35324", "25817", "63818", "13575", "24319", "15866", "44921", "44597", "66618", "26087", "18368", "37174", "55813", "28963", "454", "48493", "14257", "24657", "22938", "12993", "14391", "15237", "40421", "48045", "7141", "60662", "69196", "28606", "27527", "16520", "66180", "39097", "56418", "18464", "7459", "27612", "29944", "48500", "59461", "64701", "18557", "66694", "20712", "20472", "1360", "66448", "57693", "64821", "13906", "4640", "25702", "46910", "17397", "53241", "34868", "4864", "24914", "31548", "337", "24635", "64538", "15324", "53226", "31496", "47393", "49434", "22007", "20893", "3486", "54701", "27003", "33805", "58175", "43202", "12222", "51921", "52714", "53038", "63402", "32531", "14040", "69439", "28351", "50764", "43673", "50821", "54381", "33367", "44282", "53912", "71843", "21463", "20308", "54103", "72903", "60784", "11478", "8145", "39943", "62870", "56579", "68566", "69507", "65711", "58451", "52450", "17880", "26447", "64868", "33552", "24103", "19222", "42786", "62872", "21711", "26132", "8728", "12667", "46233", "64420", "26306", "13511", "60582", "1117", "72406", "39550", "3627", "27863", "14925", "57819", "61732", "5333", "43502", "24444", "39448", "10800", "30537", "71962", "37455", "22625", "43536", "14542", "40193", "4627", "61200", "63738", "63243", "70478", "38354", "4883", "9373", "36343", "23344", "26527", "35455", "34875", "41138", "7090", "50243", "29961", "16201", "39786", "31349", "50158", "72315", "42014", "38243", "55890", "68830", "49531", "8028", "50181", "64409", "64617", "34954", "54363", "15557", "20638", "32522", "38341", "8487", "34743", "40677", "71807", "38180", "43118", "15220", "3659", "25601", "33853", "25133", "25292", "64030", "30015", "67877", "66024", "35004", "45804", "17390", "51814", "69260", "45379", "36129", "65775", "53833", "66349", "69949", "62830", "70649", "26663", "48538", "63013", "71448", "57247", "26289", "38861", "5597", "70283", "54263", "72520", "43641", "68377", "24591", "72692", "56302", "20274", "48508", "5746", "40703", "21638", "19273", "48915", "14037", "25524", "51111", "71089", "26598", "7457", "46067", "27229", "7585", "67675", "18630", "43904", "50167", "69951", "68733", "20159", "55141", "24084", "27922", "10682", "17422", "49355", "3527", "33426", "9981", "23484", "12460", "10545", "5737", "19134", "26682", "31173", "61987", "72351", "70875", "5138", "29496", "47910", "45129", "56874", "10243", "60125", "69578", "527", "1976", "40021", "8356", "3040", "39091", "56081", "37933", "12028", "53", "50695", "18947", "6855", "41596", "40669", "9953", "4380", "37885", "5351", "69065", "57232", "32798", "58887", "42938", "34290", "19269", "46445", "51976", "35461", "56284", "29111", "42338", "28953", "44730", "10100", "16386", "39430", "31563", "11351", "29704", "44529", "64611", "64212", "47762", "15919", "59961", "44790", "22224", "33407", "14656", "46448", "6320", "1594", "55723", "9838", "46907", "52995", "72649", "5162", "3693", "16249", "24699", "6992", "5402", "44010", "68741", "5331", "2292", "29555", "66080", "15042", "22086", "48324", "30786", "44151", "33391", "71698", "15834", "5994", "72626", "24367", "27033", "6894", "40615", "70924", "48714", "1223", "11076", "47211", "49513", "2970", "18300", "39428", "34760", "53873", "16275", "71519", "4292", "72877", "67871", "54984", "55564", "72340", "35716", "14279", "22380", "71158", "3083", "66996", "38589", "29172", "68358", "57035", "62426", "28347", "64394", "27167", "28772", "21264", "58341", "5095", "71572", "32630", "3463", "70690", "53976", "1561", "33190", "35632", "23794", "57901", "59335", "27479", "60323", "15752", "4187", "49088", "33196", "70948", "1229", "58979", "69807", "29340", "71747", "18791", "19431", "45289", "12691", "13897", "7696", "65085", "71626", "2944", "14444", "25492", "19543", "45434", "24920", "33447", "42114", "1273", "39081", "6269", "40410", "36390", "50532", "25666", "3025", "40511", "29602", "8818", "19476", "3275", "66108", "31353", "51387", "52187", "70927", "43404", "62567", "18277", "51655", "36545", "22440", "70853", "21174", "67944", "41906", "61632", "71827", "29056", "31078", "51972", "36958", "27762", "40168", "26335", "64505", "32108", "61567", "14812", "26432", "13130", "67509", "6562", "44303", "36306", "59008", "34697", "43443", "42952", "43555", "19082", "17857", "42725", "70090", "11975", "16091", "60112", "10278", "57393", "13471", "71649", "65596", "40487", "13729", "60768", "71663", "13295", "66749", "44165", "24579", "54792", "29940", "15576", "14230", "26373", "813", "33342", "34383", "53860", "34091", "20517", "30539", "37368", "19679", "16503", "5571", "56357", "11781", "23679", "30861", "11264", "39292", "7124", "814", "5078", "25758", "48735", "42140", "24073", "2239", "53588", "42963", "6667", "34886", "1097", "392", "60051", "6676", "68026", "30519", "55887", "13791", "31441", "43843", "64333", "15927", "27827", "20384", "49295", "48071", "2694", "18824", "36416", "20387", "70570", "14836", "10352", "59", "47229", "969", "698", "67740", "44466", "30380", "72355", "1567", "9950", "71302", "35538", "45386", "68496", "2003", "41362", "34131", "69010", "12174", "54882", "5773", "1586", "3900", "44729", "3453", "278", "55660", "38351", "16760", "6970", "61790", "67194", "19201", "51294", "14053", "64695", "54358", "37177", "10485", "9483", "1621", "66038", "26150", "6114", "54906", "24228", "21945", "14748", "24155", "27642", "59981", "35894", "22157", "25429", "34012", "53880", "28346", "47760", "55883", "38411", "25150", "30020", "23302", "9858", "7617", "39581", "57391", "11377", "24787", "71981", "1620", "44683", "40746", "9708", "66423", "27373", "22669", "18412", "39498", "45560", "70370", "61449", "70217", "49797", "70205", "43917", "19039", "22855", "55217", "23785", "33488", "65077", "40432", "35486", "68853", "62920", "51435", "54355", "35304", "17040", "49101", "68021", "13666", "8358", "56682", "56661", "62541", "16768", "1135", "1532", "62038", "15085", "14778", "19438", "16955", "16425", "13930", "27307", "27808", "22218", "11217", "32327", "64708", "31577", "38588", "18677", "27253", "21896", "36849", "63432", "16381", "67672", "25370", "2368", "18280", "26658", "59907", "59743", "28047", "51436", "54859", "24630", "20193", "21883", "57633", "31879", "9006", "14497", "12708", "67886", "50381", "24656", "53628", "18262", "6886", "38968", "23143", "66511", "55266", "16188", "24503", "69901", "4860", "51238", "64000", "41090", "61036", "67185", "23462", "62453", "32923", "56362", "51128", "66250", "39795", "47917", "4234", "72679", "68070", "1093", "28021", "70137", "11306", "72918", "20483", "43045", "69894", "57009", "67366", "50553", "49714", "48903", "22257", "29107", "60045", "6178", "46659", "68078", "40292", "11168", "18453", "7133", "68582", "68989", "47905", "3606", "64382", "17306", "64760", "28835", "18701", "13055", "67424", "22172", "32549", "30426", "53492", "38985", "2019", "3012", "41513", "32228", "42458", "40692", "18772", "58158", "67367", "43071", "7259", "62644", "55902", "42893", "60993", "32481", "39437", "9298", "47790", "21457", "12310", "46786", "42459", "22264", "18297", "23863", "60556", "19531", "41926", "11661", "1158", "51043", "29931", "48773", "18613", "1910", "55575", "58238", "2086", "7611", "12179", "40121", "65802", "43348", "56955", "64647", "32999", "31575", "16616", "11262", "3612", "23800", "70383", "44607", "9591", "26573", "66981", "25328", "21297", "59183", "20171", "65192", "42330", "470", "39127", "12187", "67151", "70530", "3674", "20918", "28557", "7627", "928", "9592", "40984", "12839", "70094", "14763", "5640", "39034", "54243", "40950", "56382", "10773", "64173", "18631", "10961", "11151", "16283", "56489", "27538", "52953", "55979", "61817", "22927", "14602", "44562", "30553", "3785", "32897", "4965", "25026", "30298", "65389", "25959", "62612", "34916", "72959", "69095", "29853", "17383", "45871", "30935", "48551", "68866", "64704", "47180", "9922", "30646", "59606", "50078", "24293", "42866", "450", "70905", "7031", "66256", "8925", "16802", "44570", "72493", "63968", "6173", "42505", "14197", "60089", "28279", "32795", "57586", "14834", "43610", "63353", "24003", "46180", "26752", "1263", "20959", "10590", "35387", "44009", "33980", "18912", "5694", "69777", "6941", "10981", "15722", "4451", "18107", "59986", "66601", "63307", "633", "31044", "52373", "1278", "63944", "4479", "65395", "63085", "45158", "14544", "37845", "52413", "18350", "68235", "1038", "68553", "6798", "16413", "33796", "47333", "37081", "7306", "65172", "9549", "32393", "55767", "32962", "66453", "52331", "63823", "15533", "54413", "45969", "33348", "31788", "28732", "14155", "67704", "58266", "2493", "50309", "45438", "26398", "13544", "60837", "16322", "43238", "12268", "37216", "5931", "15218", "45435", "61931", "9929", "67112", "35803", "11111", "42302", "52763", "28891", "28944", "21048", "10366", "61881", "29270", "53001", "23565", "22621", "8125", "11280", "44990", "18472", "43005", "67416", "58907", "21244", "63962", "2300", "27918", "46398", "20227", "15058", "54773", "60387", "44195", "62332", "34153", "27025", "23676", "65151", "68448", "13645", "32358", "27909", "60123", "14689", "65683", "13876", "46794", "37793", "49226", "18589", "21144", "61420", "5806", "55040", "3396", "3968", "65404", "6404", "22420", "6832", "71684", "9035", "14695", "20768", "22837", "33323", "13437", "45490", "49518", "54285", "61697", "3885", "54277", "57795", "3192", "23950", "51675", "63405", "32694", "51706", "61536", "67553", "24584", "42255", "48101", "30711", "58392", "16432", "14897", "73173", "46268", "68813", "14193", "53002", "10465", "50276", "20051", "68321", "41813", "5194", "23085", "51756", "71423", "41411", "2765", "18673", "9846", "42209", "21186", "13532", "15422", "50586", "1920", "43427", "17700", "43324", "64400", "31474", "41579", "7733", "50486", "18689", "8415", "2211", "54194", "47824", "7636", "59630", "16474", "68014", "11175", "63420", "62360", "40547", "12291", "46556", "38740", "12", "56886", "34681", "56447", "63366", "10681", "68894", "11462", "64981", "28015", "4403", "32331", "28479", "58113", "19261", "67614", "35254", "70389", "40596", "27657", "23431", "62354", "65044", "17289", "41527", "20923", "40477", "63724", "5731", "13946", "54408", "67146", "20885", "29515", "70864", "50569", "17732", "62377", "48839", "28994", "31103", "28591", "69438", "22637", "26144", "821", "11430", "62304", "7287", "36219", "40087", "40327", "10475", "37689", "1165", "25788", "53362", "21116", "39363", "40357", "68231", "64955", "66858", "24206", "30381", "34808", "72593", "8455", "19629", "15488", "49827", "3993", "64144", "56502", "56450", "65045", "58171", "46555", "42534", "71658", "68746", "54675", "12917", "42608", "51941", "69015", "10453", "36297", "61453", "8940", "37090", "21540", "34485", "37847", "9264", "65465", "51003", "10973", "73231", "17756", "45997", "43504", "56442", "17165", "37542", "32239", "29031", "29549", "72172", "38821", "43218", "69959", "68498", "68503", "54004", "26718", "28813", "27669", "6606", "21644", "9335", "57312", "40673", "25314", "69318", "45933", "15227", "69515", "30732", "49500", "71390", "28056", "38208", "42163", "65931", "51538", "56233", "51636", "66143", "54789", "52556", "58533", "53587", "13146", "54214", "55754", "27625", "35748", "11638", "67204", "29388", "6760", "62221", "32674", "72515", "12559", "6459", "3481", "5716", "62659", "69398", "53345", "28592", "23309", "58470", "30367", "55927", "47091", "20093", "11423", "6885", "2752", "42631", "1556", "54603", "43178", "26254", "58456", "54356", "62517", "56630", "19785", "18392", "1141", "60343", "71474", "8798", "72961", "54574", "42513", "31381", "9132", "16725", "10655", "43824", "57238", "57026", "34088", "9084", "61816", "67658", "22013", "17432", "55380", "59277", "3621", "27086", "50754", "71647", "39543", "29537", "60820", "44483", "42984", "62129", "63384", "60357", "24299", "30930", "34592", "49727", "28603", "20930", "3568", "39718", "65718", "30736", "46315", "33701", "43328", "24389", "58932", "24085", "7009", "66783", "23308", "39656", "62097", "23909", "62741", "30016", "13775", "72736", "37511", "19668", "56720", "32418", "9229", "66513", "44634", "58526", "37166", "63891", "36411", "3049", "33964", "59100", "33265", "60719", "21337", "51172", "30982", "5280", "44181", "19256", "51994", "29999", "15999", "10762", "50527", "39646", "2142", "52430", "70638", "67737", "36668", "29808", "2258", "1759", "70691", "31452", "9677", "10925", "60421", "17915", "53380", "9515", "51020", "38958", "20103", "52847", "53887", "24385", "40876", "40105", "8042", "64250", "19247", "27797", "67748", "59497", "19469", "52141", "658", "50948", "59465", "30337", "13628", "38794", "57075", "57629", "57234", "72532", "32309", "51606", "23840", "25282", "20398", "29343", "36119", "3895", "26743", "60567", "39429", "48005", "46513", "38629", "66506", "7380", "52706", "55372", "72928", "40815", "53526", "41941", "46323", "8869", "18173", "63820", "70597", "39070", "58770", "3591", "37540", "19199", "52740", "18924", "11675", "3763", "48867", "44814", "20686", "28261", "25309", "11513", "39378", "46079", "70016", "14855", "56366", "8298", "24182", "46806", "7838", "49440", "38105", "4610", "68883", "29321", "20002", "63836", "21317", "69509", "13795", "11098", "6684", "3305", "7985", "55081", "55094", "62178", "29528", "24133", "44531", "11924", "14015", "62939", "17795", "53559", "9257", "65545", "49970", "66835", "1228", "4222", "71577", "7255", "70502", "23818", "13769", "13204", "47203", "54733", "9698", "10922", "35401", "40201", "9675", "29763", "55965", "16910", "25250", "33359", "3741", "33411", "26584", "41327", "31008", "958", "28272", "8226", "37788", "60785", "27175", "39932", "39601", "44673", "34968", "66005", "49041", "37860", "35470", "69154", "53064", "62588", "10624", "14940", "43476", "66492", "6158", "54313", "41990", "66584", "17920", "33731", "28761", "66610", "22310", "31313", "59063", "72796", "23515", "917", "14060", "68354", "7601", "24611", "37298", "25360", "57125", "6187", "41376", "18501", "34415", "65955", "10538", "35247", "9805", "39649", "30884", "15131", "16641", "52317", "66909", "27706", "25827", "49742", "5723", "46895", "44242", "43614", "43898", "50256", "48013", "31165", "10924", "44824", "44622", "7538", "15763", "60914", "55802", "13885", "49321", "69533", "15552", "63172", "22547", "27654", "62495", "69403", "56098", "302", "54085", "32874", "52392", "69349", "26567", "10665", "33068", "35431", "55317", "30106", "73011", "8434", "16693", "67195", "3757", "18301", "14963", "60179", "40414", "33330", "47249", "11440", "478", "64075", "47659", "49761", "9489", "13090", "47269", "26268", "36696", "45965", "5904", "38677", "8437", "49946", "42436", "4036", "69373", "2981", "49717", "63114", "44760", "35839", "60074", "12196", "4742", "22624", "19010", "12714", "56338", "2332", "69081", "19091", "29064", "49893", "55122", "38524", "59033", "32641", "10738", "49545", "27780", "54503", "13949", "23509", "58740", "14674", "71061", "24190", "54188", "9935", "19338", "65412", "39553", "34602", "39776", "30963", "26375", "26978", "36229", "16469", "40290", "63742", "7816", "53011", "52834", "29103", "14828", "1123", "44539", "39399", "21431", "27757", "64873", "5948", "5817", "4734", "9236", "42933", "29716", "69825", "51964", "44608", "43314", "48293", "56003", "39272", "66299", "62523", "3391", "65858", "48706", "25544", "69527", "72585", "3515", "58405", "47258", "56535", "22388", "38413", "9112", "31581", "62843", "14203", "36997", "5135", "10193", "65794", "27456", "9106", "27962", "30689", "66862", "34524", "46529", "38272", "45545", "45352", "4219", "57887", "37609", "24716", "71526", "37580", "20087", "193", "35862", "50225", "33123", "59062", "67964", "10657", "62394", "27682", "63488", "21377", "20759", "59674", "26983", "39207", "12961", "67726", "7540", "10224", "63202", "68087", "5816", "65624", "23052", "56653", "60393", "67046", "22802", "33139", "22834", "50077", "38035", "62194", "39475", "34395", "64915", "45311", "10013", "27197", "27866", "49965", "51863", "52237", "5375", "53422", "71706", "35310", "9928", "12964", "58977", "56342", "5370", "39456", "21567", "67866", "66449", "43446", "49216", "31591", "31587", "9868", "11681", "29227", "11329", "10664", "61768", "57643", "61949", "32465", "9613", "49875", "2080", "11637", "42552", "15580", "1122", "33298", "60205", "69926", "21469", "39835", "20489", "72189", "22874", "52608", "4011", "24325", "68135", "9404", "71589", "40321", "62413", "58412", "1819", "48090", "67485", "29813", "33271", "71372", "49701", "37079", "4244", "54093", "2167", "53821", "27143", "31108", "6803", "71047", "32914", "11610", "10856", "329", "37709", "24244", "33303", "23533", "16720", "1830", "3983", "63340", "12173", "37980", "50011", "63789", "41219", "4387", "69890", "49024", "41402", "30644", "49201", "67340", "43853", "21515", "43992", "20958", "25474", "52076", "15727", "40169", "58403", "14432", "60209", "71864", "68388", "41869", "64048", "6021", "62418", "53916", "6946", "13271", "52640", "13301", "36016", "33934", "41814", "53318", "63639", "18530", "48249", "71214", "11437", "33663", "6755", "29563", "52808", "24848", "25528", "57708", "66959", "58235", "3648", "25572", "23171", "13214", "51786", "63627", "39878", "63277", "28333", "61603", "9812", "42362", "35676", "23692", "66941", "45752", "21527", "34237", "13706", "47615", "1010", "49704", "46657", "13839", "25825", "8080", "61054", "40902", "13372", "36993", "50433", "31324", "51641", "19909", "47639", "44605", "27293", "67923", "53451", "59849", "25014", "54300", "70897", "22916", "18966", "52118", "63206", "2240", "55458", "25985", "39659", "68833", "9244", "39059", "37651", "28699", "37237", "57941", "1513", "55800", "28370", "52262", "31019", "4082", "17661", "58851", "26556", "11540", "13569", "33526", "21532", "70537", "4295", "21328", "69660", "43013", "35320", "63950", "16516", "17684", "20246", "66397", "64292", "29310", "965", "9071", "44024", "29079", "16446", "668", "2851", "42363", "55148", "20875", "50895", "21975", "49799", "18386", "17174", "49103", "34639", "4205", "66114", "24145", "9116", "67598", "5023", "7597", "58108", "62239", "52617", "16408", "9620", "24954", "38167", "70902", "16340", "57342", "25972", "21628", "25167", "558", "51676", "65239", "36279", "58219", "4309", "41027", "47522", "31876", "11421", "5777", "55419", "65262", "18353", "56675", "27712", "29658", "67967", "7192", "64407", "42550", "60406", "31294", "11495", "41007", "52", "56976", "6388", "29747", "36983", "70360", "32492", "68240", "56968", "35307", "21021", "20363", "34605", "26055", "15204", "31505", "44097", "3911", "49588", "29162", "29811", "36055", "33797", "66274", "7979", "58206", "18324", "12695", "23225", "66598", "9174", "55930", "35702", "49638", "23504", "56150", "20770", "16286", "48210", "72578", "22002", "24797", "27368", "42482", "41375", "25940", "70863", "18998", "32073", "18023", "20863", "44647", "25889", "70650", "21210", "1731", "176", "54891", "61823", "52084", "15894", "100", "72054", "11062", "48804", "34770", "45153", "23053", "5583", "11002", "14942", "14999", "14187", "53301", "46337", "69208", "48653", "3569", "55364", "45148", "58184", "39136", "66137", "7474", "39907", "5379", "36205", "32821", "64484", "69694", "11311", "21218", "69323", "36533", "26431", "54889", "2674", "2122", "11219", "8796", "9970", "47429", "1064", "10055", "30918", "6993", "28779", "60776", "15381", "51071", "52264", "12815", "9871", "45755", "28545", "13047", "46125", "63284", "60626", "70972", "7081", "10478", "7326", "23893", "66665", "43637", "60475", "62603", "53964", "25472", "63014", "12878", "6354", "13291", "27386", "26938", "36614", "54147", "69953", "7989", "55252", "10241", "8732", "20061", "18163", "66061", "63100", "36179", "10608", "33636", "20287", "57053", "63161", "7294", "7519", "35886", "49928", "32352", "2162", "65705", "70851", "14009", "18878", "4247", "70969", "8092", "26733", "26793", "44661", "63256", "10697", "72716", "67543", "63418", "65824", "2637", "68824", "12410", "36104", "9177", "72694", "15186", "32386", "45630", "9472", "42903", "7821", "6358", "7129", "47253", "16434", "43999", "34816", "44928", "36723", "52503", "43787", "64790", "6426", "35429", "15466", "49950", "885", "11209", "25098", "72652", "26290", "9357", "38948", "12414", "8476", "20657", "3166", "25554", "62472", "73075", "16345", "3345", "45039", "54671", "56217", "62047", "27070", "14290", "69484", "57188", "21435", "51239", "20481", "37787", "50266", "32584", "37065", "63611", "16194", "26974", "45009", "26215", "6795", "21424", "62818", "69309", "34107", "33793", "45912", "53549", "32281", "2492", "46225", "63070", "48727", "53780", "23550", "68132", "12720", "58244", "23227", "63869", "23938", "38086", "44137", "63852", "8360", "15321", "35268", "52931", "5198", "39502", "59010", "45320", "10165", "46404", "50762", "42434", "4097", "26560", "50160", "26766", "3685", "42930", "41642", "15242", "12420", "1787", "16427", "43770", "17074", "50925", "42203", "12942", "22716", "9251", "13776", "47602", "61755", "573", "32759", "39800", "11409", "63141", "69517", "47079", "63946", "46320", "61774", "67969", "27264", "8322", "3132", "9857", "72765", "59006", "22368", "50474", "30991", "48548", "42502", "38081", "5935", "71080", "45416", "24300", "54165", "3799", "63862", "19983", "68419", "64607", "40347", "29041", "29863", "25576", "69563", "55237", "55006", "57683", "56232", "63659", "50851", "60984", "42749", "24331", "58065", "56031", "7814", "26326", "25449", "47647", "13105", "7447", "48075", "53111", "60936", "20426", "43043", "14677", "34102", "31305", "51147", "60250", "63651", "32432", "55787", "51823", "16346", "61244", "25048", "7410", "33835", "60082", "26737", "47137", "13594", "27991", "53589", "64971", "70480", "13333", "46552", "6806", "64169", "63722", "60697", "5463", "38857", "14857", "57349", "38623", "41218", "37809", "32959", "3821", "48031", "22014", "28565", "21884", "62965", "47822", "72726", "41006", "55899", "45454", "70633", "61894", "12397", "3258", "3661", "27154", "60199", "56764", "57457", "12497", "54715", "61784", "17064", "32736", "707", "46133", "24295", "17018", "9332", "33287", "62625", "51729", "16154", "9225", "21493", "71794", "6840", "72233", "17465", "64265", "39355", "51077", "31435", "67764", "64809", "55155", "26223", "47677", "38225", "11776", "65715", "24135", "7355", "32303", "34921", "36705", "25188", "26052", "36302", "47828", "27835", "72771", "46492", "71375", "63758", "70944", "72892", "70738", "30326", "16399", "43337", "58056", "31429", "55817", "63469", "19296", "5113", "48901", "53054", "66520", "42405", "21288", "18079", "61815", "17260", "26729", "31549", "41114", "55294", "13891", "47441", "66336", "12496", "59957", "9573", "59835", "7220", "31711", "34560", "28090", "63396", "7812", "45612", "62525", "54513", "54148", "28898", "57117", "29878", "52537", "28192", "9693", "11617", "17618", "64005", "58870", "37475", "62967", "47940", "1283", "28991", "58445", "42500", "4106", "73015", "44896", "66825", "1276", "33336", "7240", "57665", "50440", "15920", "34624", "71149", "47581", "65206", "59730", "41537", "61400", "48227", "22083", "2581", "8538", "66840", "19163", "64056", "56503", "28208", "32550", "52403", "35280", "11531", "51766", "40062", "46177", "44681", "69251", "69390", "65994", "69790", "34409", "9743", "12786", "9434", "31756", "52727", "48342", "19400", "38965", "48753", "26337", "61323", "42524", "9027", "20319", "6054", "66682", "1095", "14402", "13006", "27354", "18601", "51735", "9364", "39243", "43444", "52793", "38369", "25039", "42090", "34141", "33030", "21363", "62976", "10423", "7204", "62078", "20841", "46653", "22479", "47763", "71671", "13540", "60335", "60014", "48392", "50662", "51608", "44358", "47015", "46396", "52480", "50711", "6466", "17091", "49171", "59766", "22570", "56818", "66872", "5552", "10686", "17098", "8005", "61651", "26535", "4952", "9960", "36456", "16184", "24287", "14486", "17615", "61101", "40858", "57947", "59481", "60215", "24354", "69001", "25574", "43192", "37834", "8646", "68444", "18369", "913", "48478", "42289", "54187", "38998", "20585", "12052", "69057", "50102", "23315", "16659", "12016", "38292", "6529", "15332", "62456", "21487", "51496", "1086", "12018", "56921", "42752", "26876", "5755", "48286", "14267", "43943", "46085", "28025", "22426", "3361", "63854", "42061", "48161", "9602", "31658", "10535", "54034", "31672", "67517", "15797", "47814", "31728", "69671", "43101", "54090", "56197", "39884", "47442", "5555", "7425", "33618", "40162", "8490", "22494", "13898", "52178", "35110", "51717", "11164", "63587", "13051", "19567", "45156", "12648", "68329", "3647", "45717", "20990", "58208", "35379", "33599", "46184", "47829", "33479", "58360", "36622", "60811", "16555", "19098", "2828", "52304", "42723", "45622", "56186", "11946", "1083", "68677", "24210", "18503", "49319", "24685", "30393", "5975", "52557", "48584", "4347", "63080", "36148", "14176", "22417", "53507", "37109", "22100", "603", "68646", "65029", "19218", "62507", "60763", "43164", "36687", "45299", "47073", "36217", "22374", "13005", "43919", "4379", "42372", "42947", "41028", "52188", "61590", "1763", "29973", "12568", "51379", "70766", "65060", "60664", "17219", "1130", "71298", "60922", "47041", "64732", "39405", "21806", "24390", "5468", "16988", "58466", "28613", "37745", "10618", "72709", "35946", "56355", "2153", "32344", "44573", "47597", "16116", "53607", "4815", "31519", "1082", "60040", "12650", "34923", "12860", "68532", "31171", "10678", "46050", "51242", "51278", "36532", "1351", "63580", "61840", "66499", "14637", "31399", "62382", "65571", "51638", "39765", "60854", "24937", "71993", "63513", "4138", "54014", "21673", "16781", "66474", "39665", "3932", "40945", "42685", "63463", "40571", "44173", "8523", "36492", "33516", "8519", "38232", "33872", "65014", "52928", "31853", "45498", "14928", "22216", "9692", "72145", "42672", "13586", "5718", "32153", "46654", "56993", "55063", "46597", "13347", "65488", "19263", "49049", "33609", "4645", "6111", "35973", "7342", "16150", "39044", "15614", "33044", "67196", "54229", "30387", "68410", "43237", "68820", "17167", "31511", "46015", "21068", "24811", "19047", "72126", "55366", "32370", "24625", "2041", "40241", "53423", "33425", "36266", "55270", "4021", "30382", "23637", "22056", "33077", "42783", "18742", "62637", "56932", "25179", "50132", "10171", "34960", "2530", "65968", "14024", "25728", "10418", "26085", "4398", "3671", "40284", "55647", "71053", "32384", "68803", "46981", "1293", "44650", "28233", "71293", "51425", "33743", "53634", "28321", "37510", "32700", "9492", "18921", "35919", "58876", "22602", "21880", "66582", "71134", "3607", "68103", "58647", "50772", "17743", "57821", "4949", "4547", "17693", "59256", "2893", "7925", "69280", "66854", "59274", "10416", "8875", "3843", "42489", "72024", "13474", "20910", "44957", "49815", "200", "46502", "31030", "48386", "60668", "64282", "771", "45534", "23342", "9152", "36125", "54771", "39700", "55726", "18305", "69077", "1518", "67983", "61839", "49707", "62254", "17114", "27630", "9604", "58161", "62873", "36562", "32775", "24143", "18661", "35923", "18124", "46501", "41458", "72827", "54809", "57942", "21067", "66273", "46403", "16097", "62098", "45726", "25231", "18144", "36630", "1726", "60348", "53667", "43082", "63575", "59984", "54347", "15470", "35262", "43775", "51476", "42394", "35991", "47468", "20295", "22442", "31292", "15515", "59491", "12776", "14371", "52462", "47756", "51611", "2486", "70214", "45083", "55049", "29335", "45525", "69100", "30716", "21796", "32617", "20158", "15738", "47873", "72870", "60060", "65256", "22299", "54904", "37854", "15032", "37107", "26780", "6231", "26463", "62468", "62480", "13138", "59913", "65727", "55727", "45343", "27528", "32808", "20113", "56189", "55584", "57746", "22004", "57925", "71454", "18988", "27050", "3640", "30418", "55263", "23179", "16594", "6265", "69029", "72399", "39838", "70590", "55675", "26242", "27840", "13228", "72183", "8651", "8518", "71621", "60460", "39836", "53347", "5315", "33916", "21739", "64646", "22840", "28149", "50927", "15537", "70132", "45949", "57402", "46017", "9100", "15835", "6324", "17619", "8270", "20842", "37896", "58160", "44019", "69311", "39483", "9392", "33581", "6061", "2393", "45378", "3311", "44864", "51247", "50670", "28649", "7999", "29854", "33082", "34204", "16245", "66760", "69230", "31884", "43884", "53774", "10811", "27399", "71086", "71412", "43618", "60847", "72109", "21305", "67597", "27774", "70815", "61796", "47043", "22075", "25903", "56185", "54985", "67209", "14292", "12938", "53732", "46406", "8405", "51711", "3507", "15415", "39854", "69170", "30894", "58388", "26577", "69502", "29870", "24963", "20324", "51845", "57416", "40753", "27608", "54310", "26914", "16556", "26384", "68296", "54070", "45889", "29010", "13501", "36301", "6929", "7739", "13545", "65819", "10153", "6329", "9764", "66773", "3727", "52193", "59259", "4402", "42929", "8777", "68991", "23272", "12734", "13622", "5124", "47577", "58602", "5778", "56445", "37521", "13135", "22631", "12339", "46137", "69865", "10869", "5521", "40340", "40597", "3800", "72616", "6555", "548", "61982", "31805", "63262", "39294", "52954", "14584", "13247", "68477", "63654", "28948", "58434", "28343", "25241", "11993", "35176", "70993", "30587", "7117", "27496", "46921", "34787", "71622", "7463", "52300", "27691", "41954", "66613", "37265", "4111", "63169", "14210", "14718", "57514", "36582", "72527", "68720", "71822", "70589", "57705", "62080", "27401", "21571", "7834", "9080", "16731", "1208", "65527", "3638", "11206", "64636", "17395", "47564", "68299", "38499", "65542", "57156", "44952", "17324", "45244", "35570", "66958", "47422", "36072", "7691", "15047", "12946", "21009", "60765", "18209", "62348", "65177", "32245", "28574", "20310", "36786", "1695", "20630", "4289", "34561", "50049", "11524", "5305", "9172", "55554", "12080", "64946", "48891", "30876", "7483", "55920", "9327", "38866", "19659", "37218", "70356", "72193", "55056", "21184", "9557", "27547", "35606", "6312", "22862", "55769", "62890", "65230", "3851", "20414", "34354", "6120", "62090", "19389", "56896", "58927", "48587", "48105", "11099", "22702", "18438", "24130", "60500", "41661", "70549", "54865", "27285", "44162", "62196", "59216", "64661", "23766", "67163", "18733", "35459", "19444", "39464", "44778", "9444", "22127", "59871", "42522", "47090", "37069", "45841", "51876", "64351", "48461", "3770", "46213", "61363", "27379", "13019", "20364", "5368", "5764", "72549", "17031", "71624", "50195", "67440", "19344", "39993", "7194", "12242", "64135", "47210", "67860", "46862", "71521", "32397", "3792", "64527", "65398", "29194", "58406", "22733", "4620", "61579", "19652", "21821", "40862", "69930", "5879", "69729", "51736", "47481", "28368", "16923", "56207", "59943", "58205", "58032", "34607", "30817", "1562", "26049", "34475", "44916", "62193", "67443", "57702", "44793", "38253", "49322", "44120", "41620", "61155", "72338", "29651", "49135", "28602", "63595", "77", "61630", "20609", "52177", "70777", "197", "3779", "20300", "39360", "13033", "43689", "13161", "4900", "10126", "59356", "48782", "25783", "72886", "48911", "19035", "38571", "14786", "22768", "33100", "17696", "20170", "40137", "70403", "3203", "13782", "41858", "64998", "45693", "72994", "11029", "65102", "16627", "3665", "61909", "43050", "6963", "62359", "17420", "34791", "58446", "7761", "9408", "314", "64673", "47751", "48098", "8412", "70301", "70012", "17630", "4735", "2407", "41365", "4554", "67811", "23300", "51384", "33431", "35462", "68400", "49118", "71347", "67649", "38692", "10616", "4096", "52703", "63574", "63394", "69305", "1526", "26750", "52752", "4238", "42926", "66933", "8402", "49838", "59591", "3544", "16542", "44325", "53303", "63530", "15251", "13304", "33745", "13577", "61827", "39967", "72062", "49966", "72943", "3521", "39533", "9277", "61004", "25114", "23088", "48269", "70153", "7904", "41247", "6870", "38018", "9108", "60353", "9841", "41030", "3330", "48262", "51973", "20083", "25609", "31017", "35015", "66082", "23755", "24050", "42461", "34565", "28658", "4948", "4584", "62704", "67230", "71094", "70031", "22928", "43664", "32415", "45412", "30949", "7518", "18789", "20040", "37584", "36918", "65208", "30052", "15228", "26901", "45096", "4786", "67650", "53997", "52933", "34339", "51109", "46127", "12091", "32078", "46408", "64170", "63735", "59078", "60671", "62497", "45001", "9966", "10905", "23310", "37690", "49288", "414", "32645", "22154", "42222", "43811", "22407", "68606", "69150", "7382", "67360", "28834", "11318", "8365", "38681", "13226", "7921", "68076", "70988", "61157", "16352", "67826", "34197", "9511", "70225", "44426", "31366", "48925", "55579", "47717", "68131", "20106", "30192", "71785", "16767", "64157", "31696", "12335", "34799", "12547", "59963", "45561", "73037", "41385", "52326", "43730", "38044", "45362", "36889", "17621", "53067", "37092", "63949", "36718", "29033", "66177", "28263", "523", "16157", "14282", "12545", "17371", "54485", "49325", "36446", "65959", "57496", "50941", "11806", "50540", "61789", "368", "46926", "39773", "10835", "17728", "16861", "16073", "49032", "47095", "3532", "3216", "26327", "54913", "32535", "26012", "31625", "28901", "48305", "12507", "42538", "44845", "4228", "67534", "46730", "15323", "35034", "31009", "36920", "53532", "62710", "53720", "36175", "44416", "20935", "46338", "65915", "52448", "12343", "56613", "5719", "15425", "6896", "10574", "27509", "62551", "14890", "9425", "11707", "4934", "45854", "9721", "20665", "18203", "10990", "38749", "21141", "29311", "61711", "72848", "55135", "16392", "15283", "1538", "30243", "3418", "52232", "8908", "65169", "49662", "15397", "8862", "17797", "43006", "32227", "59874", "47893", "32619", "32659", "36164", "19626", "62200", "56806", "18553", "6268", "72077", "69215", "46591", "19709", "4950", "40793", "37730", "8572", "11232", "43208", "23689", "45239", "21495", "21407", "13020", "24813", "66068", "4778", "25243", "15674", "24489", "61369", "19751", "14275", "67863", "17072", "35836", "29693", "44563", "7953", "55673", "30990", "49126", "40310", "38255", "1275", "25536", "39610", "58210", "30974", "55363", "9607", "26065", "69560", "51768", "40063", "44417", "18962", "9188", "8036", "37287", "37011", "20962", "68247", "39743", "52093", "60055", "70747", "50846", "70497", "51694", "41452", "40917", "66293", "41093", "57826", "67219", "36393", "31159", "55174", "21664", "24110", "48523", "21347", "60815", "1060", "42366", "21971", "72304", "46516", "39661", "28040", "57292", "11864", "61190", "3961", "66236", "8364", "2645", "26746", "54781", "56814", "65414", "10726", "2987", "9925", "54738", "56448", "5612", "19753", "14140", "37912", "72411", "44600", "58965", "67285", "72444", "2755", "21949", "23335", "68792", "63537", "36971", "11718", "57937", "53908", "54505", "66211", "46237", "33567", "36859", "73093", "41476", "25051", "66404", "3371", "51151", "29226", "63380", "40", "27513", "40770", "6966", "50521", "56635", "37472", "22164", "19685", "5047", "41801", "30850", "70486", "17840", "7923", "10093", "53960", "36653", "68166", "62217", "72803", "60206", "25379", "60980", "27818", "52596", "63360", "30170", "59357", "36619", "53453", "6668", "18327", "72337", "1146", "6657", "56526", "38853", "47890", "70244", "67548", "47190", "10054", "27694", "50001", "14814", "41149", "3789", "58473", "63642", "44578", "37277", "65111", "13184", "53442", "69794", "937", "37222", "7866", "60574", "47308", "29232", "63865", "57806", "41703", "39072", "8019", "34215", "17727", "54118", "65608", "71805", "69837", "1546", "26679", "23666", "14014", "40068", "15906", "36311", "5356", "51280", "3144", "14217", "2469", "64345", "32760", "26133", "38803", "7723", "52116", "63539", "7006", "62421", "5643", "13112", "32703", "15203", "32784", "67057", "1053", "62265", "20987", "19396", "35917", "17192", "28066", "52179", "25196", "2895", "68893", "57190", "55318", "57454", "25272", "30069", "46731", "63631", "49252", "17775", "71435", "71175", "53013", "46284", "62091", "23482", "37098", "27132", "19271", "25224", "6162", "20932", "60407", "4116", "48950", "10385", "45828", "6188", "30742", "57922", "51301", "39817", "25736", "72750", "62322", "65363", "30881", "59130", "41043", "65712", "62113", "37837", "52132", "36728", "43220", "70197", "31543", "34718", "36385", "42074", "14551", "30706", "61702", "61337", "50916", "51834", "49150", "46834", "51049", "13864", "30288", "63042", "56702", "73106", "59350", "53166", "12954", "37769", "58302", "20281", "42829", "8923", "61633", "57361", "24684", "26503", "17680", "59262", "9526", "18617", "7832", "13770", "22296", "32405", "14234", "8193", "33005", "38671", "57177", "52728", "10906", "36353", "24951", "53027", "61794", "54036", "39167", "23717", "15566", "38798", "44830", "55131", "8472", "48343", "72027", "42460", "64350", "53175", "65193", "47305", "35763", "36797", "53861", "48418", "50423", "7230", "6691", "56612", "34355", "29302", "18606", "3498", "52138", "28999", "44726", "56798", "62247", "35979", "25207", "58939", "3067", "34495", "61583", "56433", "60046", "25289", "61038", "45150", "29355", "66919", "15688", "551", "60137", "66218", "32250", "14511", "21143", "62718", "4846", "31320", "17211", "28541", "58582", "45677", "49273", "63655", "9772", "37480", "38344", "54482", "44718", "43143", "43869", "45116", "56140", "53853", "64485", "35904", "46328", "28128", "29661", "63824", "24352", "56219", "70685", "59230", "12324", "38333", "12914", "37993", "32860", "18", "69049", "14355", "33309", "3920", "70503", "72556", "51544", "9202", "48957", "65918", "20118", "72207", "37038", "26126", "10152", "37753", "23303", "56783", "40112", "50682", "43834", "2093", "10321", "15055", "50999", "29336", "60455", "54048", "42408", "53942", "38907", "17146", "65418", "32124", "72562", "19930", "63478", "69647", "50869", "36472", "47283", "10347", "8198", "322", "40075", "70366", "41353", "30485", "41700", "9370", "34672", "13332", "33460", "69123", "22743", "15511", "69770", "14609", "64584", "6752", "48568", "40307", "51347", "71033", "1557", "11965", "12518", "68449", "46491", "71428", "19893", "72703", "70467", "2855", "21605", "6677", "61304", "6502", "37103", "60550", "67932", "21238", "18406", "23923", "44052", "38708", "64525", "24868", "17061", "66549", "72455", "41836", "22877", "1385", "56681", "66246", "24687", "67655", "30175", "40566", "35596", "14591", "35410", "26860", "21227", "36677", "60057", "23442", "54303", "72621", "50526", "8991", "38639", "61016", "34486", "62787", "16707", "19529", "30989", "17182", "63971", "57512", "38380", "63480", "58202", "52753", "71695", "11056", "57630", "45314", "42212", "70002", "72321", "61203", "36693", "45931", "68725", "1929", "23815", "11137", "40634", "17433", "25700", "54815", "53970", "12139", "17254", "60382", "55958", "61522", "24551", "16077", "7789", "46942", "51584", "20413", "10972", "25265", "747", "72904", "58923", "27290", "36591", "12900", "8569", "7758", "27601", "28746", "70067", "21811", "15290", "9984", "61163", "54483", "9105", "7859", "40743", "58994", "9904", "57326", "14799", "67012", "23946", "20586", "42476", "46250", "41070", "33312", "47859", "10406", "23304", "12099", "25680", "47835", "15650", "47697", "31641", "38850", "1190", "23673", "9727", "13287", "50237", "38090", "6625", "24556", "26111", "20360", "41273", "30531", "14577", "37039", "18783", "54112", "53570", "27284", "18813", "34536", "15403", "25242", "31036", "58042", "20622", "47232", "4368", "49689", "62670", "50271", "30398", "2864", "65043", "59690", "70834", "29472", "8563", "33450", "54766", "43486", "61734", "70044", "32911", "43338", "3059", "18929", "12095", "13804", "19116", "59718", "13672", "69347", "65154", "5964", "1145", "44548", "65694", "1700", "45915", "33173", "51535", "33722", "42102", "61056", "52236", "64665", "10279", "63486", "69682", "7676", "16727", "18340", "32532", "36957", "12264", "46707", "66876", "55906", "32829", "27382", "28902", "67685", "14546", "67837", "12542", "69576", "67673", "39183", "11564", "17724", "17817", "60135", "14901", "50729", "8172", "17719", "4663", "62040", "69146", "51398", "56105", "41923", "67153", "67856", "39079", "46074", "50937", "34981", "28445", "64429", "20675", "57902", "42098", "22722", "27586", "8233", "19575", "52767", "57288", "43507", "16266", "38963", "27080", "6733", "58454", "65042", "36180", "31166", "71015", "52077", "20626", "61394", "61602", "34015", "22790", "25582", "30211", "23414", "71968", "54872", "55836", "6094", "10588", "59658", "41955", "13890", "67999", "37337", "41734", "9917", "54181", "37841", "51856", "4899", "12294", "25305", "70556", "46896", "5488", "2614", "57002", "30605", "5215", "10291", "34694", "9383", "33706", "11554", "2802", "71960", "39578", "63769", "49361", "27417", "36293", "34545", "11049", "37399", "72435", "33711", "1420", "40118", "10781", "39984", "49409", "28476", "13884", "31765", "58627", "61653", "58102", "65351", "30682", "16564", "46202", "53871", "68664", "5263", "12659", "70068", "26229", "7487", "17811", "26204", "52280", "44241", "17007", "46766", "70830", "2921", "41611", "44033", "22646", "49348", "36531", "6250", "63679", "72535", "9760", "41881", "29127", "39369", "47864", "50290", "56158", "69609", "19594", "53984", "50438", "65231", "59700", "35336", "29749", "19096", "39406", "68186", "52846", "46595", "4274", "41313", "2356", "21822", "8105", "980", "21587", "54742", "1402", "36262", "68797", "58330", "59781", "72594", "14967", "51420", "71580", "58378", "45536", "28270", "48635", "36929", "3019", "15546", "46367", "53981", "3828", "33788", "59659", "35850", "40549", "26138", "16181", "54772", "32877", "65247", "20188", "60164", "62094", "30915", "17016", "52379", "49311", "15642", "64802", "60515", "2621", "6060", "62487", "52532", "10043", "15857", "41201", "13193", "33484", "51246", "1306", "38600", "54890", "33794", "11213", "51183", "3078", "52521", "58146", "67102", "15117", "64567", "39934", "34374", "13144", "65295", "56022", "29278", "73110", "27925", "69169", "44298", "63701", "65281", "37832", "7546", "70404", "56414", "72874", "7341", "63694", "41659", "23827", "3723", "39730", "20867", "2739", "35884", "55414", "17914", "19603", "11308", "36923", "64319", "33944", "71025", "11177", "5396", "26664", "67466", "26391", "40127", "71871", "11713", "16673", "36335", "3682", "65022", "40341", "72191", "9587", "49464", "58047", "51719", "11796", "11287", "61742", "45508", "33284", "52681", "4498", "985", "25076", "44520", "65396", "29130", "65413", "11543", "26703", "63203", "65399", "64266", "32299", "63817", "17587", "47169", "44859", "56537", "67017", "57591", "54532", "71019", "27155", "1815", "49358", "51592", "68458", "51180", "17262", "65234", "8213", "63528", "57176", "65631", "42889", "28748", "51758", "34628", "28118", "64529", "69087", "67341", "10505", "16559", "30636", "39156", "50483", "4791", "14251", "41650", "59196", "34653", "34594", "16806", "23168", "22024", "56940", "6216", "22575", "40390", "22521", "767", "24881", "49377", "66778", "28294", "63775", "23142", "8555", "61316", "54992", "29513", "10039", "35282", "44614", "37341", "48063", "60871", "15522", "54451", "51858", "73143", "29512", "49283", "55885", "65387", "10592", "53120", "45043", "25598", "43093", "48355", "66874", "46207", "61476", "31742", "24749", "59669", "60950", "23907", "28086", "38379", "38249", "5027", "16207", "18396", "13693", "14418", "17394", "43089", "69341", "55140", "45575", "45910", "56739", "25192", "29159", "42729", "30078", "14359", "29276", "13535", "40666", "61679", "11606", "5569", "62528", "64916", "71950", "46459", "7509", "49362", "46019", "58186", "22941", "70386", "49719", "72931", "16147", "2764", "66179", "21366", "51205", "9668", "29365", "47590", "68985", "38060", "65677", "63086", "18527", "65662", "34078", "25255", "21914", "54453", "17124", "30045", "44528", "34458", "37593", "19381", "19606", "54528", "62479", "35390", "43281", "38212", "6600", "5611", "27332", "45529", "41245", "9145", "2574", "10268", "38335", "32458", "71534", "69002", "10793", "37456", "5349", "32240", "42340", "53416", "56620", "54901", "35465", "20473", "64540", "10449", "44324", "33293", "28708", "38282", "71994", "68263", "37559", "37950", "38373", "15015", "8816", "30419", "69320", "41071", "2906", "23399", "53438", "24075", "25168", "50621", "51874", "9198", "26919", "26045", "38700", "32622", "19979", "39986", "34599", "34343", "41632", "31863", "1490", "65810", "32365", "706", "33680", "62784", "23476", "16062", "48642", "57678", "6205", "58861", "1589", "72919", "51593", "34882", "12806", "70116", "15898", "65402", "42857", "8820", "73022", "69512", "67766", "48471", "66329", "62503", "64349", "17121", "42296", "69700", "48331", "39298", "22067", "11508", "15404", "67927", "44276", "54567", "57765", "2556", "28166", "57539", "60989", "25645", "58055", "67884", "53190", "45219", "4660", "69988", "49637", "61504", "66508", "37952", "38713", "5317", "57447", "2721", "8052", "20283", "35191", "1019", "63219", "48745", "46351", "43968", "1916", "68242", "53378", "59246", "28731", "6049", "60555", "7030", "69061", "19144", "67078", "54312", "51229", "51613", "70621", "3485", "6573", "19695", "4751", "12856", "19706", "4168", "48946", "41621", "1925", "64326", "25497", "15124", "67302", "33588", "48174", "6230", "71930", "57448", "65334", "50950", "24318", "5809", "16124", "22149", "35311", "64643", "15497", "12688", "12263", "18727", "3412", "4637", "41962", "12328", "68099", "43847", "36869", "43527", "52700", "21311", "22723", "31812", "62871", "58452", "40822", "72511", "32377", "63018", "37665", "62609", "27252", "51667", "49756", "852", "9763", "13383", "50719", "72414", "1007", "7345", "42541", "64199", "70641", "57370", "2762", "49502", "10727", "72926", "19490", "27674", "70265", "14638", "67482", "59940", "53062", "18011", "72517", "7488", "8673", "12429", "71495", "57835", "38532", "23756", "18532", "18834", "23757", "35497", "27748", "39691", "49136", "59897", "19235", "5591", "57269", "64960", "40653", "67059", "350", "9790", "6566", "27216", "22920", "11619", "15501", "46756", "26381", "871", "24636", "39020", "28725", "48939", "32254", "10651", "12789", "12125", "48723", "58317", "58119", "71064", "30278", "52073", "55663", "58013", "17279", "2853", "14300", "5197", "22340", "13764", "70107", "51440", "3586", "60561", "17764", "28006", "61396", "67278", "19411", "69756", "43590", "12061", "31202", "20491", "20135", "46259", "52552", "65972", "70299", "47214", "44005", "1489", "46401", "36525", "34447", "16510", "19336", "54735", "54244", "42956", "33047", "32701", "56348", "56142", "24708", "50659", "41094", "3053", "32464", "45034", "50185", "15956", "10828", "64633", "46830", "50929", "22760", "71922", "42998", "44296", "11347", "36981", "69546", "20594", "59318", "71446", "3348", "47646", "6408", "3722", "54669", "10544", "60972", "25326", "28170", "20215", "13880", "21643", "47726", "47290", "29001", "17313", "35026", "31867", "68330", "6323", "4146", "59439", "58389", "30307", "54575", "1880", "17199", "4802", "15007", "12446", "71738", "58961", "11973", "30108", "12136", "71938", "10859", "7962", "25721", "15896", "66707", "43577", "71115", "39956", "33983", "48787", "29435", "71059", "17198", "4618", "52948", "40026", "45657", "1661", "59822", "6721", "19390", "26677", "25102", "50570", "21473", "60534", "23108", "31914", "18955", "33110", "1074", "57673", "18036", "65410", "14480", "71655", "65501", "39873", "6356", "30957", "23076", "10024", "12119", "69601", "68376", "33461", "47496", "10358", "12579", "56179", "37990", "18175", "15275", "55707", "23773", "52813", "5397", "27039", "26491", "40742", "13529", "51488", "48699", "69368", "35131", "58691", "30039", "43392", "54172", "17006", "26022", "8278", "11191", "36418", "10156", "34615", "36169", "34690", "11068", "71537", "15342", "20529", "28749", "11165", "9478", "53761", "26545", "38055", "38261", "69110", "30230", "66591", "21492", "1930", "70007", "26365", "50162", "2264", "65155", "70139", "34761", "2776", "5371", "17282", "5663", "34177", "72855", "28611", "51665", "63406", "49979", "38970", "48626", "33823", "68353", "34772", "6736", "163", "44675", "44577", "35818", "56704", "71136", "56156", "52032", "6093", "58199", "24608", "8408", "26413", "41924", "57812", "50212", "48858", "32020", "49505", "49204", "20865", "51821", "51359", "55846", "44210", "39407", "65833", "14044", "44034", "20601", "7014", "49124", "47628", "50129", "19985", "28723", "15469", "39965", "353", "19778", "18628", "25005", "25901", "14123", "70499", "56490", "70377", "60682", "1002", "6979", "61126", "23051", "24245", "42582", "27007", "72591", "28397", "58226", "4057", "23298", "14441", "47658", "55344", "61555", "33667", "1905", "62297", "28130", "54267", "19427", "34974", "14865", "2447", "17375", "37362", "52303", "63732", "63518", "34132", "4399", "54639", "67058", "65916", "69334", "55828", "60213", "12721", "20823", "51776", "72633", "67701", "38138", "16445", "7885", "10133", "25775", "41636", "42520", "12767", "20661", "36650", "40308", "26895", "53525", "65922", "51047", "15428", "16044", "22073", "23783", "61272", "55525", "7317", "64432", "50118", "7525", "41021", "49465", "52085", "38196", "44152", "25631", "26459", "37504", "42178", "37226", "23181", "17495", "32627", "21531", "47871", "36617", "48285", "69330", "62557", "50327", "4913", "29997", "54763", "32160", "42638", "60169", "51386", "47052", "43908", "9036", "64817", "14364", "70064", "54091", "51745", "51769", "35824", "14246", "40237", "19482", "39968", "30566", "9427", "41392", "20794", "49836", "21251", "66337", "34110", "37476", "29416", "35421", "24348", "54052", "62278", "21857", "14598", "44399", "44025", "54940", "61259", "22804", "18851", "69225", "11651", "19899", "16369", "47405", "43033", "26863", "28356", "51112", "7826", "29525", "71185", "229", "6424", "64857", "21459", "306", "72540", "55940", "65945", "41526", "71906", "24170", "20424", "32606", "61748", "51731", "25397", "30355", "23976", "4623", "68126", "41371", "71198", "57815", "15142", "27510", "59767", "16643", "35823", "22159", "37849", "32410", "66691", "34912", "3427", "35414", "65783", "36621", "60356", "63519", "18338", "61109", "23210", "57527", "29715", "53665", "2803", "54459", "57209", "25430", "52629", "54469", "44104", "52467", "66866", "40744", "27383", "58717", "19051", "5713", "34461", "17681", "62004", "1144", "64839", "52759", "62493", "29367", "24548", "17231", "66198", "50722", "64032", "53195", "57585", "2891", "44374", "14080", "57501", "28157", "37301", "4816", "67676", "62292", "68941", "50683", "64523", "60850", "27805", "6598", "51515", "27415", "10595", "60757", "72278", "36170", "29347", "14810", "17744", "66355", "59568", "65187", "46332", "476", "48294", "27294", "72423", "28869", "4141", "25027", "4204", "49406", "27812", "912", "16422", "69306", "39337", "51512", "42029", "66802", "28022", "63270", "19320", "26074", "17318", "8089", "45119", "57394", "51931", "5472", "12424", "16718", "22581", "21985", "6957", "24847", "46777", "3329", "58258", "18231", "55886", "11650", "40641", "4570", "19303", "30322", "34999", "20579", "70559", "6300", "55425", "17484", "13454", "67178", "1514", "54791", "44870", "15502", "2779", "67623", "25615", "68139", "30204", "44239", "8387", "38778", "11732", "71459", "55997", "69254", "64472", "22996", "67997", "69594", "46131", "6734", "23994", "56298", "30791", "46317", "36222", "33248", "7898", "43601", "39653", "56082", "31364", "28325", "33175", "29905", "28124", "12709", "69893", "15538", "42798", "27631", "58041", "6363", "22619", "8815", "44086", "72987", "47885", "27262", "53401", "54309", "61262", "6991", "57642", "26858", "63698", "62086", "54674", "38455", "1056", "8511", "53749", "58015", "55928", "49158", "13801", "60944", "29735", "66544", "64716", "13258", "11683", "11479", "39132", "58204", "60920", "56592", "38356", "42787", "54823", "51978", "8265", "71369", "72550", "37219", "48326", "60606", "19571", "5811", "59936", "60931", "29625", "21547", "20986", "12362", "71125", "24401", "35838", "64461", "16524", "36117", "21188", "24690", "63589", "59626", "49746", "40543", "8150", "16700", "7912", "30959", "65345", "48914", "66362", "34462", "11432", "46350", "8822", "53399", "31477", "1", "15989", "27824", "39798", "30188", "12922", "24291", "11753", "56885", "24980", "14774", "51518", "29772", "68795", "8195", "38526", "38830", "42876", "15808", "71074", "7802", "3523", "46576", "8430", "69365", "47032", "15802", "37360", "28255", "73035", "47840", "32593", "27765", "45234", "15410", "70713", "19391", "14824", "60658", "22496", "36010", "52613", "55189", "9948", "44107", "49504", "41469", "11937", "73", "71915", "16588", "27530", "61025", "24303", "18765", "39372", "60481", "4806", "25177", "22448", "12021", "20551", "58754", "1853", "940", "41829", "39655", "43496", "15405", "30475", "62188", "34881", "21041", "59500", "58303", "47817", "70700", "11315", "47369", "39215", "44357", "768", "22098", "10600", "40088", "28139", "59825", "72662", "36712", "61638", "39343", "6251", "52929", "58366", "48664", "54414", "65385", "59713", "10132", "64231", "57610", "63214", "29860", "46453", "49835", "68870", "43016", "66631", "27784", "5779", "24817", "56266", "68595", "41742", "71468", "32472", "2845", "8332", "15756", "69184", "17483", "70323", "69821", "56595", "4753", "48888", "27186", "13177", "45152", "21795", "13718", "33022", "43860", "12238", "66971", "70054", "46888", "72327", "1362", "12627", "21831", "57989", "3168", "20849", "6405", "35950", "48419", "35328", "57699", "46500", "63594", "29318", "12169", "11297", "34083", "39937", "12114", "50959", "27488", "5625", "66468", "40932", "5050", "57779", "5279", "49554", "67788", "9717", "56761", "22353", "52359", "5652", "58955", "9241", "10145", "68784", "63207", "62698", "5174", "56428", "70183", "954", "42665", "15854", "34685", "52508", "54942", "51308", "62903", "49090", "40776", "57603", "13757", "15669", "62335", "52893", "64076", "42138", "18884", "15027", "55812", "54401", "7957", "9288", "37932", "59527", "54682", "23826", "1884", "24147", "70130", "23896", "61409", "64040", "9463", "40941", "54654", "62368", "65005", "5502", "16240", "30028", "5483", "70778", "2546", "4796", "36982", "33573", "34031", "52786", "46993", "742", "2116", "69408", "45533", "62973", "4675", "51910", "63803", "7466", "21887", "54651", "6897", "45455", "71107", "5574", "62440", "12252", "30574", "48288", "63908", "56520", "58598", "27453", "66873", "43764", "62101", "36854", "4334", "46282", "20871", "15190", "30059", "45984", "1960", "51534", "72038", "53359", "71799", "22663", "73221", "39239", "39268", "53280", "42112", "4357", "38899", "21104", "47604", "63461", "71814", "3926", "4273", "46420", "52838", "51067", "46741", "69262", "23706", "39957", "73077", "32044", "43405", "2772", "31901", "40019", "10957", "13398", "33193", "36202", "56161", "66536", "70772", "62501", "30089", "70701", "71983", "6006", "30705", "68834", "18323", "70626", "34601", "56394", "51009", "8007", "42285", "58071", "11669", "57287", "25471", "23060", "69213", "3902", "48256", "7408", "45137", "66641", "72192", "46903", "300", "71710", "8911", "31676", "33299", "26359", "13377", "34390", "2131", "66279", "62176", "60350", "68009", "46773", "12915", "16591", "28375", "797", "44579", "66577", "5989", "3979", "46594", "72190", "45183", "27032", "19304", "42512", "66663", "23678", "8084", "24198", "15285", "53070", "24382", "27942", "29542", "46360", "10611", "44705", "72789", "45437", "64496", "72055", "63049", "10437", "8808", "61910", "61199", "11424", "56584", "24648", "23953", "37", "43413", "11041", "32466", "37695", "6259", "62367", "22548", "4842", "48130", "42357", "72037", "376", "15990", "71988", "56121", "21062", "23246", "19074", "6552", "5267", "41235", "68257", "49039", "58568", "72371", "65751", "55807", "48190", "2355", "58515", "22878", "27392", "14813", "53155", "10680", "39519", "36584", "17600", "5022", "46084", "8852", "11966", "65497", "70347", "34640", "60507", "56090", "34222", "60275", "36682", "60844", "49943", "69030", "72170", "45735", "11863", "7205", "71249", "49598", "37619", "69064", "29876", "22372", "42269", "9943", "69124", "10519", "59668", "17298", "4047", "45248", "40215", "11001", "65926", "19286", "50866", "11378", "58440", "53061", "12538", "37290", "44885", "70434", "60312", "29345", "32033", "36594", "26069", "41947", "52971", "44486", "35996", "70340", "16148", "24402", "2742", "4567", "14237", "32711", "10992", "15219", "45813", "64871", "51703", "62511", "4932", "23867", "57339", "24505", "14232", "14244", "69966", "45350", "26657", "51854", "71148", "30844", "65263", "46274", "50", "28075", "5701", "52515", "5700", "28987", "23212", "42777", "60048", "58597", "25693", "9300", "28249", "15879", "55314", "56076", "72284", "11961", "46446", "37929", "5470", "6417", "44454", "22466", "8456", "40523", "65258", "63690", "72488", "53180", "30210", "57147", "33890", "36863", "24443", "7831", "49674", "12477", "7005", "1268", "3831", "51243", "46938", "72851", "14451", "13461", "15134", "17708", "41288", "68872", "23965", "44708", "34375", "42793", "6244", "68877", "42533", "22980", "71540", "46839", "4190", "27675", "4039", "45557", "59706", "40389", "23910", "19664", "44064", "45840", "13812", "72718", "54450", "70165", "15125", "66823", "70243", "32539", "71835", "16129", "17239", "26208", "35195", "47432", "18787", "15764", "459", "28859", "33846", "51951", "13739", "10233", "53334", "72793", "30005", "69779", "50684", "63543", "2108", "34765", "58592", "72007", "6432", "46625", "5253", "26879", "13895", "25013", "59338", "58528", "5456", "38628", "61869", "17125", "2123", "15276", "28688", "71729", "70484", "54915", "66652", "61224", "64660", "49380", "49573", "29618", "5272", "10695", "10091", "7225", "50596", "41850", "15418", "9050", "27535", "14069", "12326", "66234", "25434", "1841", "64845", "48980", "54370", "47723", "23331", "72443", "71392", "43103", "38837", "34484", "962", "54386", "63664", "70510", "61948", "10167", "17028", "53535", "38388", "25080", "45641", "33365", "21360", "30417", "4068", "56925", "72823", "72434", "32189", "14484", "59004", "30332", "14156", "14848", "67359", "37201", "51810", "9801", "36887", "27083", "39808", "41830", "69452", "58294", "73220", "7974", "40788", "44346", "53364", "26931", "52567", "29043", "418", "15807", "15071", "67587", "23819", "25172", "32602", "30430", "63346", "68960", "46293", "4818", "27622", "60280", "3643", "44985", "28481", "46947", "23336", "39754", "58632", "24447", "61467", "48871", "61944", "39909", "51416", "14347", "35127", "69948", "13397", "9648", "54467", "70128", "30910", "33772", "9990", "58280", "59628", "64712", "21621", "55820", "22957", "40239", "52720", "13072", "12006", "23020", "18422", "6175", "6959", "41171", "39671", "50455", "4103", "58665", "59406", "36458", "56249", "7634", "308", "9109", "10255", "10549", "71857", "27839", "68268", "24194", "59141", "68896", "44194", "59097", "562", "50407", "42375", "69580", "717", "45850", "47310", "7152", "60751", "53392", "58883", "68598", "67583", "2865", "38723", "16535", "55509", "11113", "25922", "46572", "3389", "19759", "26185", "72801", "7744", "34272", "33474", "45414", "12608", "33555", "39606", "18649", "26979", "51322", "49125", "10342", "12027", "28007", "49186", "20924", "67161", "47098", "62744", "69214", "44511", "49491", "23648", "71310", "39738", "54605", "20643", "26757", "36676", "72986", "24179", "27537", "15484", "67406", "55033", "12583", "30233", "51001", "52547", "68267", "61645", "70949", "12755", "26479", "15530", "48109", "36002", "51780", "31596", "31800", "59240", "26612", "52599", "65040", "64787", "44900", "54486", "39590", "35199", "71031", "5872", "17340", "67170", "54256", "60268", "34187", "10197", "26448", "34773", "69843", "11093", "12100", "46254", "37737", "33807", "66865", "58874", "15312", "12710", "19675", "22337", "30260", "61098", "4134", "12550", "14064", "49028", "30596", "37466", "38495", "30875", "72671", "1537", "43746", "37554", "56145", "11792", "51070", "71765", "27141", "66235", "26311", "62835", "44230", "28656", "52248", "56411", "8999", "26785", "43680", "4622", "35108", "35559", "72754", "69709", "69071", "1775", "18322", "41145", "54718", "24056", "14378", "17440", "58402", "10541", "68456", "1393", "47800", "13663", "24490", "64962", "4598", "14516", "59133", "8597", "4332", "3281", "1372", "23657", "19124", "8635", "68739", "13100", "20994", "35951", "66456", "10495", "13960", "5304", "13934", "47403", "3337", "56343", "15472", "40046", "69956", "16221", "37512", "25111", "469", "28862", "61037", "51815", "6669", "64062", "69573", "26293", "47911", "55392", "30715", "25890", "1458", "61433", "56494", "45257", "34798", "9464", "29250", "4006", "61134", "68588", "51017", "26370", "6930", "15662", "56100", "10511", "51438", "49342", "29466", "14048", "47810", "42817", "36405", "67243", "11274", "48017", "14923", "41910", "6558", "26649", "8546", "42615", "67775", "66721", "5889", "21088", "46656", "34137", "29097", "26741", "35667", "45565", "65792", "23293", "23838", "14628", "16307", "32881", "24845", "18179", "67437", "4191", "60058", "12928", "44588", "49684", "50546", "41826", "837", "49811", "40044", "24044", "58484", "70357", "61308", "8440", "17818", "10058", "10987", "53279", "58129", "64970", "71554", "3226", "30932", "39096", "50874", "9459", "26948", "17244", "24123", "26847", "7276", "38605", "72276", "65382", "58953", "41264", "68118", "25745", "70844", "66203", "49369", "41439", "53152", "7624", "67318", "42155", "7660", "35294", "21855", "69472", "53068", "55086", "24234", "48078", "66344", "8515", "18625", "38547", "15865", "69861", "71673", "48852", "10441", "46911", "4514", "48292", "16848", "31516", "5188", "70198", "17752", "53332", "64733", "11355", "52869", "32220", "48224", "23844", "39251", "11800", "55402", "47138", "49025", "22447", "6153", "6067", "20263", "71722", "64966", "47131", "11081", "17501", "5972", "54938", "58499", "18633", "28260", "12736", "32446", "28887", "62431", "26", "52675", "72774", "56666", "32631", "46224", "10261", "58730", "67391", "66860", "4481", "2157", "40665", "43652", "1323", "331", "37272", "7822", "53552", "64811", "26058", "28435", "1848", "5628", "14658", "22706", "58431", "35970", "16272", "69245", "11196", "1205", "59530", "32168", "28714", "57100", "3514", "13061", "54017", "15986", "36067", "67817", "61395", "69996", "33098", "34746", "61250", "40796", "54646", "56434", "66518", "32965", "4307", "48373", "67069", "18381", "24742", "22522", "72951", "52049", "18710", "17464", "51958", "8394", "52673", "73061", "50646", "39139", "25054", "37470", "58177", "68304", "49316", "20415", "53794", "18223", "30046", "23242", "40693", "57080", "60272", "6441", "59315", "61188", "69552", "44736", "70174", "24464", "64593", "27185", "29312", "447", "70666", "60707", "18865", "38770", "40446", "35492", "4565", "67021", "36620", "68468", "2822", "61139", "18650", "53478", "32032", "38694", "37353", "30240", "54668", "46138", "53484", "62980", "8345", "15749", "1078", "66001", "31220", "48401", "18640", "37743", "72954", "65160", "36989", "28691", "63994", "29093", "61080", "61795", "44547", "72695", "51192", "32902", "41755", "59518", "28044", "30620", "10125", "1851", "9855", "26905", "64184", "27389", "21204", "36554", "34783", "44662", "8920", "39714", "29313", "41539", "67249", "46288", "60521", "5486", "9803", "8066", "55926", "67335", "24561", "25996", "36249", "3864", "5528", "34400", "20389", "22289", "5284", "7581", "26062", "56634", "62309", "63790", "44527", "19455", "51080", "24570", "61652", "35102", "29842", "8823", "73190", "72270", "22971", "45737", "39693", "61965", "30098", "8064", "51501", "9748", "40394", "14594", "16166", "13630", "23241", "35243", "9940", "46361", "10883", "9094", "31890", "50669", "26838", "32354", "32705", "39879", "65150", "211", "36172", "18556", "12146", "13954", "69229", "54714", "20697", "14280", "15918", "18771", "37927", "8658", "52741", "53801", "42267", "66345", "12301", "47517", "55809", "73163", "10789", "45045", "55277", "38739", "51945", "67508", "6116", "25376", "70799", "65356", "27471", "68957", "6557", "13040", "51691", "1680", "66411", "47778", "51401", "35148", "31751", "23539", "71371", "29636", "15899", "52857", "59903", "66233", "68796", "45832", "36938", "63140", "47492", "6964", "42609", "60154", "9932", "15868", "21693", "21101", "40556", "3587", "25166", "44442", "59272", "41243", "22168", "62478", "60083", "9731", "44509", "24115", "31031", "67003", "37024", "72240", "59534", "14248", "39751", "23363", "15814", "42882", "16306", "39772", "25382", "61237", "57729", "26799", "56640", "23161", "4027", "59725", "736", "27381", "21962", "44721", "51909", "23598", "41357", "41309", "44458", "67659", "52334", "56979", "24065", "45774", "33704", "37350", "69141", "13221", "55171", "8948", "26409", "72234", "21563", "70420", "42411", "69413", "21325", "67462", "47557", "64469", "56629", "32214", "35196", "51266", "33687", "18250", "51314", "19937", "3223", "66053", "30041", "15950", "21697", "69450", "14323", "15041", "19581", "7365", "8583", "17132", "29904", "29705", "55000", "64380", "60564", "27136", "38819", "11153", "67226", "24889", "17220", "22542", "23044", "39627", "8634", "28669", "58560", "22816", "21378", "48171", "38955", "47989", "31233", "2795", "52401", "71850", "24411", "67129", "4667", "30378", "64637", "48979", "49297", "70907", "6527", "54133", "27275", "8076", "57082", "64846", "41377", "50362", "67128", "8445", "72839", "29899", "58772", "9703", "31013", "68018", "45955", "22027", "49352", "64089", "17105", "30572", "3134", "52533", "55400", "11989", "10849", "12989", "41054", "71844", "18964", "53182", "2390", "29113", "19809", "19838", "50274", "27773", "43473", "16255", "64725", "11184", "71787", "8832", "19589", "26818", "13925", "70798", "16851", "63715", "57850", "59538", "49179", "15792", "39401", "6901", "31380", "7064", "24261", "53487", "31337", "57712", "7861", "44106", "61729", "59924", "49357", "28689", "36250", "51468", "44441", "32740", "37725", "35663", "2603", "71811", "3987", "70506", "40411", "63560", "49099", "6794", "54787", "28039", "4632", "19086", "71590", "68341", "41729", "39517", "19651", "34729", "66365", "4789", "71316", "5939", "38168", "69849", "52319", "52650", "553", "15411", "10204", "60308", "17662", "33016", "23341", "10081", "12951", "58275", "39141", "40205", "942", "28530", "11249", "22606", "4579", "60291", "22793", "55275", "8903", "66369", "19095", "57058", "40008", "54029", "46205", "38250", "13504", "40939", "61937", "25091", "20684", "54780", "3474", "40045", "54986", "34164", "60527", "55354", "21742", "67320", "31342", "66327", "58938", "13223", "19322", "54595", "67567", "14231", "7026", "39025", "40185", "59456", "46858", "71997", "36931", "71020", "71612", "61607", "35876", "58722", "16601", "27843", "17341", "69491", "43902", "38522", "3553", "11052", "59459", "6276", "51309", "22393", "59780", "31735", "1858", "29638", "43599", "20435", "14180", "50960", "34496", "31140", "25099", "21928", "68283", "69088", "55850", "38953", "29393", "43895", "51963", "49659", "23721", "2014", "3510", "17510", "66936", "21833", "52030", "66460", "48266", "14863", "46319", "66627", "16065", "44946", "67568", "45071", "5159", "29958", "40497", "62398", "26506", "591", "60015", "13014", "70789", "47936", "34547", "35556", "49716", "36878", "29424", "1834", "53212", "62658", "60832", "29793", "60968", "8994", "45793", "27014", "44323", "1411", "23320", "34957", "62430", "17651", "14522", "19554", "30936", "67752", "32197", "741", "42784", "15817", "53204", "2562", "51029", "62519", "60653", "19394", "61614", "50985", "51211", "69508", "12338", "57553", "46691", "34319", "53163", "18608", "25435", "51455", "34289", "51085", "21774", "56245", "47417", "30433", "43571", "55452", "15700", "44819", "11876", "67926", "2702", "72808", "17466", "23247", "59014", "20216", "45108", "30852", "58897", "72863", "25212", "40094", "60106", "52881", "10270", "28979", "17655", "17650", "71699", "22539", "35081", "59214", "22999", "51374", "62230", "20565", "68583", "62866", "8502", "18410", "1718", "47691", "2942", "62383", "33532", "17425", "56786", "63677", "30160", "8756", "71202", "56728", "52429", "17603", "55234", "17990", "57900", "18257", "7021", "2718", "50833", "61616", "28935", "28788", "72865", "24240", "36688", "34068", "38933", "49407", "36792", "16428", "69840", "44310", "70038", "45725", "19856", "15246", "40726", "56808", "23404", "26043", "45225", "54069", "59445", "23769", "69758", "4558", "35368", "66548", "16815", "71018", "10953", "19154", "51551", "4542", "69364", "71837", "33232", "28074", "71567", "71483", "48501", "15636", "63714", "46622", "63223", "27761", "3577", "25477", "51657", "27702", "53721", "52308", "12663", "44482", "65525", "46535", "35540", "1119", "71853", "69763", "26607", "23947", "32264", "57170", "46861", "12936", "43265", "34472", "53910", "72504", "10980", "356", "26048", "43822", "3760", "63943", "25799", "19990", "43466", "71561", "46644", "2639", "68472", "697", "5293", "49845", "67584", "58992", "1999", "36412", "64143", "37067", "18148", "47478", "32213", "65829", "47980", "64883", "5524", "32564", "71051", "417", "36372", "54383", "903", "2926", "2206", "71486", "56835", "2147", "38776", "62451", "51186", "12669", "54593", "15445", "21314", "54039", "41503", "70814", "13424", "34226", "72341", "33128", "66162", "13945", "61892", "24577", "36007", "33786", "38504", "1446", "70392", "28092", "35551", "42831", "42715", "34930", "4260", "8208", "67147", "69680", "20016", "36909", "56212", "17104", "45662", "24471", "55382", "12444", "26944", "40736", "56687", "26158", "12220", "20430", "52464", "4202", "21287", "64595", "43654", "46821", "17024", "60739", "30874", "48988", "58132", "41410", "36895", "8316", "11538", "7267", "55002", "26405", "31968", "26913", "72560", "61121", "45394", "68800", "31464", "8255", "15758", "39444", "21551", "44232", "19107", "38200", "36516", "48662", "29994", "13560", "21715", "19176", "22189", "32435", "69147", "17970", "19509", "51896", "39086", "622", "53854", "47584", "63278", "51257", "59899", "60447", "10415", "15693", "57468", "32666", "68898", "46648", "9787", "2881", "61207", "40479", "7749", "31786", "49520", "61311", "29215", "65610", "58957", "16924", "4748", "66590", "8633", "29959", "70028", "11278", "56852", "27967", "66465", "21748", "29219", "69019", "34261", "45372", "18270", "18658", "52713", "53556", "37981", "28667", "2857", "40282", "9085", "7251", "65693", "57452", "48994", "63965", "33664", "68413", "44930", "41354", "37278", "47319", "4817", "20477", "71934", "57230", "36296", "71101", "48897", "69852", "25694", "36946", "5932", "8632", "65861", "29744", "9319", "21908", "41492", "23306", "49666", "56986", "53052", "20523", "19487", "42056", "48335", "28607", "62960", "60248", "33691", "18634", "5590", "49789", "48426", "16082", "62465", "54969", "38057", "36924", "46399", "69824", "62988", "10516", "49829", "5414", "31909", "8891", "23797", "11502", "33435", "36475", "2791", "44672", "5649", "41857", "23631", "59866", "25256", "42442", "14704", "71739", "15844", "37607", "63421", "3141", "36039", "62657", "41949", "2028", "2567", "37057", "18291", "66673", "5910", "39115", "5753", "38659", "7450", "61192", "43300", "62164", "47223", "40050", "35549", "54828", "62127", "2186", "23987", "42816", "13163", "71742", "71084", "17405", "657", "25644", "5241", "29946", "25897", "35151", "64126", "55951", "5070", "49739", "8639", "15054", "30155", "7435", "67981", "35052", "20175", "11407", "60211", "55393", "9031", "52781", "57827", "65094", "26534", "4506", "52390", "52833", "30389", "48442", "49111", "69691", "48041", "11562", "941", "66146", "69059", "52597", "21413", "4534", "10570", "2512", "42167", "36857", "72840", "12553", "41979", "23574", "15414", "19724", "13154", "4018", "46189", "8235", "24898", "49490", "60634", "5780", "15234", "24996", "9522", "13723", "30103", "53839", "22887", "9406", "62713", "63315", "12431", "40300", "55360", "37697", "25539", "2295", "23669", "16401", "49098", "59362", "15437", "21331", "72087", "28248", "68753", "8904", "61164", "46196", "50313", "27508", "33928", "50164", "53139", "20595", "56340", "71544", "62604", "55949", "37310", "24770", "39514", "27838", "66851", "38711", "55377", "20433", "16003", "12823", "7951", "23195", "41837", "52233", "60661", "42308", "51618", "47039", "69739", "67760", "56474", "19828", "13904", "18001", "53168", "22265", "12075", "6989", "68177", "46379", "65950", "21474", "51452", "63877", "33120", "29956", "71383", "2033", "69750", "35743", "16498", "26686", "10712", "11302", "19601", "61759", "33728", "2850", "23571", "18539", "38624", "57639", "24146", "36646", "20544", "3962", "53326", "43431", "21154", "2397", "25928", "78", "5307", "67725", "57152", "31839", "35011", "72132", "46785", "72842", "71353", "48764", "65885", "28852", "59043", "6072", "60646", "33341", "12094", "6771", "29762", "37153", "66107", "62486", "64900", "53649", "27938", "15477", "2715", "17207", "29674", "71203", "61595", "44314", "54192", "48984", "13977", "50627", "66245", "56468", "68379", "69741", "14039", "66185", "32704", "20427", "59251", "52184", "5803", "31635", "48204", "47746", "64353", "56598", "44498", "36436", "19419", "59053", "63255", "54537", "65551", "53402", "58037", "59861", "51407", "16285", "37077", "34109", "38443", "40367", "50061", "62865", "9158", "19099", "4823", "2350", "23524", "53320", "73119", "19038", "4130", "2985", "22472", "46616", "42587", "59390", "59623", "36637", "20284", "7511", "61704", "34692", "49251", "6303", "68586", "54116", "14746", "17779", "71305", "58972", "32483", "54248", "50188", "26214", "5348", "69082", "25043", "6467", "33081", "51209", "62800", "26844", "60218", "68350", "18986", "2784", "36791", "38204", "64746", "39554", "48800", "31704", "943", "4638", "25271", "27668", "67130", "43971", "31885", "14243", "40585", "34404", "14733", "15922", "13003", "7659", "60358", "52878", "26464", "5261", "25664", "31154", "32109", "9279", "27219", "56900", "41936", "27494", "37308", "48648", "64539", "44965", "40960", "34149", "52742", "57031", "10371", "53827", "67100", "56250", "18897", "30632", "52522", "44336", "2255", "57184", "56865", "25160", "17771", "38615", "8272", "42808", "1878", "29337", "46252", "30819", "17398", "44524", "9130", "63892", "2025", "32182", "3130", "72512", "31385", "67096", "27616", "29234", "69875", "72939", "4128", "3292", "42619", "43753", "24755", "55043", "9039", "12495", "17614", "10578", "57097", "12253", "52541", "69501", "65435", "46388", "56647", "59828", "43347", "17321", "48784", "59782", "14510", "21176", "44176", "14908", "15406", "59179", "68766", "52895", "16928", "41959", "71142", "52242", "25931", "61762", "33869", "15045", "54507", "2195", "31866", "21970", "71798", "33413", "39418", "23370", "39362", "23691", "2232", "39706", "32306", "66282", "44180", "46357", "50171", "35171", "50391", "65687", "24195", "7046", "65344", "71357", "43808", "69179", "4315", "61942", "63004", "56461", "54372", "21619", "25593", "35317", "54088", "5130", "72667", "53460", "63006", "59484", "51072", "4933", "47841", "63999", "16722", "35564", "9470", "2809", "28005", "70262", "26166", "22826", "31703", "9079", "37633", "33497", "12527", "42724", "67809", "29567", "20978", "57384", "54456", "50004", "269", "65008", "39324", "24779", "46373", "5749", "39373", "56193", "25162", "42233", "22409", "39515", "28546", "2943", "26025", "4994", "53047", "2642", "68935", "38569", "37358", "33054", "50000", "61727", "2453", "64741", "1332", "61988", "55831", "64483", "64397", "725", "68943", "4151", "26278", "67047", "32668", "40010", "14334", "43907", "1107", "10936", "42273", "30189", "25480", "48458", "8029", "32607", "62682", "432", "49112", "61014", "64180", "55918", "18357", "3360", "44640", "31770", "69021", "61474", "24214", "37443", "37168", "52457", "13641", "68215", "23153", "4769", "72249", "52206", "12417", "44617", "32764", "69553", "22703", "59300", "3354", "32317", "56947", "70018", "63120", "70033", "52975", "48569", "23245", "42396", "51714", "12090", "36788", "54970", "35043", "67516", "59260", "34984", "60345", "46370", "57882", "45724", "47876", "48708", "29214", "57771", "71274", "63319", "57936", "28989", "18970", "11768", "48103", "3531", "60940", "32884", "60258", "63702", "30321", "16372", "71654", "70162", "42454", "64381", "59026", "18245", "715", "16586", "23960", "32187", "42033", "41802", "35856", "35586", "65825", "58911", "72215", "5487", "65401", "35773", "33043", "27431", "46867", "8670", "46983", "63593", "23671", "24205", "15121", "31633", "45973", "68298", "58457", "36011", "4847", "47423", "46038", "34757", "46146", "63397", "43937", "61526", "13583", "35064", "36600", "58780", "37824", "27714", "68383", "9711", "46607", "19975", "24100", "15675", "61580", "4069", "30602", "22449", "8124", "33982", "1126", "26701", "30738", "55743", "25534", "3181", "58008", "68609", "17608", "21663", "66081", "70978", "17916", "27732", "61667", "4614", "4350", "53653", "2732", "42859", "59369", "25426", "12517", "9087", "71939", "38752", "60245", "62684", "57856", "45596", "1939", "65288", "60043", "50121", "40451", "54836", "70660", "63556", "385", "37320", "72524", "68908", "628", "49402", "20129", "55482", "46757", "7603", "10612", "54452", "15210", "10076", "45192", "1299", "11911", "54115", "67245", "56400", "57750", "45005", "34939", "57470", "11192", "21555", "37268", "18251", "59910", "66019", "60612", "8344", "30314", "4306", "40350", "16748", "5416", "59145", "24277", "42553", "53210", "73241", "68110", "12899", "16210", "14885", "26181", "34739", "48754", "51437", "33520", "19052", "9490", "73047", "50165", "47952", "15917", "30804", "31841", "58962", "52496", "31048", "15568", "47178", "38996", "72018", "67205", "24830", "65579", "6164", "46111", "71711", "15946", "51819", "68491", "70069", "50145", "9974", "68346", "40333", "47443", "71434", "68369", "40739", "18143", "16551", "39380", "57793", "45823", "1638", "62457", "3020", "7412", "71761", "18428", "12128", "50563", "62662", "315", "24243", "55103", "36930", "44894", "52506", "57525", "68601", "70578", "17780", "37198", "66438", "47063", "60489", "66688", "46341", "33216", "28329", "32209", "40909", "69011", "10246", "72522", "52880", "60476", "13929", "43272", "49533", "57715", "5235", "72879", "67836", "11813", "50887", "4253", "52819", "35395", "68620", "25606", "42358", "69870", "488", "10978", "5853", "3613", "5511", "18215", "53650", "8810", "54896", "7535", "40126", "36141", "53157", "64713", "50822", "4512", "33977", "54975", "18647", "32159", "57365", "1740", "55818", "19340", "54458", "68205", "49725", "5103", "21881", "42176", "65441", "12524", "55338", "31760", "48248", "68034", "7250", "19208", "41793", "60116", "1355", "21812", "50856", "28768", "70608", "49987", "14731", "1363", "16972", "53292", "4608", "64085", "3706", "43769", "54028", "18580", "26569", "44887", "57536", "10523", "53044", "56051", "2225", "52988", "8171", "59872", "24403", "53048", "72781", "61357", "59731", "10092", "66346", "13174", "44273", "31422", "51529", "23625", "47903", "18616", "3442", "8802", "34186", "28050", "57885", "37275", "64254", "72437", "29209", "1809", "40959", "66384", "30611", "34823", "7644", "58421", "24816", "72144", "48703", "48837", "27196", "63381", "28681", "19406", "10845", "33429", "69231", "28915", "10271", "66447", "71211", "37650", "59964", "50402", "29183", "55803", "53791", "31908", "52849", "5595", "20418", "49784", "69067", "65241", "53874", "8254", "11183", "20760", "9046", "4197", "51548", "15383", "69942", "67220", "63514", "40144", "35801", "25332", "70142", "12846", "54394", "64425", "4820", "37140", "46675", "53700", "73137", "9902", "62162", "60262", "50905", "71942", "22277", "42262", "51137", "60471", "550", "64012", "57135", "37508", "13849", "42323", "52311", "60093", "39417", "24460", "26525", "32169", "1302", "26694", "51988", "21562", "57211", "7820", "6742", "53808", "58715", "33130", "38832", "64346", "8200", "58426", "25828", "55540", "14227", "57411", "61066", "23541", "44412", "14211", "70526", "39635", "52657", "47025", "26784", "67761", "30", "23025", "6473", "54978", "58128", "34993", "12124", "62694", "63164", "35776", "29768", "8856", "69205", "51623", "32796", "2027", "46783", "46483", "12867", "21749", "27904", "30010", "16853", "55750", "43448", "32553", "43819", "17552", "35354", "34720", "59586", "16633", "18284", "70784", "72439", "5653", "53206", "14308", "38430", "72785", "55308", "27822", "31509", "41756", "43559", "29650", "61498", "42544", "44133", "66678", "10534", "24578", "56408", "40303", "11842", "68590", "35560", "48443", "28431", "28957", "62544", "64365", "72467", "14891", "56281", "2339", "34871", "41232", "4947", "39370", "52215", "50872", "41637", "20143", "1224", "72308", "63607", "5601", "3465", "31348", "66169", "63670", "24212", "68687", "20155", "16766", "7103", "56913", "25533", "13574", "11227", "11242", "6780", "71023", "43722", "60806", "62978", "29642", "57625", "38198", "37721", "65041", "57398", "57028", "4650", "64211", "41693", "41182", "29262", "7918", "13978", "31038", "42208", "68128", "5365", "50040", "66547", "11995", "53742", "21921", "44114", "68701", "9344", "15563", "16960", "50636", "29826", "72092", "54751", "44054", "14271", "21014", "8347", "9060", "38647", "9092", "39764", "7209", "55316", "42594", "4826", "50882", "58712", "59953", "15104", "42365", "67710", "72579", "65840", "18797", "3065", "71103", "26296", "6038", "47937", "9401", "19802", "112", "43889", "52647", "4298", "9510", "20039", "52744", "67036", "20971", "21315", "73017", "66740", "53930", "24827", "70015", "66685", "19070", "25115", "27898", "49445", "16448", "72873", "28609", "38931", "29029", "13914", "66983", "11103", "17860", "3791", "10952", "33217", "18248", "58711", "47246", "46801", "31246", "13699", "66597", "21206", "15778", "11226", "36581", "31806", "66014", "41838", "71566", "55790", "28943", "61514", "19512", "62213", "50365", "41707", "38584", "21183", "5314", "1473", "3840", "16096", "18595", "26579", "11621", "2518", "4308", "67179", "23551", "50241", "47766", "16558", "29627", "31218", "5143", "43137", "23435", "37806", "35697", "66320", "14590", "13985", "22854", "69018", "71790", "62736", "71275", "31852", "23080", "53633", "21173", "71309", "5165", "46148", "26687", "9876", "5086", "19905", "60099", "64840", "38586", "3265", "55251", "32833", "46141", "27524", "24913", "18593", "16973", "43477", "54107", "35760", "14041", "50333", "18332", "17367", "9701", "12333", "18564", "39012", "22599", "56224", "17741", "57347", "63451", "59607", "56974", "4530", "19572", "15871", "65812", "41524", "56196", "40843", "15409", "62074", "46565", "8811", "23520", "30328", "58034", "54099", "64662", "71342", "51181", "52015", "5506", "69805", "29599", "9292", "5447", "15849", "72345", "10033", "25656", "72711", "42842", "35501", "39988", "43718", "9475", "54218", "63711", "49944", "67802", "2299", "55595", "45401", "65992", "47999", "33256", "14222", "20474", "19060", "18354", "61290", "34833", "34924", "70646", "71165", "65074", "59932", "7977", "54001", "59705", "26092", "54516", "31857", "38353", "71995", "71760", "45526", "48001", "48252", "51194", "25532", "59643", "56240", "8424", "23816", "53121", "6636", "37615", "69785", "14488", "43946", "40990", "57866", "17230", "60819", "59265", "24184", "2808", "66121", "53974", "1477", "24441", "430", "61940", "20530", "52067", "2007", "42684", "13440", "31490", "59395", "14776", "32636", "28839", "27641", "23427", "16789", "13338", "16546", "7252", "56392", "2325", "20767", "36928", "53428", "23991", "13650", "72945", "43712", "53170", "19730", "31179", "3475", "14513", "32519", "59978", "47754", "31406", "35230", "23591", "31407", "28516", "28401", "51749", "41868", "61181", "10097", "13637", "52692", "31189", "8060", "23674", "45674", "4362", "63417", "5480", "2089", "34979", "21868", "35535", "41426", "39223", "64964", "51451", "68531", "9488", "51038", "13016", "43829", "62781", "41051", "59412", "65056", "44191", "63311", "28522", "37474", "915", "29919", "54329", "27881", "4494", "23945", "14670", "25991", "1405", "20465", "11815", "26186", "19528", "34792", "10168", "26143", "58195", "9843", "38622", "55485", "50582", "38638", "33923", "46549", "22363", "49977", "48209", "29204", "71419", "48241", "40627", "39891", "18552", "55501", "46908", "15958", "48679", "16573", "70538", "28635", "66612", "20238", "46397", "44164", "36707", "5781", "17800", "65688", "13017", "30374", "17053", "19952", "70088", "1580", "37534", "63652", "17447", "66287", "59990", "19853", "11787", "8317", "45676", "34627", "28424", "59288", "8261", "38777", "66753", "28335", "70156", "15512", "16915", "5638", "16769", "16699", "25064", "52965", "12525", "12955", "14361", "20556", "40841", "26881", "1320", "69940", "58878", "64944", "43692", "4688", "2966", "72111", "42450", "51839", "2055", "45957", "43126", "56108", "33720", "11722", "60604", "46930", "18498", "49214", "28745", "9052", "71041", "41302", "23362", "65852", "43688", "48835", "51290", "15585", "43762", "12731", "18809", "39567", "48270", "53338", "31174", "62179", "65026", "1166", "23313", "47868", "55212", "16318", "72996", "5613", "24337", "45962", "16459", "67669", "1016", "33710", "583", "71631", "17151", "67907", "19712", "61106", "13980", "42583", "40656", "48973", "5353", "9446", "71821", "32454", "40034", "33480", "69591", "27040", "42195", "11742", "66074", "5708", "35088", "28088", "61744", "13229", "51789", "31426", "40982", "54041", "17874", "72210", "59474", "36075", "36644", "9794", "9774", "12320", "49558", "3720", "8262", "14319", "49603", "44934", "34737", "21007", "56519", "13009", "30567", "29115", "50606", "16631", "51708", "18082", "29616", "31932", "60222", "30863", "50978", "1267", "63851", "67001", "22019", "4364", "35059", "38898", "15322", "70113", "7944", "27224", "72377", "62025", "39001", "66572", "53868", "8745", "42456", "48933", "22309", "51645", "10752", "7966", "15356", "70776", "47143", "12893", "17546", "71319", "43450", "46890", "5214", "30343", "42758", "11818", "27283", "21032", "50589", "26017", "3115", "35374", "54641", "29168", "41387", "254", "3899", "22205", "20886", "11612", "19227", "18235", "15682", "37876", "67101", "7873", "40633", "27206", "60246", "17115", "39441", "53437", "24683", "65837", "71449", "53678", "57298", "1948", "57296", "60020", "50434", "24775", "20486", "15279", "65336", "24222", "39138", "4404", "40079", "10443", "419", "15912", "26119", "57289", "33758", "68475", "7512", "38922", "36101", "70379", "3817", "36346", "28680", "31256", "51302", "54491", "49938", "40670", "9285", "40812", "29890", "27665", "38597", "54019", "15160", "63660", "65670", "11419", "41347", "65934", "22136", "1544", "36388", "43781", "32692", "72674", "64965", "46130", "36194", "70188", "57660", "63088", "50234", "20915", "38022", "40338", "67166", "10839", "28857", "35319", "18015", "40535", "62645", "11415", "59195", "53162", "27733", "52169", "5684", "25433", "33998", "69195", "65105", "68940", "49245", "31646", "36603", "38273", "59199", "16155", "29789", "16804", "47745", "2265", "28340", "7865", "2861", "29791", "27709", "49457", "69441", "42986", "48240", "25816", "28060", "24077", "29447", "43982", "51346", "12359", "40279", "23494", "30891", "11481", "26835", "44802", "54136", "8615", "19501", "51327", "3008", "49153", "14291", "13783", "33957", "24793", "31412", "38506", "18965", "68408", "6996", "40379", "18747", "14368", "73001", "14245", "1647", "44516", "26213", "57153", "9615", "21733", "21203", "65991", "34487", "41723", "47430", "24887", "69558", "17337", "7245", "65341", "3526", "57487", "17043", "56833", "44949", "11504", "37464", "26001", "37223", "24767", "37538", "1243", "40036", "73182", "37626", "48265", "22137", "13164", "64619", "56667", "28622", "18694", "27615", "28403", "54432", "70345", "61233", "5437", "3801", "1906", "43817", "4739", "36252", "20236", "69748", "59180", "58578", "22796", "21228", "60676", "32849", "53082", "36360", "9336", "66712", "66409", "38885", "53219", "22012", "63246", "64312", "33956", "72894", "70085", "71779", "53844", "20304", "56659", "49404", "20234", "10134", "47696", "71264", "816", "43991", "42354", "53396", "27164", "62700", "55640", "59384", "16471", "39309", "49552", "57966", "2800", "12133", "71598", "65375", "5007", "17472", "8128", "35621", "24562", "69806", "12728", "53432", "5758", "63512", "28933", "47799", "9824", "39447", "43899", "49975", "32007", "58858", "42317", "47887", "69521", "17558", "2439", "6183", "58232", "33549", "45420", "67518", "47761", "71167", "20905", "4766", "27591", "25485", "31213", "72875", "27314", "30413", "20471", "45577", "3061", "4059", "24138", "3335", "39099", "21987", "8366", "55062", "40228", "45458", "71907", "62285", "67258", "56385", "15684", "71027", "5775", "23271", "12379", "45763", "25463", "60184", "17047", "9030", "18458", "10609", "61144", "69788", "16623", "41758", "70312", "15394", "55725", "66757", "7180", "30669", "26708", "22612", "59427", "28098", "43438", "7285", "51880", "24159", "59226", "63759", "6594", "44781", "35201", "38706", "6455", "41271", "63632", "29048", "58331", "42291", "55837", "60384", "33491", "20504", "44128", "36735", "6925", "32685", "27728", "43037", "8110", "41781", "29942", "57352", "7241", "12498", "64667", "36139", "50028", "50901", "65716", "55614", "44055", "52338", "686", "53198", "28018", "52609", "51644", "7296", "10311", "64104", "42701", "22937", "69755", "17574", "28962", "37708", "36028", "12809", "40304", "60647", "49664", "28079", "23875", "25229", "14272", "10521", "58432", "1101", "30723", "20634", "66350", "36027", "44193", "31468", "9529", "13740", "4604", "50206", "50794", "27722", "40231", "23635", "46987", "41553", "69467", "56293", "15014", "40092", "36759", "30455", "35740", "64632", "6584", "71453", "68150", "41833", "71169", "41863", "53363", "60675", "53004", "32500", "23795", "66814", "8500", "38095", "63881", "1860", "11899", "46763", "18180", "32729", "2334", "5751", "30673", "25248", "71292", "43024", "49865", "30970", "21558", "72669", "4441", "13990", "61938", "36108", "46695", "800", "47280", "40274", "24836", "63259", "9564", "70368", "63148", "66051", "17490", "2171", "17415", "9956", "13829", "54425", "6391", "57622", "43604", "59194", "46518", "52697", "27932", "25481", "20748", "61600", "22427", "19048", "58007", "14052", "56268", "45801", "40399", "52398", "6370", "53966", "59636", "30535", "8002", "65515", "19209", "11156", "25616", "67730", "3295", "47346", "34941", "33091", "68210", "47927", "16239", "38470", "23151", "21645", "71135", "16690", "54350", "51820", "64869", "14131", "37285", "21836", "34848", "40614", "15639", "16254", "11457", "43358", "51712", "22110", "34121", "30019", "22945", "68661", "41395", "17448", "14270", "55613", "46905", "65666", "69424", "68069", "14703", "62218", "18671", "48525", "57614", "69575", "40711", "4048", "44119", "7409", "39565", "54464", "29597", "64051", "878", "59174", "36427", "50581", "33947", "20049", "10018", "3650", "23483", "65020", "54863", "71832", "55907", "36207", "36760", "66172", "21266", "32010", "61866", "40846", "31267", "4173", "20925", "14859", "50708", "68585", "48655", "33281", "35361", "54046", "45862", "4073", "43255", "12073", "27590", "47510", "61868", "7055", "59675", "57593", "42851", "63475", "70795", "45977", "60945", "21946", "27438", "37693", "18409", "27180", "57226", "4293", "33570", "11659", "29867", "43407", "68618", "42499", "18427", "184", "16890", "25355", "65925", "65257", "61245", "1528", "12262", "61475", "68309", "41413", "55245", "10194", "62848", "55710", "28244", "52277", "10509", "4573", "27756", "37636", "35750", "51594", "55936", "40800", "42943", "65903", "14889", "54287", "58728", "51827", "3316", "48361", "26234", "35123", "71204", "25713", "71661", "51558", "23955", "61185", "57513", "41832", "51525", "54948", "62610", "24093", "37741", "21470", "33478", "72656", "8774", "47515", "30751", "14118", "17195", "49851", "23465", "57293", "25764", "17562", "69461", "34408", "6100", "22770", "66268", "50408", "1115", "58893", "20699", "71777", "18968", "45617", "44111", "67254", "22111", "8636", "14133", "34763", "62008", "45105", "28437", "41671", "62299", "43402", "69159", "46871", "41407", "7580", "11065", "10390", "28997", "4780", "27444", "49207", "42946", "8840", "5148", "8063", "72745", "2661", "50850", "4172", "39026", "41696", "58818", "39707", "5236", "68395", "36341", "65908", "32084", "6509", "51426", "19002", "19608", "4606", "56282", "44639", "35532", "12911", "6637", "43617", "29934", "49915", "71692", "20718", "65494", "50126", "68086", "57079", "28670", "51145", "47973", "20558", "45095", "12346", "16560", "34232", "16544", "29474", "70143", "10690", "45860", "62922", "70865", "6500", "7928", "18158", "29930", "23292", "3269", "19980", "68176", "57694", "40928", "18729", "11246", "60194", "16191", "56868", "64856", "25868", "26655", "51005", "2166", "51930", "28849", "40790", "8896", "6740", "68544", "53842", "24162", "42741", "34581", "64055", "36776", "67783", "52110", "23984", "25187", "2624", "41688", "67435", "14332", "42658", "64923", "13168", "28373", "8696", "69176", "32827", "71966", "67117", "19284", "68167", "61219", "33296", "43425", "12636", "49645", "21689", "27234", "30300", "48291", "26676", "16852", "58437", "45084", "33764", "6648", "60360", "53682", "21686", "41018", "70579", "57987", "53328", "4784", "15192", "42591", "28934", "14447", "10791", "3313", "6891", "55303", "22668", "3225", "60100", "54731", "39386", "41350", "11497", "66970", "71174", "37394", "32004", "57846", "49164", "47574", "19109", "18201", "38264", "39837", "47900", "24738", "56731", "56423", "18651", "5102", "40264", "66642", "55960", "18317", "61787", "57074", "39829", "24164", "42446", "69321", "22080", "8148", "38929", "17249", "31492", "7961", "23049", "43546", "69017", "31767", "13087", "65282", "56023", "26196", "50811", "28692", "18141", "28900", "30013", "22742", "9379", "34744", "3445", "47385", "25468", "70474", "55179", "59182", "66125", "8196", "56290", "36462", "6167", "62729", "38423", "37214", "38275", "59540", "16011", "12251", "13349", "61521", "33344", "29612", "1236", "17308", "30253", "13220", "16200", "6149", "35759", "30867", "17202", "33078", "17536", "10949", "8974", "24660", "68728", "42987", "61985", "9216", "3708", "66869", "25649", "62979", "28113", "53933", "35130", "20992", "67813", "12132", "32469", "18754", "32322", "32623", "72484", "27848", "62057", "29554", "4601", "7505", "20359", "58623", "33087", "51245", "4677", "43095", "29626", "11758", "30137", "46897", "21284", "47610", "22674", "45920", "35142", "9643", "39181", "50525", "62026", "28700", "13826", "32594", "1134", "10341", "7387", "34087", "15926", "34251", "31097", "34851", "1685", "71243", "47833", "23312", "28829", "55903", "4420", "36187", "24174", "60712", "70079", "584", "42250", "53775", "11477", "23804", "31085", "34403", "37792", "55280", "40759", "45918", "7893", "60120", "8276", "60369", "24885", "47630", "40777", "63797", "45425", "38234", "15282", "64325", "28767", "43711", "26668", "66695", "38784", "12436", "2960", "13961", "1046", "69238", "6671", "47839", "20648", "49239", "2938", "17903", "28316", "41937", "58776", "23461", "34333", "28459", "39860", "830", "43632", "22148", "47629", "72188", "53544", "10275", "63227", "39908", "19103", "63833", "56538", "15992", "14926", "6851", "43743", "34170", "30242", "65009", "63899", "50858", "58027", "27778", "18347", "51244", "8446", "12655", "58233", "20914", "17449", "60253", "56173", "5727", "46271", "71569", "54587", "66931", "10637", "34118", "56379", "23371", "64767", "29613", "24732", "70489", "50800", "13042", "51098", "73054", "10289", "49394", "20388", "13412", "33716", "27424", "35160", "18812", "13358", "12642", "39023", "764", "23437", "12015", "30462", "68294", "36285", "44947", "28793", "28354", "13329", "1846", "58842", "42199", "49432", "18918", "6403", "31354", "47721", "1376", "11819", "50050", "13011", "63450", "6475", "61699", "28122", "59494", "64490", "67460", "20977", "16985", "58501", "34444", "35012", "65499", "54717", "17925", "55972", "12225", "52064", "31551", "16127", "16086", "1689", "17538", "38469", "1585", "672", "16883", "62690", "23864", "26077", "41580", "44866", "11197", "41363", "46813", "57049", "25275", "65063", "18014", "50387", "54659", "46724", "4546", "7039", "59856", "61496", "18976", "47368", "14139", "18037", "32217", "44950", "16765", "2717", "57381", "62901", "34369", "39999", "18544", "22059", "71632", "28480", "20816", "47415", "58775", "70659", "26898", "60835", "66131", "44991", "39689", "27532", "68999", "8459", "15278", "19818", "65643", "59332", "28586", "53024", "31759", "31330", "46874", "28972", "9020", "11119", "67838", "46223", "24153", "52155", "72267", "11231", "34072", "71952", "15093", "38112", "38688", "56775", "56440", "64025", "5226", "556", "9411", "71285", "48845", "47051", "46059", "65660", "32613", "63969", "51025", "49471", "51432", "39672", "4670", "43044", "5820", "25112", "72311", "22933", "10974", "55121", "5185", "35209", "31507", "28503", "28273", "19014", "67638", "16737", "7746", "69460", "13001", "68621", "6295", "16764", "31513", "35985", "40365", "40516", "40660", "10263", "47065", "7452", "14228", "46661", "27393", "55395", "39615", "26423", "32732", "6254", "59519", "39660", "29089", "48855", "12566", "22267", "44159", "29835", "58187", "53928", "59345", "54811", "33776", "45588", "8584", "9920", "31361", "58298", "9312", "17359", "52261", "53711", "56581", "11005", "27493", "5818", "58018", "42597", "6565", "1927", "48851", "4723", "14494", "23042", "61676", "40492", "69602", "14162", "50249", "33292", "11644", "16604", "61953", "59902", "71216", "49401", "3276", "44335", "63772", "16039", "8016", "62459", "17082", "67417", "37083", "28600", "21866", "53851", "62882", "50797", "59889", "30141", "23297", "16028", "23380", "14181", "23779", "67061", "22344", "4763", "10796", "8383", "60745", "13407", "32792", "18774", "61798", "52229", "29720", "66348", "2458", "3743", "32142", "39017", "46498", "62063", "52265", "52340", "5974", "60508", "67197", "8713", "60445", "30965", "59848", "17001", "27862", "66476", "29430", "57653", "1340", "28926", "73060", "68033", "49476", "67307", "72089", "25270", "13210", "57853", "69871", "28154", "23638", "22608", "68043", "16782", "46646", "17117", "36775", "7931", "65219", "41075", "43280", "27295", "49962", "8342", "13382", "27814", "28655", "40125", "8537", "16035", "70155", "32966", "6112", "39819", "9309", "26195", "784", "71007", "72333", "51092", "68530", "61084", "63308", "34127", "982", "54705", "7593", "41545", "72119", "59845", "69223", "51868", "63096", "29157", "65673", "12349", "55718", "18388", "41751", "30490", "72521", "1609", "3680", "68116", "19288", "60059", "67683", "40645", "2347", "53471", "40889", "49823", "62756", "62740", "3249", "32538", "4983", "16426", "60851", "49702", "50777", "54140", "22385", "65816", "68592", "66156", "11200", "38237", "38216", "42603", "23829", "38566", "17628", "18425", "27043", "15830", "40464", "61142", "66870", "1507", "4087", "23009", "66201", "56851", "5136", "52913", "2092", "21052", "3525", "30127", "52394", "44724", "33055", "8938", "14573", "42221", "4930", "30358", "70967", "19279", "61086", "45397", "3037", "46156", "4721", "18373", "67570", "60238", "31442", "40334", "36321", "31463", "47088", "59677", "7236", "12371", "11676", "11212", "71728", "37045", "11684", "45570", "62145", "13526", "19963", "60866", "13986", "38165", "16361", "64766", "9657", "39731", "13114", "22208", "41554", "48995", "57466", "59046", "13246", "25761", "58192", "46693", "50027", "64215", "45423", "63388", "56869", "7099", "67094", "50343", "24711", "20654", "49359", "27209", "40001", "37328", "26336", "34893", "24903", "46810", "6977", "29883", "59642", "61376", "54047", "48547", "63477", "1873", "12988", "48403", "66243", "7320", "43659", "46240", "60825", "11506", "4892", "9626", "3421", "12740", "62560", "12213", "28184", "32490", "28674", "45058", "26653", "15128", "50688", "41403", "69939", "22911", "14785", "15317", "57976", "49468", "64244", "6638", "14548", "72348", "32667", "58489", "17688", "60102", "53092", "24733", "54305", "49418", "27243", "14993", "16187", "7325", "7288", "57017", "43576", "48229", "2628", "61059", "26097", "18829", "58621", "53075", "24513", "53718", "43825", "51397", "7539", "56046", "72627", "45410", "18584", "32417", "16332", "70951", "30110", "11277", "62733", "64753", "28743", "66697", "34143", "61218", "33989", "7558", "13306", "49236", "55278", "42382", "32528", "16095", "50635", "16226", "71217", "38016", "11224", "18457", "70351", "12853", "25180", "40859", "71040", "37388", "13443", "35650", "10537", "46418", "3431", "748", "43826", "68936", "52886", "56733", "36974", "24605", "45986", "17738", "31102", "43731", "40755", "9317", "43175", "56565", "17271", "20081", "23963", "2398", "23977", "12019", "33049", "19332", "38699", "6842", "71989", "68274", "45430", "46762", "37903", "56122", "33629", "47262", "17961", "5800", "14129", "35981", "53725", "23701", "27892", "58857", "12999", "22043", "38741", "61172", "25552", "18873", "7993", "26411", "22904", "49532", "19283", "63111", "13234", "71424", "53773", "71723", "69586", "22444", "50139", "62233", "6570", "48779", "6218", "39192", "49030", "35641", "46665", "18514", "56303", "72281", "39643", "54066", "66267", "64198", "8481", "19873", "26264", "70811", "38511", "50244", "24217", "53616", "43782", "56531", "5766", "24330", "61813", "30444", "37339", "11980", "31219", "12464", "26292", "22667", "40138", "22304", "3208", "49093", "46000", "18588", "41647", "67702", "1156", "66930", "7356", "914", "37352", "60473", "70493", "28366", "56894", "33307", "2326", "62509", "43029", "42161", "33997", "26887", "69173", "71326", "20540", "49492", "60778", "32919", "55271", "15241", "769", "51371", "8471", "19976", "70019", "43383", "30219", "72079", "63977", "15025", "59135", "57906", "52252", "70129", "70757", "57968", "54038", "12906", "27964", "21691", "1610", "16090", "22983", "15262", "57332", "48799", "71006", "57061", "15126", "56959", "29040", "64517", "12548", "27405", "55075", "33888", "10769", "22969", "49349", "26631", "20512", "6521", "67624", "679", "44966", "25924", "2898", "54391", "47777", "60876", "53565", "51087", "39075", "37988", "32038", "57962", "66341", "55526", "15050", "2416", "11527", "72169", "32110", "20936", "19791", "23262", "30713", "63626", "43057", "50298", "4434", "19167", "14562", "48643", "25290", "12570", "22483", "16288", "91", "40061", "9829", "5247", "12135", "20993", "67646", "16019", "68047", "48509", "5934", "16906", "21646", "5919", "51866", "48187", "104", "34525", "71399", "28186", "48631", "12644", "29426", "18108", "18325", "10708", "65895", "65223", "9073", "64154", "14433", "70398", "38952", "34356", "58991", "54376", "25773", "9168", "8690", "25977", "52257", "3561", "61423", "64037", "17368", "24296", "27841", "47342", "1674", "3781", "63177", "33726", "15972", "5282", "8461", "60960", "14764", "69160", "30692", "11789", "44032", "61973", "34965", "11133", "54233", "70642", "48057", "25874", "70385", "70426", "18793", "30923", "4686", "43568", "54632", "70452", "59348", "17314", "19425", "70242", "71385", "45159", "56014", "11510", "68846", "9128", "61256", "32058", "6644", "59244", "18587", "5182", "57718", "16742", "39436", "14995", "2811", "7612", "55917", "9137", "69518", "12205", "65470", "50345", "69983", "26652", "7760", "67470", "27214", "20625", "23113", "6003", "36173", "32669", "1748", "22364", "182", "70353", "2726", "27170", "2606", "48321", "4033", "62898", "69604", "43465", "40037", "31731", "42383", "26189", "59979", "6342", "8381", "6397", "4866", "37149", "4898", "13342", "60337", "22034", "71646", "32288", "33438", "6597", "3172", "65455", "7791", "33151", "73179", "37515", "55067", "44826", "57813", "42677", "34515", "3761", "23357", "24119", "48654", "73046", "66822", "37002", "23614", "25428", "26536", "11057", "54776", "45830", "70540", "49754", "47398", "19970", "71669", "71350", "18220", "15859", "25916", "59217", "60743", "64715", "16412", "3221", "11737", "72381", "25186", "50111", "4943", "9330", "72541", "20559", "68667", "33109", "68153", "6731", "67939", "24911", "31396", "55612", "44889", "1527", "4024", "62981", "17150", "39597", "70334", "42648", "44895", "35914", "32768", "32394", "24745", "5068", "71724", "52365", "22138", "48177", "14684", "26820", "35364", "30878", "6074", "58408", "12949", "1637", "56015", "3139", "8057", "35605", "4156", "28498", "51579", "67862", "29066", "51844", "39769", "13491", "41744", "69297", "35278", "42734", "59438", "72076", "36029", "13422", "5922", "64752", "51947", "19951", "6878", "62154", "8873", "36919", "20160", "33669", "14467", "41997", "72555", "35713", "69595", "18008", "10554", "51904", "11055", "25181", "37432", "57048", "6099", "71903", "24957", "58785", "51812", "56113", "8807", "28641", "58022", "16396", "64729", "33647", "42675", "57479", "46452", "50576", "31766", "2011", "18857", "14991", "50326", "29124", "58078", "71600", "11403", "3493", "2415", "60225", "38759", "7140", "386", "29524", "41047", "40515", "13356", "37291", "7404", "32925", "72042", "62391", "39222", "33746", "43642", "69422", "31298", "50845", "15151", "43000", "49312", "51566", "68398", "23552", "45338", "3430", "51283", "16278", "26857", "38377", "54546", "15981", "69139", "69186", "9227", "27172", "3667", "58644", "42080", "50275", "29146", "57756", "34420", "6944", "50962", "2539", "17008", "3805", "35443", "30445", "50648", "12466", "63077", "41156", "59596", "55717", "20315", "63917", "27533", "9123", "14377", "59555", "57231", "34426", "27692", "12807", "59211", "21296", "35493", "62005", "7115", "11709", "10080", "18833", "11741", "61384", "2716", "836", "39668", "4600", "18153", "44860", "43761", "61684", "38810", "1453", "33768", "57168", "67974", "65621", "4939", "46186", "47809", "56486", "27660", "62337", "18919", "14019", "11720", "54799", "19493", "12202", "53031", "64320", "69428", "18874", "54242", "3487", "61472", "804", "63251", "8533", "65367", "37970", "61135", "29282", "47881", "24004", "72260", "65471", "51803", "62751", "17573", "21280", "43980", "52147", "20929", "36855", "4563", "19637", "16632", "46678", "58891", "4455", "60877", "21180", "50519", "4562", "27139", "57121", "31955", "12007", "19530", "48759", "9410", "3639", "59722", "33589", "7635", "9321", "67922", "57050", "62919", "63621", "72854", "69308", "52174", "30158", "3833", "47904", "34488", "29632", "37833", "41123", "36998", "6036", "62792", "31402", "8219", "68965", "48044", "33350", "48731", "51761", "6339", "2971", "43936", "28415", "57726", "9043", "53996", "50924", "43145", "22435", "18096", "52865", "24773", "2997", "18669", "62925", "16713", "66925", "14745", "65469", "46548", "38120", "71746", "30467", "10983", "42474", "4770", "63555", "49586", "2040", "56990", "61969", "23986", "32361", "36431", "43967", "72409", "3733", "7247", "40734", "11846", "27238", "50046", "8161", "4194", "6495", "8812", "62481", "15072", "65320", "41080", "64827", "72501", "64770", "29126", "69754", "56306", "23743", "62958", "57396", "15553", "18155", "50134", "28252", "52033", "19943", "19805", "21471", "28563", "35599", "36528", "68617", "15005", "63279", "4978", "48918", "40694", "41142", "48195", "45049", "40866", "29689", "30834", "37588", "54499", "23478", "52107", "54977", "9765", "71017", "48214", "69827", "18819", "56688", "15438", "50109", "33228", "27325", "52231", "39788", "14541", "36040", "67749", "61230", "70380", "31861", "8522", "56867", "45348", "30109", "61382", "47406", "54750", "19810", "9819", "63075", "32941", "20021", "49057", "65365", "66929", "5542", "67409", "35998", "69914", "42348", "55137", "11865", "63146", "11324", "60304", "28285", "36716", "43403", "25763", "38123", "13469", "48453", "64453", "49540", "2880", "21657", "32654", "55034", "23117", "69342", "35197", "71077", "1961", "22288", "6274", "53533", "36310", "6538", "15870", "63123", "15801", "63194", "43638", "50803", "4941", "13870", "9484", "37323", "69947", "69718", "65139", "66493", "20828", "45357", "6208", "20774", "26023", "9352", "45354", "45011", "57960", "7756", "10370", "11371", "68181", "66989", "3079", "20094", "55162", "35065", "1652", "35852", "37535", "44123", "51621", "51427", "64597", "24070", "20984", "16527", "7308", "52428", "49287", "50500", "36224", "12613", "2335", "63661", "21794", "2113", "28512", "11121", "21434", "56054", "1359", "69052", "36666", "64155", "7420", "59202", "45026", "23802", "25508", "2272", "71120", "44954", "37426", "45556", "71666", "27362", "57601", "9807", "4148", "67582", "58687", "68727", "57716", "26037", "63352", "53573", "2442", "26726", "53733", "18242", "13608", "20574", "43906", "67806", "23281", "35140", "51494", "56182", "33282", "72202", "3923", "51892", "4749", "68542", "44101", "56498", "1792", "46881", "71288", "45456", "38755", "8116", "68127", "20309", "1758", "16348", "29703", "51146", "69405", "22371", "5733", "47625", "58384", "32224", "11695", "47884", "16652", "31192", "49011", "56694", "21039", "65577", "60128", "30164", "54643", "2148", "817", "41384", "69829", "23340", "22981", "28203", "40243", "2047", "52565", "34707", "93", "1672", "67067", "66104", "72984", "3438", "52058", "21483", "51208", "10931", "59901", "40993", "48287", "18843", "45698", "13203", "17389", "68285", "12607", "41603", "63552", "72211", "62647", "1523", "69024", "35489", "33368", "66702", "53443", "56334", "52726", "14569", "69850", "40850", "3700", "42762", "63041", "30758", "19489", "44058", "69989", "29314", "52622", "50624", "20102", "4555", "4919", "47933", "72625", "36866", "1670", "24783", "35989", "41328", "53074", "46514", "11520", "29655", "52385", "63286", "20827", "13453", "40117", "65384", "62107", "6179", "71410", "12245", "11521", "19339", "43552", "63819", "13975", "9144", "12920", "50161", "28816", "70839", "14838", "49858", "35435", "12288", "25263", "49967", "31736", "25209", "30752", "68291", "50801", "16492", "51800", "45571", "35042", "38729", "22880", "4587", "12606", "18578", "14815", "38797", "33467", "64452", "42320", "47277", "14816", "28555", "31644", "44591", "23723", "41750", "72923", "22109", "1937", "32046", "41613", "35922", "64430", "9531", "53465", "54788", "25191", "35289", "59430", "7606", "54183", "21053", "34788", "70810", "11890", "46243", "58700", "57505", "37205", "48605", "3000", "56446", "14629", "54342", "407", "14092", "19251", "60952", "19066", "64976", "16749", "43631", "53699", "7557", "55909", "26014", "24426", "71130", "832", "27553", "17494", "65156", "25950", "57207", "17770", "7200", "36441", "63616", "66820", "35617", "9393", "13463", "36291", "15889", "47313", "12519", "9372", "46210", "46809", "10625", "20392", "44595", "7844", "12143", "45695", "29794", "37996", "69543", "24326", "21751", "68672", "16934", "27480", "16779", "3299", "40810", "45035", "71750", "37916", "11253", "16799", "61064", "52043", "72730", "14959", "40203", "1387", "25749", "47174", "21260", "7072", "57607", "51643", "59093", "48967", "14452", "38554", "4819", "53756", "23497", "27543", "28391", "37865", "69493", "63808", "38322", "30501", "64110", "68414", "57109", "49620", "45720", "63845", "9155", "952", "17136", "23359", "68786", "43174", "41679", "45173", "57917", "53310", "11996", "65024", "54169", "54216", "57510", "53129", "44088", "72274", "4867", "13916", "39493", "42427", "50307", "57769", "13173", "58650", "12445", "48132", "62051", "71491", "29797", "22198", "55590", "39721", "11628", "644", "41356", "31258", "32530", "39895", "30293", "45544", "22934", "33988", "53134", "4552", "32568", "37318", "70800", "61925", "59610", "29877", "11936", "5097", "47427", "7193", "16057", "22947", "67741", "60581", "6811", "26897", "23658", "4682", "44080", "46711", "33017", "32031", "69877", "65948", "67305", "18161", "52803", "60066", "27168", "71138", "6937", "48920", "3819", "5173", "19967", "37685", "26477", "26180", "8311", "15154", "9196", "33189", "46804", "24558", "64874", "58816", "13798", "3220", "34738", "28471", "55421", "6253", "49767", "8734", "63409", "16818", "33503", "30504", "72101", "38104", "34062", "44693", "60783", "38098", "62024", "39856", "39893", "34569", "71623", "34247", "12186", "28477", "3735", "13392", "38806", "46838", "57991", "63362", "28109", "71335", "48487", "5206", "57227", "61243", "14871", "4801", "70378", "67727", "40146", "38754", "37299", "13113", "24992", "30237", "45475", "19704", "43891", "21080", "70446", "63179", "23578", "49280", "18193", "52519", "19104", "28197", "23257", "67222", "61469", "18559", "1634", "4473", "56917", "46424", "3307", "10546", "49631", "67439", "48937", "30902", "19570", "43810", "5621", "996", "19719", "6936", "27603", "24704", "30166", "33861", "41854", "53050", "35967", "10875", "57294", "7354", "72687", "48278", "12434", "2306", "50918", "57635", "65721", "20420", "7897", "62253", "28921", "48405", "72178", "41282", "53440", "48975", "31195", "64994", "42009", "57359", "17654", "58980", "2730", "41789", "70220", "41467", "54281", "21543", "58252", "56116", "67055", "5012", "64115", "60400", "24275", "60334", "64892", "24086", "55610", "72227", "9530", "43105", "48670", "28398", "67845", "42982", "28483", "46395", "50154", "32778", "63838", "42646", "13305", "29754", "52560", "63709", "60916", "20233", "41444", "39161", "40816", "71755", "8070", "21864", "19114", "53555", "63571", "37621", "71381", "72166", "22437", "32122", "72410", "40294", "59799", "54466", "38240", "12438", "30216", "52395", "18737", "56168", "20478", "50340", "142", "45978", "15013", "61960", "66643", "13525", "63230", "63264", "59454", "24809", "31900", "1179", "29902", "10473", "72989", "47671", "33683", "30190", "43663", "41416", "66556", "56102", "16521", "28769", "21991", "21648", "44879", "63211", "1279", "18112", "45907", "24651", "54326", "15799", "58764", "45982", "2887", "30036", "9967", "20716", "8140", "29800", "21684", "58507", "60271", "37429", "15521", "55580", "40154", "50054", "39178", "63861", "33830", "17196", "57066", "14687", "9127", "66806", "15610", "52983", "47297", "72525", "43959", "54708", "54239", "66204", "67882", "6967", "69555", "14572", "9662", "41493", "18033", "10510", "28687", "23888", "66523", "54221", "21094", "4571", "60625", "28784", "61168", "20683", "41809", "36235", "40965", "66566", "8068", "19161", "60056", "31948", "66444", "15349", "33747", "20265", "25863", "35428", "54198", "18263", "49675", "68962", "24341", "6707", "46899", "13209", "65342", "63210", "63181", "15908", "18208", "69266", "602", "68160", "32094", "42883", "16746", "35700", "12440", "61228", "34250", "8941", "69409", "16658", "57752", "62147", "57375", "50967", "51161", "53255", "8915", "6119", "57566", "27611", "38950", "47450", "16070", "59045", "26392", "30901", "64882", "62491", "2564", "69085", "27820", "14644", "60433", "43957", "14734", "26127", "68058", "36691", "66382", "27719", "14930", "64690", "13816", "22057", "1435", "48628", "10147", "31370", "32310", "35831", "35594", "32243", "16701", "52822", "59802", "51693", "57202", "56025", "51831", "65847", "19917", "4259", "17135", "36721", "18536", "44027", "32856", "44360", "51801", "5656", "32357", "47493", "27286", "54686", "44042", "57114", "53105", "59238", "59032", "70933", "58804", "13614", "12004", "24700", "23353", "8758", "46927", "16885", "12026", "41828", "66583", "64010", "40335", "60365", "58297", "58481", "47148", "54284", "11609", "43256", "34532", "38536", "57975", "5923", "58966", "43301", "32681", "51372", "65313", "70269", "10899", "18335", "30860", "7047", "43966", "72400", "67352", "24896", "30660", "41952", "35747", "10907", "39979", "57446", "27955", "29527", "28750", "5918", "38932", "20173", "62441", "11345", "11255", "68977", "60958", "15615", "2948", "437", "29815", "25083", "57744", "57963", "48212", "71471", "61275", "54935", "48277", "60072", "23841", "69962", "26512", "65195", "22636", "42761", "32380", "49317", "70247", "53076", "72962", "15725", "20848", "70756", "48381", "15541", "64102", "70390", "60326", "29647", "16119", "129", "46129", "43474", "15677", "65707", "9751", "68679", "62224", "45259", "59788", "67131", "71910", "57920", "32185", "22835", "29047", "57319", "6745", "57523", "34338", "58357", "13391", "66443", "46132", "52927", "18782", "4041", "19816", "40048", "30077", "21756", "68289", "27718", "64445", "45205", "40029", "11746", "64762", "35134", "49969", "28969", "23219", "64555", "37181", "54152", "62044", "2250", "8631", "41360", "481", "2286", "22215", "59296", "71932", "61588", "27124", "61770", "53606", "18120", "57431", "64803", "68199", "73034", "20785", "48080", "26935", "52465", "70118", "42462", "41525", "53965", "14346", "8561", "70257", "19745", "46939", "9354", "46349", "27129", "19892", "25514", "40388", "44127", "4381", "38558", "21989", "19703", "70592", "44031", "53632", "29634", "22952", "13318", "68577", "23529", "20373", "5655", "64482", "72380", "31515", "53259", "45547", "32982", "48375", "70614", "31194", "35670", "68371", "53592", "54178", "71395", "38573", "41323", "17409", "33202", "28771", "4577", "18264", "8849", "55764", "959", "11439", "38851", "65669", "12068", "69326", "50176", "16183", "41759", "51767", "42119", "27747", "54803", "47074", "63447", "21036", "67303", "64887", "21583", "70138", "32268", "57800", "68812", "65892", "30420", "13136", "20070", "36", "58721", "49285", "25494", "72353", "45701", "42162", "4229", "39903", "15336", "22963", "47349", "68238", "26305", "6508", "60470", "46174", "27729", "70724", "64187", "1926", "172", "44280", "31545", "4651", "66312", "5374", "13707", "12666", "6131", "68848", "18020", "20696", "41098", "32811", "48776", "62723", "8190", "21005", "5558", "7783", "73026", "61830", "27673", "43110", "29026", "24037", "53473", "15751", "43489", "22195", "8306", "37520", "44839", "27567", "60049", "35321", "48542", "56837", "42714", "43052", "55340", "19031", "41349", "20266", "43801", "390", "8783", "5311", "68527", "23608", "4180", "54209", "9206", "50792", "27836", "20114", "49129", "66006", "50645", "37955", "16111", "25790", "19069", "47977", "14872", "22334", "56622", "17570", "43935", "2714", "28141", "53268", "59693", "52896", "23147", "35188", "70755", "62556", "7369", "16309", "62869", "52730", "57935", "69907", "8871", "29108", "5476", "48019", "60201", "67020", "65340", "7315", "15781", "21719", "25594", "15465", "37490", "45033", "72567", "33653", "9709", "53126", "38578", "32828", "73065", "30591", "71082", "26113", "18168", "36876", "35367", "313", "39542", "72374", "28790", "16761", "73057", "35040", "58324", "65368", "8782", "41397", "62983", "55618", "42858", "37445", "72057", "70391", "57753", "8039", "24819", "46746", "48821", "17749", "42141", "52442", "41181", "27688", "938", "3749", "23156", "54919", "55095", "55250", "33328", "11148", "72648", "40124", "8292", "54535", "25970", "19483", "48718", "28402", "29239", "53077", "64514", "58281", "2105", "67208", "2224", "55257", "44505", "50877", "64310", "37963", "62364", "40459", "65232", "66109", "71535", "21570", "61077", "8482", "15759", "56155", "20218", "20207", "30471", "22044", "36758", "57239", "20981", "60340", "16867", "31022", "58021", "14073", "60813", "29319", "56522", "67271", "18614", "36192", "43515", "19367", "23463", "15915", "33677", "42282", "66525", "32800", "60141", "52100", "63149", "9061", "67404", "67308", "16143", "20454", "53758", "4643", "65453", "23544", "16711", "3866", "40460", "2032", "9289", "60472", "18365", "69177", "70282", "13871", "33517", "57600", "8616", "30783", "47345", "70237", "35296", "58069", "66360", "41897", "22639", "66442", "21081", "39187", "34948", "32428", "25754", "31835", "28365", "26912", "25859", "21334", "14921", "18360", "17193", "72759", "10863", "55320", "36368", "37907", "63872", "9065", "46676", "27505", "31747", "51948", "57196", "5268", "8222", "27221", "16271", "47794", "23897", "13399", "55379", "21514", "12640", "21526", "29684", "6242", "13487", "16112", "32946", "16622", "53459", "30547", "21951", "40987", "40359", "22614", "58091", "11363", "14861", "47375", "62306", "8937", "21507", "9162", "25908", "36908", "71618", "65255", "48319", "59618", "30065", "4484", "31620", "37687", "41773", "64498", "29193", "15559", "49259", "7305", "30084", "49641", "32484", "40898", "1571", "29869", "19848", "62338", "38833", "818", "58626", "64518", "7975", "44493", "1373", "26530", "14415", "37062", "51408", "22661", "53677", "8551", "56069", "57475", "67590", "26513", "48505", "10259", "12267", "242", "27786", "27711", "1625", "9016", "42852", "46972", "1979", "68121", "70020", "27065", "69128", "4453", "52008", "60998", "23561", "69083", "40332", "73102", "38839", "12025", "61908", "51547", "13661", "57148", "27619", "48154", "6483", "64084", "3850", "8846", "62749", "8767", "37413", "5036", "24163", "69969", "50515", "7817", "57854", "24265", "41972", "11706", "66551", "35685", "27677", "66091", "5821", "71933", "17726", "60526", "13808", "23562", "52899", "36673", "32932", "27966", "29212", "57602", "62069", "17623", "15770", "25465", "70588", "41520", "10775", "69355", "53998", "49205", "26101", "57875", "30825", "4531", "26482", "36585", "51349", "42422", "1247", "42032", "62093", "72093", "59374", "46581", "13490", "52631", "20161", "59020", "71870", "4166", "35765", "46836", "49058", "23338", "52805", "61778", "29628", "36228", "49268", "23107", "5395", "45282", "60816", "64389", "46640", "244", "40521", "30360", "54412", "45669", "37055", "2424", "65082", "71611", "22801", "26565", "21541", "59024", "32579", "22917", "45121", "9585", "65152", "5233", "19182", "46912", "68173", "61044", "33854", "5593", "15777", "64950", "49703", "45051", "69058", "56344", "28142", "32111", "26179", "8290", "31389", "27758", "585", "64148", "28710", "56529", "65321", "28916", "12152", "67227", "1160", "12472", "10743", "24644", "63939", "34864", "44479", "18720", "39216", "14463", "72063", "52825", "71330", "57685", "51338", "23464", "46817", "39137", "30062", "2890", "47650", "58714", "34246", "67389", "36451", "55132", "39414", "21565", "10778", "63258", "19193", "16514", "57993", "64989", "58594", "62643", "5034", "43580", "46279", "4098", "5763", "43864", "17173", "29297", "21211", "26000", "12843", "57344", "71916", "35962", "21990", "25800", "58777", "51303", "455", "4870", "55569", "8067", "71114", "52144", "48695", "43523", "7473", "25845", "27854", "57563", "34526", "34778", "64457", "66498", "16121", "10052", "67052", "73249", "63821", "67481", "3538", "11070", "72957", "43453", "64497", "55736", "66240", "34528", "64342", "326", "37408", "10328", "62878", "38630", "74", "7788", "18917", "22221", "8098", "48874", "21107", "60904", "2367", "54989", "27371", "20906", "32112", "17214", "59508", "32585", "47018", "8188", "48524", "37960", "40469", "39676", "27396", "14849", "52486", "24467", "4905", "31709", "48274", "59119", "14356", "70983", "18411", "40434", "71606", "34003", "18481", "46682", "33441", "21161", "52483", "17835", "24747", "10317", "25993", "10700", "38262", "25774", "51281", "25316", "38464", "50605", "44053", "32634", "1303", "16574", "65357", "72415", "59944", "32475", "35742", "50721", "1706", "65188", "56564", "37946", "49391", "6421", "22968", "18067", "63926", "49948", "30181", "33270", "57070", "208", "34704", "60781", "30543", "60244", "38994", "26059", "68434", "70892", "26125", "32149", "24766", "41752", "18716", "45847", "41590", "8894", "71081", "12451", "62809", "32450", "30225", "58614", "41049", "40371", "50528", "52777", "54965", "51727", "12748", "62828", "68396", "36484", "69526", "49859", "31536", "8834", "23956", "45650", "72481", "67494", "6346", "55494", "68715", "820", "25716", "7994", "17939", "72124", "17620", "67537", "72828", "60311", "5513", "72261", "71172", "55349", "56984", "29456", "1475", "12598", "17646", "27427", "20090", "59666", "45629", "23301", "50564", "11685", "9041", "47535", "31933", "20097", "40623", "24652", "17722", "5771", "43001", "45496", "72369", "31524", "24930", "35469", "29359", "47948", "36377", "18393", "33234", "24662", "12616", "43934", "30708", "44621", "14506", "45114", "67913", "42045", "56664", "8334", "57334", "6001", "56577", "13021", "35992", "55507", "27607", "11051", "16139", "52181", "33436", "21475", "49104", "30299", "66963", "49462", "68270", "18895", "53274", "15458", "30064", "6641", "66289", "53060", "66009", "32170", "39165", "31947", "12104", "34794", "2094", "3306", "7871", "22078", "21446", "57964", "16963", "59363", "69291", "14675", "18344", "70986", "36523", "53073", "41185", "41332", "58159", "12884", "3709", "42622", "10306", "67984", "51141", "72476", "55597", "536", "22250", "33169", "4401", "23234", "46745", "6998", "13240", "18691", "32210", "37396", "8022", "22867", "53006", "19330", "68062", "9500", "29110", "7492", "10016", "14657", "71599", "50685", "27165", "35618", "45505", "44487", "44491", "71551", "20492", "16913", "38891", "6884", "61315", "24102", "11012", "21274", "63048", "61689", "52723", "3118", "71927", "59328", "44657", "8327", "58993", "61412", "70740", "32737", "8642", "19468", "56614", "33812", "56018", "32761", "14766", "42763", "63839", "61926", "51733", "34963", "37624", "29539", "26910", "53308", "5642", "70637", "55975", "30845", "17863", "49522", "42421", "44733", "8257", "67472", "52873", "51475", "288", "17183", "3595", "2001", "63521", "22305", "15044", "16165", "18960", "31636", "43725", "64512", "59938", "73097", "10982", "61114", "3981", "67339", "32498", "781", "18837", "41276", "57046", "24256", "54847", "7208", "7577", "41441", "58273", "69181", "67471", "10001", "53218", "1673", "48632", "17533", "37517", "26697", "21202", "69787", "3332", "47698", "43615", "58377", "40007", "17391", "11645", "22895", "57276", "46425", "42048", "68114", "4469", "10012", "45356", "59278", "56134", "36457", "2128", "1732", "31606", "59648", "14016", "63261", "55929", "10513", "10946", "61236", "31259", "61091", "73209", "789", "4065", "31789", "61495", "68557", "54174", "69345", "48769", "59108", "67936", "51660", "68674", "24844", "65978", "19622", "53194", "18212", "57825", "4465", "14866", "50055", "28341", "40637", "1464", "13172", "35753", "30730", "25769", "44267", "8821", "64992", "34608", "41776", "38257", "14382", "11925", "17141", "1827", "58677", "61379", "23127", "31434", "12351", "24239", "65967", "15867", "60210", "7674", "11544", "51550", "21589", "36825", "25743", "29249", "18171", "23537", "55005", "31188", "44109", "3250", "66358", "61664", "19447", "15370", "11759", "16763", "73123", "44920", "31251", "48931", "6374", "20897", "3996", "71718", "23426", "47679", "6635", "11852", "12746", "2279", "25926", "61934", "18560", "51484", "13444", "4647", "17731", "9938", "34734", "61525", "7639", "40667", "60655", "45543", "21929", "48150", "8688", "58815", "57300", "49296", "67713", "34328", "48961", "61941", "36051", "71865", "71299", "8677", "7531", "41031", "60812", "30325", "46071", "3409", "31500", "17373", "35239", "32012", "56213", "44592", "38927", "53249", "68091", "62929", "52624", "21728", "56802", "61895", "62399", "67698", "12690", "52052", "12208", "43108", "66777", "73048", "44145", "45351", "13266", "55574", "20892", "41944", "65780", "39236", "8560", "67312", "25531", "28639", "54226", "60907", "34508", "29436", "51837", "12290", "12895", "35703", "16625", "53506", "9436", "6215", "69136", "42992", "61107", "46075", "20288", "33453", "36790", "27834", "26152", "42676", "65468", "33014", "20249", "60973", "47543", "23989", "5661", "72254", "26129", "71533", "50698", "56373", "26407", "9226", "6169", "14588", "11680", "40120", "16735", "16170", "67253", "15099", "57363", "57979", "63247", "38153", "37074", "31700", "44557", "24890", "25543", "30238", "20670", "67347", "35053", "26494", "21089", "72507", "4426", "31217", "47183", "22704", "47031", "56601", "61075", "5728", "45876", "17766", "42062", "4717", "5952", "54976", "8285", "17027", "34361", "29753", "64054", "34207", "62686", "10667", "19324", "67474", "31775", "19079", "7901", "25070", "25319", "25408", "55385", "45645", "33763", "34155", "43564", "9827", "20357", "3780", "14732", "2254", "40489", "27076", "47356", "39902", "57124", "31960", "60374", "70216", "2631", "67823", "5406", "47239", "22452", "20691", "17004", "34801", "51959", "11017", "32859", "241", "8154", "16334", "577", "46629", "66564", "65899", "14205", "30798", "16078", "41983", "72300", "9681", "54556", "42736", "59017", "20012", "12244", "70011", "11635", "32767", "10780", "24430", "72682", "11897", "59980", "57084", "11276", "42989", "23543", "25999", "36598", "67888", "27621", "11836", "49220", "30813", "55735", "71530", "40775", "16021", "12673", "55230", "29641", "17438", "55046", "60613", "38471", "22833", "43702", "41655", "33963", "61993", "60978", "15993", "23859", "540", "32921", "69624", "19281", "49909", "45443", "29413", "35721", "27015", "37679", "70145", "59483", "4188", "16796", "63485", "42180", "56908", "52734", "56328", "67251", "12752", "31817", "1262", "3973", "39182", "56308", "60170", "43721", "70995", "45056", "69603", "27126", "31540", "43078", "5522", "29547", "32324", "9378", "17240", "21610", "9248", "17505", "32089", "42654", "67259", "70882", "13629", "4659", "31718", "2731", "22431", "36459", "36535", "54054", "17023", "16870", "54428", "21549", "71121", "63465", "3777", "56735", "63776", "12031", "6750", "13726", "34371", "11474", "62707", "34019", "16909", "30521", "34138", "10075", "36745", "33245", "70566", "67697", "30709", "25559", "3439", "2954", "5942", "8204", "25617", "21124", "28760", "6834", "71570", "8747", "63205", "67791", "9545", "60562", "64313", "43727", "30224", "65598", "18200", "56541", "54201", "58664", "1462", "58952", "30075", "52325", "43027", "22063", "55545", "68809", "72000", "926", "63378", "38766", "22446", "35863", "29245", "40404", "2132", "57095", "44445", "20746", "234", "14247", "11837", "60505", "7533", "14683", "72265", "54582", "20005", "29669", "16667", "42369", "7203", "4160", "28636", "58031", "17241", "56648", "49570", "36848", "32835", "7375", "51778", "39090", "41709", "59594", "14341", "6144", "28715", "7336", "55258", "39211", "56969", "59037", "6069", "60022", "24801", "21503", "72025", "26519", "48565", "10191", "21888", "60818", "17827", "4055", "7913", "24989", "49776", "40227", "70274", "58313", "47791", "19852", "39512", "31077", "61175", "42153", "51802", "45937", "37976", "12854", "71225", "52341", "37666", "3004", "32586", "34259", "3670", "11725", "8282", "52992", "6526", "30438", "24565", "44395", "45010", "496", "51653", "59528", "14003", "27385", "8099", "39425", "36171", "69436", "54357", "15084", "60011", "27369", "52056", "5207", "12384", "24678", "55093", "5525", "65558", "48317", "44602", "44219", "40773", "15357", "8544", "34205", "16741", "13638", "64263", "45310", "3664", "53447", "25613", "8545", "4319", "70727", "40325", "43373", "72802", "34044", "59139", "52313", "64285", "9575", "57781", "28061", "27772", "47409", "755", "70736", "55755", "51926", "11599", "20952", "9893", "53659", "14482", "11053", "61211", "70974", "66075", "28193", "49485", "54849", "63960", "11189", "32725", "2266", "67361", "49978", "554", "65757", "30969", "39015", "19157", "54177", "27292", "58133", "10356", "23708", "28392", "19272", "10068", "51039", "45421", "35096", "29464", "66244", "37811", "8568", "38862", "33212", "37332", "66079", "38357", "12944", "35652", "61754", "36474", "53412", "4768", "61538", "35905", "29682", "32680", "63025", "63144", "56572", "17692", "34970", "61309", "14868", "14558", "46960", "16424", "53032", "62009", "72052", "25899", "43129", "46592", "45462", "4548", "22994", "7809", "35309", "4431", "1773", "12261", "38097", "5237", "3013", "54411", "1088", "55248", "72491", "50873", "31687", "5856", "5328", "53807", "21051", "58758", "21025", "49102", "71536", "57233", "22140", "54954", "49743", "15345", "20235", "12226", "18384", "40086", "15351", "29295", "60645", "48991", "67288", "58100", "28850", "57594", "36009", "63734", "26600", "21513", "61411", "37786", "24169", "34721", "65555", "60053", "59954", "63855", "68332", "68225", "38084", "32364", "27990", "29363", "59660", "7640", "10405", "28085", "4195", "27413", "14071", "19133", "59052", "45318", "66386", "72255", "18607", "20455", "33358", "29644", "46790", "48385", "51673", "64784", "23959", "40191", "52616", "68356", "59817", "15768", "36551", "14146", "33694", "55080", "56221", "13821", "44582", "60669", "9365", "8760", "36819", "6553", "720", "43388", "73242", "37160", "50506", "39504", "39538", "59712", "43491", "8453", "62536", "18101", "13833", "57592", "36952", "8367", "17893", "71740", "30129", "57113", "21020", "72947", "46904", "69073", "33580", "3687", "65337", "7332", "29499", "30357", "66625", "39179", "69704", "2074", "69835", "42942", "71380", "57063", "7025", "39189", "63011", "4599", "33481", "27363", "17504", "50626", "18881", "65377", "47598", "69434", "56668", "65303", "14411", "67707", "1791", "41846", "30631", "61480", "4684", "3185", "12832", "16301", "54571", "57590", "34243", "21699", "35154", "16326", "7732", "9733", "46116", "17139", "14406", "40083", "61819", "37678", "34052", "17161", "67060", "17078", "64097", "64413", "16724", "57001", "12183", "1444", "31480", "14108", "47263", "38359", "10354", "4443", "72420", "55100", "73154", "40526", "56525", "51490", "68090", "36047", "3584", "36275", "38188", "54246", "30221", "18253", "65133", "59399", "48020", "31893", "62466", "72637", "20947", "62261", "5742", "13736", "60233", "70615", "1456", "10830", "28250", "23594", "55066", "39847", "27457", "7594", "35103", "58058", "39073", "55242", "29112", "69122", "48099", "48231", "15078", "33352", "30047", "14805", "60181", "24232", "54982", "62372", "19259", "39143", "73165", "40089", "63196", "28138", "30995", "50652", "36956", "17090", "15560", "57667", "34517", "58702", "26943", "24550", "20044", "66159", "43242", "61850", "35539", "49590", "27565", "46732", "15686", "31270", "49275", "10937", "70788", "55633", "36805", "34736", "34501", "28619", "8865", "52391", "61141", "52014", "4911", "53055", "68272", "23469", "58193", "53577", "50499", "13147", "42432", "58971", "58812", "7163", "67970", "21429", "8951", "53374", "66670", "22243", "20271", "7637", "9524", "6271", "11761", "37215", "4397", "60724", "2068", "6433", "70942", "70009", "51351", "73095", "68548", "54526", "32862", "21788", "61235", "70215", "36211", "47030", "64769", "35974", "50016", "30206", "1398", "49073", "62801", "18310", "16750", "8407", "58046", "13022", "57990", "8343", "63558", "48814", "28651", "11698", "55268", "31539", "27420", "12223", "49594", "18516", "4010", "9586", "62142", "57037", "55276", "13813", "66458", "68486", "49242", "42399", "17808", "60105", "38518", "27378", "61877", "64255", "39945", "52502", "27681", "15947", "16710", "60502", "17759", "35000", "4476", "24492", "51531", "49410", "5973", "41159", "26342", "31684", "27384", "33832", "25438", "56845", "69561", "28009", "35445", "72393", "4961", "11542", "70053", "21012", "335", "1891", "42767", "12279", "62182", "35299", "56651", "56602", "35277", "45417", "37738", "44514", "52194", "72137", "49705", "11877", "39348", "64979", "34440", "53449", "68756", "22500", "12235", "23356", "49661", "21959", "10944", "8046", "11022", "50472", "36111", "3815", "1644", "31198", "11763", "643", "21769", "48942", "44161", "67975", "20382", "46852", "69811", "15767", "51429", "48910", "24349", "59655", "3634", "964", "9723", "37663", "14939", "42593", "41063", "45868", "62911", "6002", "68208", "31847", "66426", "48907", "41164", "17435", "66220", "11018", "27945", "21792", "56608", "4303", "21117", "67121", "51168", "63266", "11974", "9380", "7274", "10071", "1478", "15584", "17392", "8428", "30848", "8109", "69022", "6607", "72073", "49210", "43569", "7510", "58218", "63985", "35381", "15971", "14158", "72784", "13060", "15137", "28553", "18941", "29344", "31114", "38479", "43131", "5835", "22973", "71642", "65781", "27227", "39905", "12108", "30563", "44206", "25226", "49812", "32643", "36178", "58136", "11597", "49830", "11739", "14726", "3237", "59111", "63410", "38183", "30058", "2849", "59656", "48709", "36560", "17126", "15891", "6413", "18334", "28936", "29256", "20854", "55119", "24752", "12830", "68249", "48153", "71024", "1324", "27422", "42626", "32241", "7461", "33952", "63019", "639", "31375", "38551", "10451", "34179", "5013", "33102", "20109", "38773", "32059", "50422", "64372", "30892", "26862", "16726", "56093", "68603", "70946", "42444", "7896", "26029", "37227", "41589", "37734", "41102", "51274", "29978", "19516", "65659", "66810", "39733", "40368", "2115", "29920", "43243", "12147", "58269", "1430", "64442", "4568", "56556", "66785", "42964", "5393", "5752", "17929", "51797", "68826", "22113", "62371", "25015", "39832", "44875", "44620", "23440", "24808", "56910", "33448", "63561", "12822", "47758", "48737", "57521", "66252", "28357", "7573", "13648", "43742", "42508", "63155", "13093", "43273", "24255", "32657", "39460", "72897", "66041", "67338", "70219", "30743", "47452", "25312", "28460", "22481", "63934", "68044", "20144", "64486", "9400", "43484", "45728", "35819", "21489", "37581", "60792", "44006", "32390", "29933", "54774", "1168", "21535", "69629", "3255", "17576", "68622", "8349", "64495", "70886", "38941", "63609", "20763", "3868", "53151", "34032", "59070", "64694", "25575", "38805", "12336", "36243", "56656", "66154", "45307", "43378", "15091", "46490", "11918", "43686", "57792", "6678", "11969", "19748", "9877", "68133", "11976", "4080", "383", "62708", "72972", "41665", "59184", "22184", "41479", "34828", "7307", "22476", "55694", "51063", "69119", "4690", "70304", "71767", "20561", "30820", "24884", "52999", "30090", "56139", "48640", "68771", "33938", "22091", "59161", "52037", "36183", "9120", "30179", "9323", "449", "72717", "49981", "8571", "47259", "21118", "61643", "15341", "35807", "70010", "5928", "49107", "28494", "23407", "59859", "38728", "25835", "1983", "20700", "20949", "36808", "19693", "15619", "47665", "50254", "11346", "41798", "68804", "40443", "24833", "53276", "32525", "72326", "49200", "48589", "42125", "4743", "4566", "73009", "54624", "32697", "25030", "27585", "10644", "34714", "72847", "52382", "55652", "5795", "9305", "47195", "56278", "1781", "29739", "16043", "39117", "10504", "31530", "39613", "21405", "63472", "45632", "18331", "14214", "62964", "58019", "12297", "69419", "22989", "17118", "57371", "34779", "26990", "10464", "72981", "3772", "13121", "36334", "33965", "67544", "35485", "53306", "40664", "4990", "14681", "48039", "21612", "42971", "5578", "56460", "29629", "61765", "36046", "60117", "27845", "35933", "52024", "69292", "65329", "47019", "25706", "7949", "39285", "66111", "39061", "8778", "58970", "42150", "56534", "42742", "57507", "34822", "5812", "7377", "15690", "13958", "72676", "19819", "8410", "54581", "53504", "11885", "23019", "47688", "26568", "39057", "6437", "3603", "13170", "67038", "24542", "72590", "39657", "25423", "1607", "9712", "31930", "44523", "71563", "63992", "51783", "17330", "54538", "19130", "50250", "1442", "38227", "13684", "46879", "12331", "54525", "14631", "55562", "24057", "57502", "3533", "57948", "19363", "2821", "54449", "61621", "60849", "72450", "58848", "4674", "17891", "2930", "42174", "17364", "19189", "15284", "5925", "42738", "51216", "15943", "46055", "35601", "52923", "33254", "26115", "72660", "444", "51154", "56452", "72930", "56563", "46462", "61165", "67844", "8087", "65708", "53572", "51895", "9568", "47645", "62514", "2278", "50339", "33142", "1578", "68714", "61273", "63218", "24104", "31965", "71674", "10702", "24653", "69453", "10180", "65447", "37973", "57265", "7012", "40420", "47865", "5692", "62948", "17186", "57240", "12808", "10855", "19934", "14054", "20600", "9009", "28057", "39770", "44103", "3808", "62296", "63650", "38883", "17544", "45059", "64152", "59758", "69974", "37591", "23699", "54580", "5330", "19027", "33267", "6934", "72290", "53859", "56241", "47312", "40055", "38827", "59855", "42910", "20911", "42612", "22749", "58368", "67616", "71772", "37726", "245", "38949", "57783", "50015", "53812", "48872", "67024", "61871", "11321", "18859", "17473", "23251", "21900", "5146", "29895", "29004", "60868", "54628", "49191", "23759", "52157", "72539", "34129", "20757", "20708", "36572", "58220", "57409", "20762", "49448", "4849", "60886", "2404", "2417", "68642", "36098", "68662", "13188", "14666", "43532", "3931", "56024", "38004", "36556", "69283", "20247", "53800", "27605", "32935", "59104", "42973", "18850", "58997", "39510", "22039", "31117", "46793", "64696", "37167", "10530", "69752", "59741", "11416", "73255", "59453", "73083", "56127", "72152", "45542", "21770", "60129", "12257", "3056", "41865", "19826", "55301", "59646", "3767", "22105", "30569", "458", "27670", "5190", "44810", "36073", "13252", "20286", "4428", "246", "2794", "19146", "49170", "60874", "60136", "41086", "48250", "2357", "68874", "29307", "60758", "6166", "72358", "39113", "1308", "29546", "23132", "58367", "2241", "47912", "1539", "44079", "19971", "51412", "61564", "61921", "64586", "12816", "32718", "5387", "66188", "42420", "64073", "25793", "66117", "29818", "35858", "20991", "39582", "27696", "70675", "52791", "16499", "16539", "50511", "15995", "49261", "33994", "68875", "28757", "8462", "50277", "13125", "36629", "10172", "59584", "62587", "37562", "5961", "22089", "62388", "5791", "23926", "66260", "59661", "15702", "9735", "5458", "1033", "51050", "65034", "11693", "30324", "28129", "37816", "54163", "1861", "7349", "43495", "41139", "20641", "14134", "21932", "45428", "13579", "34437", "47869", "41120", "840", "23496", "52979", "60122", "31196", "28131", "20656", "45519", "43713", "63970", "55973", "27890", "62998", "72462", "38878", "41847", "71227", "4777", "27367", "63591", "13410", "41855", "24449", "68526", "16878", "11563", "64707", "8426", "40494", "63325", "61060", "37658", "38791", "35513", "61936", "18605", "23386", "9569", "47372", "38867", "631", "58144", "1103", "36191", "26619", "395", "34158", "47053", "46566", "61321", "63036", "36960", "4704", "70209", "26216", "40880", "46176", "72102", "11404", "23058", "41543", "11529", "55860", "812", "66991", "32708", "37949", "41840", "26304", "29538", "70148", "18474", "66692", "12995", "41455", "41373", "56209", "4447", "23167", "59991", "19435", "12543", "14177", "43851", "71578", "10391", "50065", "21082", "39758", "38289", "50697", "8295", "61519", "63878", "34482", "44903", "17828", "40222", "35892", "15037", "17901", "13389", "39625", "29723", "32637", "66368", "26105", "48715", "71513", "43786", "55164", "25784", "16265", "5840", "42498", "70431", "34683", "58418", "36549", "42154", "34363", "72610", "60263", "8011", "60744", "5912", "72143", "20706", "37121", "57904", "70577", "49083", "67940", "61115", "58476", "7847", "68718", "20213", "55827", "53657", "19290", "26643", "33311", "801", "15306", "51135", "28210", "53708", "41112", "10536", "7102", "66745", "35800", "1918", "42737", "15940", "72623", "68201", "17833", "15633", "13259", "58738", "62269", "9007", "69250", "12975", "46854", "70417", "14154", "39938", "62349", "22652", "23387", "46726", "55942", "56010", "59721", "58227", "21388", "19430", "63870", "28502", "57433", "16171", "62986", "59290", "32992", "52350", "12254", "48612", "29037", "59679", "38687", "56609", "47365", "4464", "34556", "29096", "11161", "4418", "32731", "58124", "68939", "24081", "73253", "49469", "24357", "28137", "46807", "1754", "32088", "17403", "55568", "49839", "43462", "24343", "49544", "67183", "60373", "8797", "42249", "10506", "889", "50723", "65586", "18937", "64462", "67502", "72807", "20148", "69589", "37524", "58277", "33986", "35033", "1029", "39734", "37567", "47637", "30173", "5563", "32834", "28218", "45296", "36921", "26984", "20484", "17879", "36034", "46561", "40953", "2901", "45606", "57509", "18692", "20497", "46985", "46257", "28019", "45089", "34836", "420", "42230", "43079", "49258", "56396", "21954", "42305", "13411", "57071", "57175", "70901", "17951", "71010", "12793", "35582", "66734", "53600", "6789", "19813", "10101", "38127", "57757", "2571", "16317", "8626", "37322", "53154", "18292", "5990", "7489", "55588", "28094", "51999", "72547", "65099", "48535", "67925", "46455", "25870", "48", "72470", "55244", "50494", "25752", "26374", "32225", "1031", "10264", "2838", "684", "3417", "64404", "15253", "26380", "63171", "51158", "43250", "34089", "20351", "18187", "4525", "62836", "37397", "14608", "43152", "49134", "14399", "38221", "23149", "64502", "70873", "29075", "55870", "40103", "18109", "54102", "40819", "62118", "61528", "36030", "45992", "27795", "31404", "1761", "52817", "43036", "65893", "37111", "72609", "32724", "2598", "7907", "26399", "1098", "1859", "47105", "71898", "8933", "69908", "61006", "32030", "14430", "52240", "64188", "32844", "6792", "31115", "25747", "55133", "47494", "10614", "58790", "13800", "52324", "52101", "34291", "35934", "9262", "18590", "40463", "61776", "10976", "32507", "9898", "4970", "32141", "15648", "66675", "43260", "23323", "27593", "69992", "53738", "27031", "66141", "46033", "15811", "20390", "32105", "58183", "49551", "43447", "40480", "18293", "661", "41912", "16855", "6984", "50614", "36799", "14256", "11305", "8686", "877", "24984", "23348", "3908", "39275", "19885", "44177", "47712", "29921", "51981", "34583", "13928", "33527", "26590", "18057", "49863", "26016", "39148", "61587", "63749", "7431", "345", "45285", "8152", "62675", "29450", "26622", "70184", "58223", "37250", "47836", "64330", "40142", "59531", "42026", "44594", "13434", "29152", "53201", "42821", "26485", "29169", "16529", "67005", "28536", "28000", "20666", "46249", "35066", "71057", "54806", "59925", "45021", "70957", "61032", "43239", "64640", "69455", "12478", "27202", "1587", "21476", "883", "27045", "51899", "1014", "53670", "69897", "38474", "55302", "19867", "73053", "35994", "68556", "55386", "33622", "72342", "31350", "50349", "64377", "1629", "39588", "28310", "29308", "7807", "58531", "20469", "31394", "40275", "2026", "67174", "10098", "62424", "41078", "58213", "25871", "55288", "17689", "24399", "60967", "72856", "19407", "60840", "38587", "23553", "4012", "56698", "40214", "34231", "35757", "70396", "60732", "13694", "70462", "47681", "9399", "39749", "34312", "195", "37007", "58344", "27407", "60456", "43846", "63557", "9637", "25818", "42083", "30843", "20224", "25037", "62721", "48050", "62434", "27486", "70878", "2088", "43161", "49155", "47620", "72232", "24894", "46277", "60798", "70683", "22050", "5739", "19904", "21171", "26707", "41209", "72914", "21850", "56421", "62575", "68782", "64763", "17093", "29745", "36953", "429", "23126", "57301", "47703", "52055", "62228", "7666", "71026", "34542", "40616", "29201", "56591", "20276", "20400", "33906", "69816", "10386", "45143", "42451", "10794", "45308", "5535", "26283", "34800", "37481", "22032", "25161", "11784", "12116", "71066", "35382", "7927", "66118", "61276", "6990", "43710", "13970", "24315", "71731", "15608", "69445", "31845", "34277", "46672", "48766", "13918", "3174", "59342", "42874", "55922", "2481", "2459", "25436", "18638", "68781", "42237", "33437", "67790", "44714", "47217", "60817", "31815", "17039", "12461", "56039", "64", "69072", "15243", "56744", "57302", "19555", "26362", "56016", "22962", "34264", "60547", "61876", "29595", "19720", "4569", "56830", "49233", "12504", "55698", "27944", "59087", "13492", "65828", "55758", "31979", "16971", "39688", "65254", "47462", "15164", "12982", "9776", "59378", "61563", "55745", "47870", "24763", "46831", "70479", "14754", "4175", "2935", "27755", "23912", "34460", "45292", "9836", "26672", "8749", "8287", "58066", "24995", "15391", "5560", "68200", "57348", "48329", "73153", "5544", "62927", "56596", "44740", "15840", "53729", "15673", "16822", "58540", "2343", "53188", "66540", "3100", "14719", "55768", "69724", "72033", "30194", "5017", "39449", "7213", "49998", "59633", "51263", "21904", "49521", "12775", "42403", "27912", "73010", "39587", "46925", "64568", "24593", "40780", "33538", "48454", "19110", "2734", "69728", "52312", "64100", "71856", "70567", "5137", "61249", "24882", "39361", "33404", "37533", "33744", "23419", "71192", "51306", "12780", "25113", "41880", "70511", "16076", "6344", "21906", "34048", "67833", "33972", "29407", "47985", "37043", "5854", "58359", "22881", "43217", "25120", "41489", "37835", "11923", "36683", "9270", "44800", "57864", "10403", "17537", "37234", "47336", "3075", "59609", "1535", "53689", "67560", "16946", "1406", "59578", "17158", "44435", "29080", "56994", "2193", "45253", "56248", "8663", "25510", "69492", "51986", "56228", "43505", "25173", "71108", "5275", "30773", "25938", "70900", "64379", "29643", "45484", "15590", "7125", "62070", "14020", "62577", "3658", "10393", "59178", "29106", "60584", "34399", "30273", "5227", "42879", "49446", "8279", "2441", "36167", "66550", "4250", "36942", "4314", "19974", "9450", "2020", "17698", "52792", "51084", "48553", "24022", "52645", "60990", "22909", "43253", "25662", "53341", "39069", "66477", "11945", "52563", "47592", "4586", "49912", "17402", "51373", "45355", "56680", "46", "33308", "22600", "58571", "62242", "25205", "71443", "16922", "63640", "47094", "4915", "13497", "20128", "49375", "2729", "71848", "70742", "29980", "7970", "39129", "60503", "40113", "62868", "17859", "6501", "19162", "70507", "40091", "72714", "16250", "14586", "70761", "42990", "1733", "32511", "73002", "25729", "3520", "4009", "65373", "42105", "10129", "16819", "62793", "8134", "62126", "46559", "36272", "4081", "32070", "30794", "40225", "11205", "57949", "58109", "31525", "15670", "23628", "61998", "5347", "70622", "63907", "32720", "28455", "13086", "26331", "49076", "29370", "59219", "5644", "24999", "69344", "50501", "44626", "68203", "44565", "68602", "52172", "8361", "44225", "4044", "52013", "65422", "54417", "67033", "52159", "26247", "54027", "46965", "50294", "16402", "12544", "27437", "61483", "12456", "67491", "66300", "63216", "27750", "4246", "35245", "58469", "35552", "2818", "36215", "26261", "63231", "53397", "28589", "13473", "68694", "50839", "28765", "2373", "57897", "14619", "69183", "47033", "22079", "6143", "71336", "37637", "39080", "64977", "52068", "66917", "67800", "41867", "33200", "38109", "59330", "36448", "12233", "30152", "24118", "65090", "15145", "42374", "65361", "30512", "6108", "7770", "38270", "22660", "41109", "44691", "17426", "61461", "22053", "44713", "25918", "15070", "72164", "18709", "64779", "23209", "60111", "46183", "54251", "25648", "21497", "50623", "713", "21764", "18402", "54786", "40690", "8061", "63434", "9696", "36045", "63154", "40632", "55691", "31554", "43199", "45921", "64257", "34417", "60423", "34464", "25883", "49489", "9908", "51865", "43606", "44196", "24362", "38620", "58645", "9603", "6375", "64615", "12088", "9983", "31221", "50019", "13730", "68214", "44596", "47181", "28271", "5001", "60050", "38593", "65692", "24090", "33145", "69732", "23686", "7548", "41386", "64455", "59163", "64751", "61843", "51934", "63523", "11008", "4446", "13550", "58114", "9729", "69068", "25249", "41267", "23620", "70598", "35843", "15443", "56755", "28283", "50914", "881", "20811", "38410", "23586", "70266", "67576", "63813", "15377", "8588", "17858", "55029", "33886", "966", "8554", "55260", "13836", "39329", "25352", "49329", "11387", "54157", "48521", "26839", "10841", "25603", "58830", "36781", "46432", "16013", "49549", "34122", "72659", "37902", "71991", "58374", "29362", "2918", "70521", "68024", "360", "45943", "69485", "30699", "1258", "32132", "12834", "65153", "13484", "51848", "2482", "28328", "17878", "11913", "25324", "61691", "16752", "50282", "34185", "70505", "22972", "53680", "59011", "57328", "10956", "2229", "37909", "52683", "72468", "65117", "15223", "45278", "60298", "42430", "35322", "65826", "41891", "188", "60747", "67966", "37802", "8479", "51396", "46411", "31084", "9533", "59131", "49610", "24835", "52882", "19223", "33029", "64119", "34082", "48866", "3243", "57658", "7430", "38769", "22626", "70862", "56511", "66163", "63476", "29331", "55142", "45471", "30802", "25757", "61439", "51555", "31942", "20623", "2056", "32836", "41322", "3583", "59922", "2452", "54518", "46898", "13998", "36832", "45896", "55298", "529", "28258", "53686", "66843", "31552", "30714", "37760", "14650", "70425", "23849", "34768", "46787", "441", "46409", "33187", "6784", "72546", "11047", "30266", "60983", "38896", "4283", "65118", "14090", "31088", "69698", "28412", "14502", "50069", "59479", "32081", "63333", "59086", "70630", "55638", "45799", "68384", "15287", "43875", "52099", "6125", "49656", "15174", "9550", "55666", "36438", "67929", "50439", "67539", "58511", "58140", "65513", "7708", "59159", "51990", "67986", "67382", "31918", "40890", "5015", "19542", "70771", "15654", "3985", "30074", "62202", "4757", "50826", "68366", "16592", "48497", "51730", "66504", "29671", "54706", "10908", "42955", "1084", "19552", "54601", "73204", "53971", "39104", "3958", "1667", "68878", "18189", "63723", "22711", "4894", "61534", "71247", "56187", "10589", "71726", "19145", "66672", "45062", "11327", "25812", "69658", "65539", "15612", "47055", "68918", "6853", "41297", "50316", "51968", "17896", "22413", "31291", "31638", "37978", "56130", "7331", "45220", "2466", "21574", "53599", "43162", "64754", "37673", "10006", "5252", "64301", "66353", "67660", "10760", "8330", "65323", "58288", "33121", "33373", "67014", "41784", "68137", "41124", "27743", "23948", "65562", "31613", "8892", "34258", "52267", "13015", "26647", "29382", "42381", "48549", "8914", "51527", "62314", "38246", "29166", "34675", "15848", "29068", "32638", "45778", "21596", "27746", "66295", "13004", "57697", "17933", "40312", "40256", "27166", "30899", "33468", "42031", "36447", "15191", "22544", "73162", "69284", "6576", "35696", "24524", "53714", "63516", "23746", "20766", "39899", "62561", "34491", "28874", "21577", "6572", "9142", "22866", "51402", "38717", "55760", "31966", "54620", "21694", "58240", "7868", "34805", "12201", "72744", "18657", "21859", "38002", "63695", "21369", "59880", "6918", "63136", "50699", "61383", "16712", "15107", "21270", "28825", "21861", "44149", "1894", "54012", "14223", "60130", "17530", "60524", "16467", "56509", "61640", "10810", "24860", "293", "33414", "48618", "31871", "47943", "31495", "69282", "42392", "13973", "21665", "43069", "26453", "20130", "14196", "72035", "15933", "26719", "54127", "16488", "33920", "43880", "53994", "17253", "43909", "44340", "26322", "58327", "8143", "24798", "13099", "4400", "16394", "41563", "9024", "65872", "5080", "64395", "70161", "34578", "5240", "13498", "56419", "60890", "23287", "29819", "13713", "4207", "53989", "28458", "72884", "21389", "48180", "961", "20698", "20444", "5016", "53123", "61125", "40697", "41336", "55017", "38947", "47826", "41914", "35494", "34450", "20931", "21196", "39122", "1394", "279", "47399", "19599", "1341", "35847", "58126", "47340", "18154", "23491", "9926", "3482", "31053", "68467", "15309", "40419", "45604", "71014", "57057", "19368", "27783", "40355", "5930", "14721", "26640", "54191", "49415", "16938", "56616", "25127", "31629", "45088", "24786", "16075", "16526", "49647", "61263", "55634", "25792", "58981", "48828", "909", "32080", "32180", "38913", "42996", "29373", "38389", "71587", "53158", "20898", "63298", "65764", "48310", "54950", "53036", "35851", "50039", "2217", "44422", "4294", "30851", "41255", "62546", "50427", "48242", "66331", "44665", "39206", "19115", "14299", "68922", "40476", "3084", "5302", "12022", "48467", "49172", "59723", "66684", "37431", "1820", "25025", "8831", "53353", "62601", "27791", "39083", "46553", "60166", "47784", "16105", "16336", "38745", "30809", "33542", "25199", "24858", "62321", "30526", "33388", "35353", "66847", "55490", "62420", "62392", "29", "55547", "24395", "6063", "30755", "18835", "43635", "62034", "36925", "70858", "44496", "57174", "68447", "20189", "33931", "32774", "36290", "22330", "65103", "35891", "72331", "25134", "45623", "61398", "65884", "44338", "34357", "72921", "39523", "52383", "2052", "9197", "47231", "21902", "72882", "19727", "54293", "34971", "13755", "8439", "21633", "18785", "50272", "27127", "23924", "19644", "5884", "19769", "51652", "29565", "62778", "41529", "18043", "48037", "4715", "9739", "47693", "42791", "10070", "72963", "4040", "31666", "25589", "45440", "47854", "28890", "34908", "2592", "34033", "33756", "69884", "53145", "39949", "9334", "28416", "10110", "59171", "45863", "58823", "71845", "63804", "45794", "27572", "27502", "41477", "4090", "66824", "21000", "7266", "62259", "5814", "9734", "50252", "33996", "19087", "15663", "43179", "31749", "55740", "59015", "65526", "47212", "18572", "21894", "47449", "21017", "8954", "38820", "5303", "5386", "64167", "30865", "26130", "33656", "72843", "71953", "51765", "21723", "20321", "45431", "10606", "13139", "68757", "12940", "23824", "52149", "47132", "2401", "27549", "14417", "5572", "16916", "34398", "9833", "57554", "26570", "71595", "28854", "10228", "13784", "69696", "30793", "39051", "36269", "12968", "9888", "25415", "18025", "3460", "50725", "51064", "50479", "71901", "30849", "36238", "56807", "63630", "10696", "23643", "42965", "20442", "32558", "7024", "30885", "51755", "38894", "36699", "9077", "68524", "15268", "9543", "42835", "57867", "67820", "53264", "19188", "44865", "58001", "33673", "23521", "68293", "55462", "34780", "36008", "72028", "7686", "66416", "31941", "37054", "46325", "59982", "57957", "48346", "42678", "39155", "12926", "29269", "14538", "1863", "58829", "19113", "56927", "22055", "31138", "34953", "23929", "65080", "63391", "68006", "49897", "29070", "31874", "40220", "5184", "2471", "339", "40954", "45599", "48275", "1588", "27418", "13534", "25221", "33146", "33572", "38235", "31526", "11128", "33357", "70838", "10280", "32115", "28339", "72307", "46355", "69909", "22009", "11251", "59772", "61512", "16615", "819", "21650", "24031", "49650", "64931", "36337", "48629", "44", "20775", "35095", "71404", "38493", "19257", "23279", "9562", "64536", "10622", "3177", "7460", "31710", "39789", "45502", "48125", "42570", "44694", "32307", "55026", "24351", "52905", "19580", "67141", "13548", "69162", "20030", "8844", "16052", "53819", "68614", "31282", "14606", "60605", "73129", "64449", "49935", "674", "48554", "38206", "1423", "2053", "45044", "66359", "69722", "5288", "37892", "60152", "65509", "38266", "40052", "50557", "25197", "51333", "148", "16273", "44631", "28304", "23579", "13617", "36234", "3133", "11654", "20301", "37879", "64493", "63002", "14796", "31725", "46877", "28929", "51563", "12012", "32198", "50108", "3861", "43508", "15459", "14087", "6371", "6958", "13697", "49861", "45433", "34236", "14012", "48435", "19913", "64368", "12963", "45233", "56567", "56941", "15067", "71863", "65961", "19551", "21426", "12833", "52083", "25432", "46266", "35575", "39760", "33598", "21146", "69613", "64765", "43286", "70716", "18748", "16341", "2213", "10072", "43168", "61000", "32013", "32907", "31910", "19497", "42704", "8307", "27690", "12586", "3971", "55704", "56131", "17882", "1980", "18329", "35226", "63547", "70731", "27556", "40448", "44428", "12910", "53394", "60828", "37377", "61253", "66446", "18567", "62490", "37123", "28493", "38812", "34606", "38507", "28720", "27655", "35822", "17502", "17385", "32107", "73158", "26804", "58300", "39785", "319", "51329", "61353", "50377", "2788", "14948", "13927", "41176", "46760", "53355", "37220", "33261", "7032", "12890", "50104", "53968", "49050", "13450", "4827", "35939", "66440", "13263", "48438", "1173", "36658", "70401", "11229", "45283", "10997", "1669", "72978", "71207", "27301", "62006", "68453", "37551", "45065", "49192", "46239", "58497", "3318", "4972", "42935", "2222", "67464", "47127", "68315", "19121", "8275", "10577", "11831", "19758", "61634", "5108", "15258", "9595", "41701", "38199", "12620", "68836", "9025", "13023", "42759", "70059", "65522", "8890", "21381", "56300", "1622", "63198", "68101", "8126", "1536", "72218", "16358", "16523", "57006", "36761", "5043", "3039", "57430", "61780", "21671", "37794", "64331", "54534", "35590", "25325", "58155", "10444", "23928", "66915", "2659", "13747", "18387", "72418", "27315", "20379", "59339", "17851", "57491", "7242", "24695", "2290", "24647", "20965", "14265", "6779", "71733", "66089", "58836", "54551", "23099", "19784", "53734", "11", "549", "50718", "37395", "67824", "7339", "2682", "3946", "53410", "14261", "31183", "66921", "61093", "48003", "48085", "20599", "29431", "49390", "30657", "13272", "43147", "9610", "47317", "47390", "2408", "14190", "12554", "9825", "27343", "43240", "49766", "55236", "23973", "25152", "44635", "13423", "66680", "31230", "58800", "30124", "43211", "68897", "66735", "65113", "10378", "33452", "67955", "48761", "5969", "29323", "62331", "43364", "66209", "28729", "11665", "43649", "15611", "66568", "47506", "61441", "30283", "59319", "175", "33221", "23719", "13955", "45448", "37027", "15964", "27257", "36382", "2349", "50820", "36052", "55749", "23029", "33262", "49902", "47888", "63422", "62287", "28190", "70640", "42985", "33925", "61788", "21879", "15088", "41381", "11381", "52938", "27606", "397", "2457", "57453", "22598", "25452", "58693", "28168", "33535", "65947", "14011", "20781", "17784", "65000", "11299", "69678", "65862", "231", "1011", "68648", "16267", "47453", "21253", "8557", "2936", "31107", "53596", "69234", "53041", "20580", "62238", "34014", "45697", "52034", "37208", "64367", "29696", "7933", "2446", "40687", "44780", "28637", "11465", "60631", "52145", "2980", "46181", "19215", "31771", "35512", "52548", "56410", "27423", "23617", "18090", "36814", "53696", "16175", "61150", "67711", "33556", "58096", "59716", "34306", "26923", "43350", "65434", "35453", "23698", "47684", "49639", "71003", "67895", "13539", "349", "27594", "34818", "41615", "30372", "19348", "6649", "68916", "35616", "49441", "9032", "22584", "27006", "37710", "61331", "40682", "59647", "71842", "48741", "47558", "3390", "61003", "9078", "2709", "7777", "69362", "43464", "21709", "3095", "5257", "9396", "10466", "19195", "14110", "5383", "1380", "28421", "66050", "19946", "32117", "24338", "53546", "6817", "50617", "24573", "56124", "24771", "23900", "9581", "32229", "54092", "52122", "6983", "25770", "23374", "39629", "44449", "19604", "18421", "51902", "30319", "60364", "18106", "21392", "33730", "32493", "5104", "70021", "68250", "6386", "18027", "42089", "55897", "33421", "52440", "44932", "2669", "39024", "55337", "65049", "43285", "62315", "19918", "283", "1018", "32722", "5926", "21614", "21516", "61533", "1222", "16739", "43695", "28823", "12428", "27940", "70167", "65229", "61581", "27217", "61234", "10852", "64697", "72271", "27999", "65400", "34267", "60937", "4459", "30249", "42188", "20055", "48536", "43739", "11511", "21842", "50793", "48040", "35133", "47560", "14801", "28744", "53071", "9250", "39342", "45000", "37183", "7464", "6751", "68719", "1338", "31542", "65937", "30976", "35403", "62316", "23883", "32295", "63007", "57486", "15564", "6380", "67546", "32728", "500", "33430", "58200", "41214", "45453", "47396", "29906", "54291", "10493", "3254", "3263", "60319", "45293", "51060", "72344", "20570", "33945", "44766", "35965", "9680", "28169", "69883", "6595", "5438", "73196", "45818", "33402", "48357", "21384", "55073", "18407", "44910", "16453", "17369", "66910", "48016", "42545", "52602", "41604", "15634", "24455", "14739", "18804", "13386", "24879", "47807", "14501", "35853", "64171", "47327", "64891", "1509", "29734", "24572", "11949", "59625", "19213", "9538", "68005", "59003", "63809", "46044", "52711", "32603", "54610", "69440", "34812", "11071", "13054", "18770", "20354", "33530", "18751", "72244", "15842", "21390", "14694", "10118", "63081", "59089", "17637", "40574", "13756", "64179", "14958", "7800", "17641", "9853", "35186", "9048", "38164", "36184", "30384", "66356", "30043", "17580", "50705", "68624", "34626", "14688", "28819", "23170", "3653", "57150", "42691", "34776", "12293", "25439", "10646", "5547", "33682", "24421", "65508", "52801", "68470", "34123", "71244", "22517", "49302", "16944", "18869", "22147", "20860", "53046", "17386", "28235", "51024", "70080", "23809", "51743", "20450", "40270", "45382", "71271", "64759", "60964", "2403", "17834", "32028", "55472", "40351", "20859", "32148", "7132", "10661", "19788", "4267", "13257", "37568", "67934", "68365", "29807", "42243", "33702", "25833", "26410", "68882", "49916", "68426", "39442", "60385", "40386", "59702", "22220", "33138", "52291", "47134", "27782", "13488", "22516", "2156", "29392", "9387", "4850", "60517", "51907", "4029", "7268", "34459", "48667", "67", "12305", "67946", "23886", "9767", "25189", "49300", "42472", "69744", "32336", "39922", "28188", "46855", "60037", "39862", "1497", "50445", "9282", "34321", "30116", "10032", "68319", "14918", "52680", "18138", "54422", "53035", "29657", "9634", "73195", "42435", "31032", "17177", "42909", "68194", "36777", "50147", "13827", "73251", "39085", "48479", "69679", "50322", "48312", "3236", "47034", "21827", "40400", "21093", "28828", "8113", "37945", "16477", "73135", "48308", "70254", "66296", "60504", "26019", "50380", "63603", "20968", "59748", "3355", "67885", "26073", "66966", "47347", "36686", "11104", "39270", "30118", "56546", "67272", "58285", "46031", "13859", "29283", "2209", "33576", "13027", "65115", "64065", "15413", "30750", "52642", "47765", "68275", "54262", "1307", "6713", "6588", "12069", "28500", "14949", "70230", "64459", "1424", "11591", "3033", "7235", "40756", "54129", "12824", "3849", "38309", "56827", "71879", "20066", "47896", "6476", "31288", "20722", "32769", "26792", "6460", "53509", "64112", "50572", "16745", "16581", "22359", "19285", "32495", "22097", "40988", "52160", "14788", "48179", "51090", "45562", "5941", "66031", "3988", "11259", "12638", "70376", "27026", "67461", "872", "41137", "16042", "13771", "7054", "66920", "72289", "53838", "52165", "58729", "39591", "19331", "44311", "13296", "22884", "22354", "56963", "56870", "63001", "9783", "62769", "62488", "18952", "9326", "31939", "69659", "18026", "73023", "63225", "92", "13658", "13654", "25842", "2523", "43463", "17300", "66062", "34171", "36144", "2090", "44271", "68525", "49706", "17572", "28112", "5025", "54434", "8838", "68197", "7184", "59081", "3278", "34388", "47046", "46723", "14646", "7386", "11933", "53377", "72600", "7328", "1896", "43196", "73149", "37642", "16849", "70564", "22724", "46636", "45255", "40577", "16969", "59080", "59225", "72352", "25101", "25469", "17820", "64424", "18795", "41783", "47085", "6873", "69556", "12998", "55082", "61076", "10917", "61197", "39007", "63929", "43765", "60997", "14693", "37602", "47988", "51096", "37483", "1837", "9126", "55410", "55324", "29701", "51324", "72236", "28851", "46159", "52855", "20704", "53668", "18185", "8698", "68649", "69244", "25563", "27204", "72602", "12350", "46863", "30346", "61792", "41984", "66517", "20731", "3122", "52935", "68246", "30088", "57824", "28344", "24689", "29164", "3810", "69216", "15704", "16330", "6142", "25866", "6429", "55447", "12792", "33721", "41240", "9038", "35495", "70457", "50459", "62834", "43121", "52196", "28886", "15692", "11235", "72645", "40548", "17354", "39208", "59574", "27236", "66010", "36828", "32391", "32555", "28089", "29581", "29415", "18845", "14175", "21209", "61024", "9210", "14066", "13286", "29761", "15510", "67287", "43705", "14618", "34340", "46260", "6109", "342", "4906", "26462", "38599", "72631", "60428", "20895", "55992", "52524", "18856", "12144", "47731", "60863", "39218", "41607", "49086", "63904", "47945", "10283", "65339", "10445", "34618", "17194", "43189", "14633", "25707", "52776", "51037", "38299", "25236", "11375", "28205", "28840", "52811", "59820", "18113", "28780", "53741", "39617", "55705", "70496", "31493", "30637", "25367", "64917", "55541", "53336", "24580", "69987", "41494", "21834", "40491", "10327", "45696", "8382", "65046", "14097", "31345", "1917", "38890", "10151", "66105", "50094", "70126", "19550", "26041", "39686", "24067", "67765", "62933", "22467", "31410", "40809", "39531", "5234", "36943", "11917", "33933", "65386", "5319", "51508", "57547", "31744", "38757", "35975", "21391", "66000", "4162", "8085", "10051", "28879", "59286", "31896", "38130", "3702", "64689", "36429", "20989", "10008", "2646", "50394", "42740", "51094", "17872", "63894", "26671", "47142", "19355", "73178", "21237", "14578", "30544", "20466", "40860", "19317", "37188", "64329", "59947", "61824", "70794", "66390", "49577", "37891", "67528", "62077", "71716", "27878", "58141", "56557", "56011", "46362", "14966", "4863", "6447", "16038", "25880", "14471", "7462", "40306", "25365", "28505", "6252", "46828", "16914", "63993", "53548", "41812", "8517", "48645", "58950", "40145", "35273", "61458", "28564", "9971", "31674", "49483", "57994", "18337", "12115", "22406", "65433", "57911", "63314", "61858", "900", "65546", "38960", "11288", "72417", "9885", "54854", "9537", "66844", "21814", "3892", "65397", "8773", "24907", "15313", "58370", "29668", "18695", "30165", "42557", "70868", "44816", "20041", "896", "57069", "32256", "19504", "43879", "4656", "9316", "4760", "59537", "64302", "54415", "4805", "70147", "42727", "17180", "55635", "60305", "52595", "66186", "38715", "57755", "15440", "36227", "56645", "5495", "72068", "40877", "51138", "39188", "57085", "45743", "53469", "11893", "44754", "28642", "31694", "70686", "10331", "69864", "71346", "31973", "50270", "299", "66469", "11149", "69317", "35828", "20571", "7917", "17616", "34867", "34230", "38813", "39301", "18581", "21759", "36927", "36498", "48456", "33141", "72700", "35669", "60029", "4043", "22746", "65517", "45783", "49007", "35642", "59727", "50401", "1004", "678", "2383", "55291", "895", "66321", "62117", "65486", "21744", "71303", "17155", "19343", "63033", "36134", "43565", "44448", "55775", "60654", "11582", "42624", "19380", "24144", "48528", "46957", "28572", "61721", "67843", "33069", "57998", "17814", "17782", "60540", "45585", "37826", "35396", "44799", "64035", "13235", "32302", "67996", "54824", "45012", "18889", "6288", "58586", "53101", "2487", "44668", "4544", "20719", "57907", "24759", "17380", "70235", "24269", "17497", "67605", "17075", "7084", "4383", "50910", "34283", "45707", "37204", "44043", "45185", "26103", "12256", "9705", "72681", "19173", "67770", "40891", "71246", "48809", "58648", "20196", "66947", "38378", "33889", "12654", "11334", "25504", "14268", "64814", "7189", "62648", "46485", "51662", "60677", "71162", "14084", "22469", "15311", "54087", "62504", "58618", "34842", "6749", "26339", "66429", "65467", "1246", "53865", "59021", "14632", "63623", "56749", "49952", "49855", "23645", "4550", "35253", "9496", "52588", "21335", "68691", "21112", "61605", "67124", "18236", "27937", "58704", "27297", "43560", "44832", "42617", "61435", "35448", "8637", "58261", "31782", "4311", "6512", "4352", "27313", "13219", "60717", "20202", "44455", "55799", "24482", "40038", "49408", "20278", "42257", "37434", "16063", "7792", "12774", "15149", "6155", "37487", "33791", "60197", "27635", "72404", "63538", "35563", "46232", "55061", "65299", "24790", "60158", "69761", "3928", "46739", "3589", "30459", "41950", "49335", "25675", "66455", "21004", "58088", "34534", "6278", "57776", "14427", "9368", "33690", "64585", "10767", "36578", "14007", "24672", "38181", "4537", "16468", "8723", "10244", "24502", "56435", "6096", "15977", "53340", "9890", "61802", "42347", "15858", "59654", "47392", "68929", "70552", "54857", "3146", "11007", "38930", "67609", "23734", "28809", "4698", "40128", "16324", "28843", "72323", "52839", "12150", "13840", "32878", "60873", "8273", "2256", "21044", "2551", "22292", "21635", "46478", "70911", "7022", "54843", "24358", "6877", "20071", "7655", "68705", "59950", "53369", "70504", "40864", "23196", "51478", "45048", "975", "14283", "32753", "19450", "2246", "66388", "15452", "21255", "29257", "65319", "16758", "6504", "67011", "41684", "19607", "52256", "21125", "2245", "37967", "58212", "55010", "32053", "61180", "23109", "34180", "5048", "32514", "8922", "1599", "4233", "50449", "37772", "20545", "23048", "14653", "35198", "62797", "30856", "63831", "11929", "57221", "8858", "734", "32339", "63281", "73166", "56583", "53973", "24663", "14605", "34894", "5126", "39471", "20432", "35166", "31983", "46526", "60297", "17057", "63617", "72953", "57690", "18456", "63703", "51339", "52958", "47236", "21239", "45061", "49718", "11417", "2133", "73114", "5398", "38753", "48111", "5127", "43602", "1898", "49398", "60831", "32805", "26396", "19226", "45771", "57714", "2641", "9726", "35295", "43629", "43488", "31360", "67857", "52011", "38102", "52561", "70572", "51806", "71956", "30272", "22615", "30492", "6203", "40473", "73122", "55028", "24815", "63997", "348", "68914", "57805", "1364", "30265", "52778", "11102", "44526", "53518", "35681", "28152", "49991", "12385", "8441", "70293", "28433", "33434", "57311", "732", "69571", "710", "14468", "9407", "65264", "37997", "16496", "35576", "34550", "56582", "17243", "31131", "1449", "43480", "15689", "16894", "14238", "41190", "5082", "30785", "27256", "31626", "41609", "15362", "19315", "60644", "11294", "2139", "28452", "14912", "72201", "57530", "65423", "45583", "11871", "43316", "27235", "40427", "25709", "49217", "4215", "45824", "23441", "21415", "67109", "21344", "40592", "48616", "27959", "57754", "57235", "58339", "63062", "45715", "62581", "71157", "40861", "53847", "56077", "51074", "60438", "1271", "60674", "51114", "45795", "69131", "38521", "7050", "7316", "56351", "31125", "25058", "72440", "44272", "59035", "24912", "32804", "24768", "1105", "51086", "57379", "33984", "11593", "55203", "36795", "42620", "13789", "27056", "1015", "45174", "49035", "8717", "57257", "3026", "16834", "73103", "68323", "47540", "17200", "43542", "26721", "33904", "1811", "24736", "58670", "7848", "65362", "15939", "15308", "30815", "17442", "33408", "23873", "57268", "71479", "54966", "10740", "537", "21702", "19559", "35787", "64134", "53636", "40608", "37566", "56198", "70885", "35952", "33513", "44685", "8465", "20498", "9982", "60510", "18779", "72601", "2233", "8102", "59181", "3121", "47607", "65006", "19100", "21966", "39890", "70805", "66182", "31898", "30922", "33661", "40132", "60399", "33766", "62058", "10932", "42974", "70463", "52685", "29733", "53893", "35178", "55815", "68437", "60283", "59703", "21995", "445", "34055", "243", "64590", "66461", "22252", "10559", "18424", "68678", "513", "21580", "65561", "57107", "33548", "42854", "7135", "51603", "36386", "43653", "68507", "57271", "14800", "70970", "53811", "1479", "3922", "71460", "13044", "21851", "56291", "32536", "45592", "47605", "49677", "66759", "70850", "11239", "29294", "30004", "55712", "44535", "36493", "70635", "54095", "42900", "34650", "57434", "14353", "72876", "52302", "28213", "34612", "42391", "61646", "18632", "34797", "18794", "3123", "39537", "60657", "40114", "3636", "17444", "11910", "46140", "13580", "8133", "19033", "25628", "59213", "18389", "35634", "46606", "37124", "35323", "67013", "33600", "50319", "51363", "43374", "66965", "43317", "3280", "70348", "61929", "24800", "9552", "72292", "9263", "49847", "50599", "22108", "58009", "2344", "29444", "37210", "451", "35857", "12055", "17320", "18302", "12249", "8945", "47115", "63918", "17423", "36307", "6246", "54690", "2470", "46286", "67864", "2204", "26197", "364", "20642", "53923", "40457", "27888", "67201", "29750", "65797", "34684", "775", "70936", "20636", "10831", "40594", "16967", "67979", "65929", "5588", "14804", "11397", "6716", "59495", "41921", "22300", "32376", "18038", "21793", "18529", "44253", "41146", "19053", "27148", "56824", "64299", "52114", "71042", "64044", "27104", "23677", "16650", "56533", "43807", "62458", "8586", "45972", "21425", "35250", "21903", "26775", "24514", "72330", "63840", "67868", "28605", "41048", "37134", "42794", "6122", "43859", "56269", "72510", "15793", "35708", "66103", "26250", "36345", "13818", "57460", "12698", "25917", "10679", "33847", "20445", "53029", "48756", "72932", "47348", "42351", "55659", "223", "32867", "46924", "53620", "8447", "4171", "19953", "32846", "16442", "16220", "5570", "17294", "35366", "61586", "39094", "41587", "18080", "68155", "58334", "65658", "35430", "41128", "43187", "17994", "54161", "68405", "22689", "56165", "57160", "71408", "37641", "4922", "41971", "29260", "44584", "73067", "61707", "68", "3314", "17445", "35070", "42792", "22718", "35094", "66637", "9847", "10486", "14839", "55256", "695", "11811", "32559", "70854", "39256", "32488", "69572", "58674", "10808", "39468", "53503", "67396", "15021", "44117", "66974", "47503", "26276", "22375", "33372", "72162", "1207", "21852", "51784", "4126", "45457", "26840", "22558", "47386", "30017", "47939", "18377", "35635", "40099", "28814", "64129", "16036", "28434", "33050", "43370", "35335", "31715", "25240", "59282", "16193", "25697", "2181", "53830", "60196", "63484", "41845", "52674", "31227", "16860", "4854", "32865", "23559", "59795", "4785", "53325", "62327", "50607", "28778", "52670", "47789", "38523", "12979", "36261", "3972", "26896", "23216", "23563", "6477", "44853", "5452", "35212", "33306", "2686", "46384", "68954", "32373", "24043", "47694", "20879", "12401", "57951", "44589", "43645", "59505", "31945", "15963", "41631", "6024", "62308", "42120", "66064", "19771", "16800", "64471", "14428", "16169", "53980", "40635", "23761", "17336", "24666", "11942", "67566", "48658", "16106", "31555", "26104", "40527", "14655", "43783", "65110", "44132", "14722", "63566", "1890", "73206", "11723", "66832", "68482", "8668", "43914", "55128", "26269", "46244", "68017", "37761", "65949", "59543", "29165", "3125", "33351", "68637", "43779", "48689", "36270", "14636", "9034", "55173", "13537", "13445", "69497", "2636", "11127", "23148", "49392", "12112", "36571", "70199", "19274", "17506", "36574", "34427", "25110", "66537", "56937", "45734", "59072", "31104", "15653", "8300", "22349", "41197", "66043", "27903", "35257", "40603", "31614", "9278", "34292", "237", "56470", "46791", "21133", "46994", "16634", "28324", "66094", "30808", "44874", "48692", "29664", "26346", "67778", "69157", "62815", "15216", "16940", "61343", "9042", "71921", "35768", "41531", "27724", "58017", "21276", "21713", "50071", "35437", "73086", "51710", "54274", "45469", "7810", "61782", "43439", "29690", "40158", "61392", "43716", "47979", "16051", "57770", "65704", "4091", "52090", "66811", "12370", "3165", "41125", "8225", "58629", "22987", "72666", "21595", "38549", "52119", "19635", "50297", "3832", "72436", "56673", "7196", "47437", "30832", "68136", "4896", "40395", "50063", "14690", "28143", "46984", "8982", "4355", "32601", "31430", "21373", "65407", "21267", "42425", "19369", "14083", "3955", "33802", "58043", "43458", "40567", "9973", "50110", "33992", "48087", "58921", "50325", "9119", "22858", "67122", "2538", "54932", "1172", "56415", "19050", "28727", "43062", "13830", "16174", "69932", "63407", "70388", "39522", "3046", "50096", "52604", "40644", "58316", "56555", "27767", "17137", "5196", "56600", "36628", "41229", "67095", "44277", "37546", "9219", "1026", "57236", "47299", "24615", "55662", "53595", "29855", "65820", "70025", "68832", "40704", "20272", "57333", "9493", "12033", "36838", "3443", "10333", "24213", "30172", "73172", "72911", "39403", "15869", "65998", "62614", "25756", "50066", "28068", "65050", "10576", "44252", "31660", "40741", "16530", "58612", "46503", "3164", "56002", "72235", "67492", "3876", "56588", "4862", "69401", "13351", "11794", "41258", "53147", "8181", "11930", "594", "56222", "32334", "28023", "35664", "49037", "34563", "33188", "68297", "17527", "13084", "5213", "823", "600", "34279", "19220", "71104", "20824", "43421", "67064", "50059", "69934", "26592", "59158", "53207", "71088", "2951", "33469", "16439", "37001", "52626", "34945", "32476", "39915", "27201", "68567", "47569", "48390", "3848", "38572", "28226", "31518", "61706", "13768", "41998", "68218", "43975", "8979", "59075", "39737", "50531", "10676", "25881", "30968", "47488", "60589", "26736", "7988", "53393", "25417", "38774", "37747", "23396", "17848", "27374", "21126", "27825", "5220", "66354", "13137", "68988", "20470", "62715", "26778", "45407", "10985", "53716", "49203", "8543", "5049", "50529", "26774", "46340", "61628", "27638", "48836", "52939", "35248", "51376", "4325", "59522", "16418", "62295", "56745", "48018", "30879", "66215", "38463", "62205", "52469", "13702", "43165", "52911", "34901", "53978", "13592", "35261", "4669", "60597", "61962", "48484", "23443", "33072", "53398", "63318", "52779", "9274", "17676", "60038", "20060", "71073", "32858", "44095", "22734", "33780", "68508", "14149", "31269", "63762", "3574", "15474", "49584", "15132", "56826", "14528", "40546", "38971", "52679", "23694", "12590", "22451", "24692", "66519", "5679", "58170", "24259", "34531", "53181", "512", "37513", "17159", "15548", "63706", "54081", "63336", "7715", "18451", "33489", "46231", "45574", "23601", "12419", "37536", "64547", "37042", "38598", "4199", "29202", "58545", "45324", "55884", "50681", "17507", "61151", "42164", "53921", "58079", "25679", "36804", "47037", "32161", "44036", "29128", "49363", "42653", "3044", "59444", "5858", "53673", "3135", "45075", "26700", "14969", "55368", "24976", "27613", "56072", "67459", "43533", "51410", "33674", "26424", "70061", "53897", "14304", "41440", "37974", "66669", "17712", "31148", "24209", "32022", "64548", "21001", "56372", "16776", "8576", "47654", "45099", "47960", "23", "54548", "28126", "26815", "7195", "45572", "3590", "61661", "46253", "28265", "16180", "13888", "37037", "4908", "29023", "37194", "64727", "30096", "22658", "35548", "1947", "44659", "56484", "41013", "15505", "13344", "36879", "72514", "51042", "52555", "7588", "30864", "2862", "64228", "43059", "47111", "34917", "72060", "36501", "26357", "18326", "33276", "22047", "3820", "15658", "60113", "59877", "107", "33364", "20166", "12957", "49690", "72772", "40823", "22873", "36507", "30573", "7622", "37009", "3838", "63717", "62002", "43844", "47248", "58577", "7801", "65717", "32850", "54406", "60024", "28173", "4034", "10107", "48010", "70542", "69536", "31055", "24986", "18766", "54155", "6824", "5318", "40437", "29943", "36420", "13685", "69513", "46375", "71011", "66191", "11627", "44756", "1319", "69427", "56192", "54779", "47626", "52283", "1195", "53321", "1909", "55996", "68490", "40173", "7751", "32325", "64608", "19663", "52018", "63588", "61348", "8185", "18562", "46577", "17334", "6209", "58863", "23954", "23146", "1743", "26036", "61339", "56279", "69462", "21913", "62019", "48583", "37171", "61242", "31001", "49453", "17938", "59836", "56356", "71949", "70190", "16644", "31249", "27664", "60947", "61444", "35287", "66316", "55679", "52292", "52127", "15450", "28164", "36042", "8059", "41760", "25049", "53193", "18526", "44414", "65946", "12435", "41442", "44017", "3032", "11539", "25963", "24251", "20866", "6241", "31616", "14438", "22965", "70114", "23564", "17343", "46600", "57192", "14075", "16142", "24655", "23613", "38823", "2110", "64174", "2311", "15748", "9627", "53850", "24392", "12796", "22422", "38128", "53116", "25965", "47334", "36154", "15630", "9131", "67747", "29244", "36972", "507", "34914", "13602", "46098", "4342", "48539", "33662", "27299", "518", "48686", "24673", "71070", "25103", "54493", "41290", "11330", "6662", "21198", "68466", "55096", "10090", "71961", "43302", "72646", "22379", "54898", "56627", "33086", "23690", "10691", "52757", "18466", "60861", "47748", "28542", "39683", "38297", "25795", "55504", "2984", "32134", "53160", "36378", "1650", "18097", "69695", "26200", "29189", "3089", "28510", "38311", "56310", "46538", "72979", "50539", "71749", "32710", "39335", "55500", "66422", "12199", "44897", "10298", "11173", "7561", "12389", "32715", "31371", "32848", "38601", "32967", "69039", "35879", "70307", "64761", "3317", "38490", "63963", "224", "10503", "39282", "12837", "56329", "56991", "68476", "26155", "70528", "5592", "52102", "46798", "8131", "63107", "56390", "33370", "63419", "49228", "45022", "53293", "29453", "3234", "65874", "14206", "21586", "36370", "49461", "3548", "21601", "29328", "30132", "59186", "30202", "11085", "52012", "64798", "34936", "63174", "67454", "37279", "63302", "41935", "50854", "49152", "17678", "67223", "32063", "30674", "1749", "44553", "66828", "23165", "12514", "30007", "38067", "50585", "52612", "4164", "37985", "56623", "67799", "61887", "65165", "50211", "15529", "26971", "51640", "12381", "14085", "2553", "42324", "4136", "70532", "55522", "54110", "48378", "24091", "29237", "57341", "48561", "57636", "19631", "42248", "66588", "51887", "7910", "15244", "63491", "72574", "67586", "58396", "33657", "49304", "9683", "66986", "51323", "1949", "56636", "7383", "45494", "25670", "58143", "34252", "68712", "8247", "51272", "11372", "65251", "3005", "31416", "14337", "45939", "39245", "16068", "14798", "13092", "11686", "18573", "15352", "17717", "28556", "28806", "60342", "26291", "27349", "30604", "9578", "43970", "69109", "51546", "39547", "3669", "27676", "37467", "37502", "50230", "64782", "1367", "60607", "69429", "54752", "7119", "59094", "17223", "63727", "55877", "29881", "13479", "11926", "9684", "22708", "793", "50332", "54909", "7528", "44723", "60864", "68560", "58231", "40258", "11134", "41141", "17918", "5360", "35360", "61002", "29699", "12877", "71239", "19562", "16646", "50120", "2136", "5944", "29045", "70408", "48455", "49292", "14641", "47566", "47663", "56238", "72741", "8466", "32191", "39939", "5462", "17535", "61879", "63412", "5056", "65995", "31584", "13431", "43633", "41692", "15307", "3566", "5970", "9534", "59789", "16032", "35468", "40318", "14841", "206", "57263", "40517", "59617", "65449", "51360", "8027", "47561", "44477", "69093", "51116", "8539", "68549", "24696", "38313", "49420", "43918", "66580", "6663", "14532", "19757", "35244", "6287", "27023", "11760", "68996", "39357", "46005", "23152", "64101", "51100", "35827", "50632", "28070", "16831", "36433", "58049", "49713", "29845", "46789", "35377", "47720", "37622", "59826", "26325", "38088", "24795", "10873", "50101", "70609", "61654", "17372", "10396", "65979", "3291", "30095", "68612", "33443", "28836", "36116", "63338", "696", "36765", "8906", "33613", "66621", "72997", "53909", "61964", "6836", "37180", "31701", "32587", "18473", "35440", "50769", "57458", "29460", "5204", "10063", "7197", "22787", "72366", "67418", "56571", "35183", "69751", "73072", "25394", "44497", "21993", "54156", "40115", "19610", "45126", "30842", "32909", "70089", "22316", "61888", "24764", "69032", "45789", "69592", "67002", "60637", "12319", "7453", "14335", "60178", "31868", "31621", "19579", "52474", "282", "56999", "35727", "25579", "55829", "16610", "12821", "72582", "42265", "73161", "59927", "4750", "63959", "58529", "9056", "1274", "28408", "16854", "32313", "52729", "22319", "14354", "20966", "21301", "18258", "22995", "4470", "44719", "9821", "44305", "18535", "22654", "56891", "14079", "70270", "32378", "11034", "69537", "31141", "47931", "11279", "28488", "16567", "776", "2341", "73068", "63034", "24045", "72266", "40330", "43720", "11083", "33505", "8504", "17528", "44769", "43088", "32689", "48106", "68593", "62845", "70894", "50018", "50930", "54861", "70973", "50148", "8927", "73089", "60618", "6883", "41853", "73175", "58333", "23470", "72890", "21243", "9273", "58937", "62763", "47161", "59013", "11857", "38859", "17850", "7348", "51651", "30626", "71045", "35533", "13547", "6117", "72319", "43326", "27566", "35293", "26136", "58106", "6646", "58675", "16080", "11450", "52653", "12506", "28049", "27560", "45408", "71283", "46285", "69303", "58928", "8814", "37893", "14851", "4678", "11853", "50473", "63035", "53350", "20606", "41958", "37033", "30777", "33867", "45405", "2244", "45858", "17049", "70674", "33316", "33033", "37407", "10252", "27391", "14974", "61898", "38616", "69444", "35714", "65806", "40022", "39128", "70048", "61251", "29078", "55161", "7615", "11267", "50857", "29801", "52694", "70471", "34617", "38435", "2505", "66943", "30377", "62469", "30144", "66173", "42708", "34335", "66400", "52479", "69762", "42075", "44071", "8826", "34576", "16365", "36479", "1354", "11580", "18004", "47907", "59824", "28363", "44711", "22501", "19424", "63382", "41818", "52686", "58694", "42370", "21370", "29439", "30806", "561", "55213", "54744", "23644", "47487", "22563", "7631", "29180", "29743", "29225", "34112", "72173", "17513", "12127", "35030", "6848", "10094", "44888", "26484", "62042", "47315", "41274", "72163", "55729", "10902", "56864", "41325", "934", "33919", "19102", "42049", "54702", "67513", "6978", "72285", "42562", "68280", "67848", "28711", "31333", "70928", "9486", "25126", "38617", "56292", "60172", "13763", "19931", "43185", "54286", "38979", "21341", "22757", "56049", "60794", "13619", "43522", "30768", "27599", "11884", "32177", "47379", "10222", "42657", "26184", "63317", "56898", "6393", "55074", "44750", "13881", "3071", "50450", "49817", "18012", "68303", "18034", "33781", "17459", "34559", "22808", "30480", "33762", "11827", "19662", "9593", "53528", "67261", "57291", "10106", "8675", "28115", "66427", "21037", "69302", "43953", "26789", "46954", "54846", "40415", "42939", "54634", "53510", "61753", "12945", "73243", "50150", "29575", "11900", "8230", "63810", "66467", "33028", "41862", "43049", "66980", "24757", "49021", "21499", "4064", "25107", "52963", "35398", "56589", "58081", "41640", "59601", "48913", "40836", "39022", "12619", "67257", "65786", "56625", "32422", "11429", "41623", "58916", "47954", "14213", "53898", "53835", "32406", "56586", "46587", "36013", "67016", "68368", "45218", "29828", "44218", "63879", "15547", "29443", "56929", "62065", "17702", "41745", "65549", "2840", "57392", "43920", "34470", "68308", "19297", "60436", "61912", "12200", "58290", "24156", "19398", "34632", "62357", "5367", "31090", "44384", "31712", "38247", "39920", "68790", "65713", "47875", "26024", "37058", "46045", "48279", "7638", "37971", "34595", "13180", "61963", "33424", "42719", "60620", "41601", "7775", "23683", "5632", "66095", "37995", "44590", "54647", "71294", "61911", "55796", "26319", "24181", "11100", "25767", "17816", "29535", "68514", "2360", "4129", "57483", "49932", "1991", "50784", "72005", "41180", "53251", "26795", "3388", "18279", "8827", "22058", "15303", "22984", "56075", "23908", "51946", "14869", "51291", "61861", "63613", "66698", "47220", "43318", "10062", "64378", "49234", "27311", "15198", "9991", "29448", "59036", "52046", "20368", "23889", "31791", "73229", "33993", "36607", "65491", "24958", "239", "60315", "13211", "10376", "8648", "10933", "65667", "5896", "8114", "28878", "8354", "57404", "4218", "23481", "32977", "20614", "54517", "41116", "57102", "65815", "638", "67415", "33253", "2308", "34784", "31488", "18256", "62606", "27288", "13831", "62918", "50690", "64500", "45821", "7151", "44044", "60165", "9992", "2284", "43221", "56599", "42304", "64438", "71764", "47281", "25262", "9658", "53655", "33936", "40539", "9571", "68591", "22315", "30757", "50141", "29825", "528", "27661", "61405", "56273", "53087", "49783", "71717", "66470", "27518", "58224", "49922", "30323", "43897", "55459", "22107", "26071", "60702", "70895", "62720", "59449", "42046", "16146", "48068", "36454", "26942", "9881", "61915", "26963", "39524", "67827", "26256", "68202", "40285", "51924", "57533", "71686", "47060", "9394", "50390", "21396", "9713", "61715", "17100", "13956", "67075", "20182", "27220", "54024", "4848", "32614", "70208", "67191", "68452", "70883", "53505", "9223", "21073", "68697", "34991", "59805", "181", "438", "1964", "10934", "52281", "28041", "63438", "32927", "34434", "59806", "31650", "46280", "14405", "26832", "2588", "27069", "34896", "20805", "37880", "47062", "41508", "72105", "11291", "1407", "41975", "69172", "49331", "2058", "31996", "68013", "46392", "17273", "52400", "23556", "38197", "40728", "20436", "24233", "52655", "64541", "49450", "43790", "14934", "38537", "64185", "20809", "35868", "9088", "49745", "40702", "67123", "58139", "35274", "63289", "52666", "53335", "3927", "72665", "64865", "45548", "69236", "12265", "32423", "15386", "56417", "73224", "18674", "27666", "23236", "20011", "60261", "8293", "35075", "56878", "1806", "48173", "9314", "28927", "57929", "19876", "1506", "47603", "19761", "36213", "70195", "38184", "35878", "16736", "38560", "34490", "70453", "5381", "13067", "54075", "56099", "41355", "504", "71098", "40385", "45535", "45426", "30609", "26870", "56286", "57062", "660", "9193", "7981", "64755", "38577", "38284", "21096", "56528", "61821", "34809", "67634", "35438", "17418", "25306", "37419", "66952", "52961", "17836", "61301", "2134", "72464", "13995", "22462", "23762", "1840", "17813", "60592", "1365", "56505", "52135", "3964", "5369", "43930", "4193", "8315", "49089", "23410", "14649", "59652", "24645", "20617", "52494", "11250", "25949", "1280", "14380", "42775", "18870", "28856", "36764", "45961", "297", "20662", "19175", "7930", "43836", "34597", "29095", "5710", "32540", "36329", "8238", "7167", "20179", "43916", "20176", "27907", "66848", "25056", "27526", "64324", "34589", "59834", "48290", "51076", "18551", "50832", "47718", "37954", "44138", "60542", "35849", "57682", "15558", "39477", "35225", "68462", "65686", "47495", "29848", "56007", "66486", "55013", "27248", "49686", "6680", "13595", "7363", "41708", "28295", "24857", "55950", "31571", "4851", "116", "38632", "52004", "57316", "22988", "43311", "4858", "63452", "35960", "36996", "50406", "39654", "68111", "51252", "59937", "15424", "23204", "6705", "23289", "38814", "62568", "21247", "27700", "60807", "33472", "24946", "61421", "26492", "1297", "54856", "33930", "20819", "23110", "47582", "63841", "5382", "62223", "25726", "30603", "54712", "46245", "25848", "47614", "4779", "55408", "26057", "72622", "31512", "54912", "36880", "55222", "30565", "42220", "7614", "52143", "17002", "52173", "47898", "14864", "61046", "13290", "38543", "2150", "3447", "13073", "69437", "23890", "58420", "52856", "54812", "8012", "26617", "31804", "12237", "66264", "25690", "26298", "51654", "33628", "33381", "34698", "48806", "7836", "23778", "17773", "14130", "11848", "4415", "32103", "46876", "43298", "8542", "61445", "37295", "55561", "71553", "20850", "42949", "72010", "8486", "32894", "8350", "24594", "18806", "42649", "30113", "33174", "25345", "70338", "55688", "45539", "7576", "26109", "66961", "59476", "25519", "58283", "72043", "68717", "47482", "3879", "37128", "32034", "68911", "7415", "10500", "38625", "71769", "56215", "68776", "53231", "42158", "30186", "48861", "32259", "26946", "24369", "66086", "3155", "62559", "57472", "37798", "60531", "45497", "44644", "16429", "27430", "35937", "48353", "30728", "15607", "43241", "44968", "57497", "9669", "12248", "13590", "63490", "51489", "48682", "48226", "21310", "6540", "60415", "64481", "41236", "61174", "27022", "64945", "6868", "6612", "54387", "9883", "57657", "41556", "9140", "32343", "36317", "7591", "67402", "34432", "18131", "73247", "68903", "26282", "23214", "7698", "36995", "22281", "5466", "54578", "7131", "1540", "72133", "25004", "14566", "19566", "35639", "17410", "39954", "66724", "14862", "30362", "41622", "36159", "67916", "33036", "42671", "35581", "56026", "33533", "52512", "5861", "32648", "56476", "6924", "60482", "50363", "64288", "48207", "2456", "30839", "62191", "21129", "57832", "58415", "48462", "47306", "34367", "61597", "60971", "55265", "479", "16364", "14831", "14224", "20870", "12210", "19228", "72346", "23460", "2680", "50992", "5630", "25847", "37372", "69098", "40883", "30308", "54044", "51750", "56895", "292", "20045", "53701", "66861", "23282", "50638", "15481", "22112", "44240", "58016", "68149", "24049", "43233", "45881", "19186", "19781", "12315", "22233", "12730", "68207", "9866", "51166", "28560", "5373", "68663", "41803", "45057", "15022", "6628", "38263", "50619", "62376", "9849", "25936", "7796", "23650", "29692", "57999", "10802", "12295", "20576", "22163", "15587", "36818", "15599", "54344", "6986", "38352", "56254", "48849", "68510", "69848", "33363", "55058", "38841", "1353", "56444", "64983", "2807", "6932", "13495", "18024", "64172", "28728", "5843", "16975", "5943", "70261", "31902", "34314", "42877", "73151", "51670", "57789", "25659", "66954", "47536", "41470", "69749", "57476", "65456", "11602", "58898", "42968", "10459", "47759", "38134", "10779", "32436", "72693", "28638", "30945", "14678", "31820", "16621", "9356", "60833", "55833", "31828", "55143", "56551", "33315", "19782", "64289", "72088", "33838", "39626", "62380", "31276", "60402", "23632", "51153", "52539", "49884", "67670", "10116", "15182", "70300", "1187", "42911", "61026", "55269", "31186", "785", "62777", "4189", "49729", "19901", "39100", "42288", "52393", "66849", "3293", "38178", "64913", "57860", "24782", "31146", "56128", "41304", "12723", "21717", "34465", "73092", "45579", "17809", "58346", "63697", "30734", "64571", "2815", "1876", "40498", "10127", "45439", "7916", "45029", "72297", "12160", "65273", "53084", "66116", "27817", "22001", "48532", "1417", "48336", "59544", "33922", "66431", "41884", "10770", "52434", "49990", "58854", "30352", "13031", "7260", "6416", "30376", "32662", "50072", "3399", "50807", "22072", "20172", "5514", "45019", "44856", "60852", "18282", "54438", "57129", "21897", "60689", "56909", "40320", "15260", "15843", "55208", "18509", "46008", "6945", "16156", "42600", "26078", "25994", "6685", "49945", "37830", "50119", "25681", "37989", "37243", "17664", "19507", "28818", "51251", "32546", "31951", "86", "1241", "53348", "42040", "19960", "8784", "57051", "25740", "69643", "45613", "50785", "30695", "11472", "31065", "55020", "39258", "8898", "42905", "34616", "18746", "9413", "56083", "8986", "47003", "6702", "22196", "58060", "11129", "30305", "8037", "39980", "71759", "47020", "30199", "8712", "36986", "1628", "50399", "12684", "52362", "41058", "26551", "40189", "10427", "61589", "26438", "64854", "23975", "34619", "39244", "68233", "52733", "16362", "35058", "13555", "57254", "24601", "58763", "11491", "27215", "51472", "8034", "10401", "51377", "23202", "2507", "63551", "8700", "21992", "58838", "6423", "25986", "14794", "44287", "72705", "72268", "55589", "62607", "47858", "36959", "3992", "41747", "57823", "43823", "36661", "72838", "12526", "23510", "65548", "651", "69219", "59775", "7574", "27571", "10095", "54689", "15447", "26964", "64722", "47389", "33507", "62183", "68108", "51781", "402", "66838", "61739", "40830", "35362", "70326", "60065", "17478", "21816", "43144", "20575", "24010", "20518", "54145", "22165", "35684", "8936", "34075", "52007", "19234", "22992", "2697", "41961", "54953", "10305", "49495", "18848", "15150", "45188", "52947", "49323", "33968", "33985", "65406", "199", "20651", "13657", "46917", "19598", "51213", "55169", "17463", "41555", "18076", "251", "10613", "58881", "36999", "17229", "18987", "53691", "40172", "49605", "62136", "5107", "62120", "68906", "36363", "60780", "43268", "73156", "56686", "65027", "50447", "52258", "51885", "51075", "21550", "52977", "26927", "60954", "62187", "51753", "892", "42540", "45691", "1132", "45947", "44700", "64335", "51983", "67787", "57041", "1316", "40058", "10245", "30021", "32981", "27643", "5933", "28562", "13919", "47964", "18725", "29248", "50475", "71701", "14061", "8212", "30122", "25766", "30301", "30104", "15960", "11886", "64132", "27600", "564", "8359", "51841", "12154", "37221", "17703", "64209", "20174", "37613", "63763", "15461", "59132", "12123", "19942", "48223", "16982", "37176", "13799", "16256", "26244", "50351", "15886", "38406", "34873", "10913", "40305", "56637", "12561", "42126", "42376", "56267", "66830", "48196", "43903", "1728", "55152", "24475", "53025", "33423", "26301", "21090", "53040", "62409", "63130", "16505", "41072", "30263", "57939", "21938", "25939", "63533", "2198", "69733", "41321", "72248", "33326", "67445", "50771", "67314", "72212", "72384", "27036", "68779", "41165", "41505", "29034", "16264", "21376", "25674", "10605", "72773", "22620", "35745", "71935", "62089", "44256", "66817", "61701", "8981", "56370", "902", "5421", "6277", "7211", "57405", "72612", "10446", "7392", "62779", "30756", "72291", "7290", "8770", "48624", "67092", "28382", "69103", "19403", "22274", "20988", "68057", "66628", "57915", "59863", "3309", "15169", "65997", "54980", "59065", "42417", "9924", "61678", "36739", "7773", "67961", "48027", "60727", "19359", "70802", "58505", "40720", "27468", "67793", "39385", "55427", "47521", "60214", "39708", "57538", "21121", "53342", "65629", "42611", "21632", "63208", "55963", "32726", "3552", "38111", "42297", "71817", "35504", "11643", "38924", "45334", "33836", "22152", "33377", "67453", "50956", "30079", "55782", "17885", "47191", "35072", "55499", "69097", "68643", "10322", "166", "6818", "52175", "45991", "20726", "33966", "59155", "71085", "37539", "10995", "34518", "33946", "11623", "51097", "60054", "66480", "52039", "34043", "7673", "9457", "56226", "73197", "49992", "71924", "69412", "26149", "15235", "19794", "29838", "20969", "73100", "22721", "35623", "65162", "41969", "71741", "2600", "8393", "50738", "56821", "1270", "31726", "27203", "32320", "27703", "29326", "16269", "41417", "43503", "53191", "21651", "23259", "41335", "26823", "61593", "26518", "64038", "28487", "67959", "64710", "56432", "24973", "47247", "67859", "64972", "43591", "62770", "34922", "61191", "24521", "33500", "18433", "39948", "42169", "2478", "63788", "44039", "52289", "57790", "11486", "33133", "52853", "73138", "57758", "47575", "9414", "14535", "37435", "7092", "29722", "40710", "32349", "42632", "42506", "33280", "61341", "46815", "50383", "59088", "63766", "9021", "9623", "215", "58151", "40629", "60583", "63135", "6666", "38743", "4051", "22197", "39848", "23390", "40761", "12008", "69833", "29871", "62552", "60714", "39427", "69140", "39526", "56214", "67299", "28986", "52672", "51210", "22246", "22181", "24165", "64162", "56138", "21573", "57281", "31532", "38500", "28571", "61836", "43862", "59985", "15853", "46034", "61832", "65939", "47469", "70725", "36558", "62106", "37921", "56561", "891", "64281", "66055", "14273", "13337", "2079", "5099", "48077", "68243", "62277", "65505", "25953", "53183", "47444", "40426", "53371", "8886", "40849", "56111", "60156", "21327", "26038", "8168", "17634", "39735", "45247", "6519", "35936", "40918", "22414", "8210", "56260", "16983", "71430", "57089", "59147", "1769", "52263", "33179", "28924", "26471", "46188", "557", "43397", "38302", "24837", "56402", "41648", "301", "38481", "1704", "44648", "41735", "634", "58169", "66856", "31631", "41401", "12986", "57977", "11069", "53675", "19965", "18997", "51704", "13379", "35039", "45773", "63707", "51696", "11624", "49274", "66189", "27713", "45339", "49315", "9259", "36977", "55855", "48260", "14367", "49087", "63424", "37890", "35061", "10103", "43772", "42969", "71562", "69014", "71974", "38327", "36137", "2506", "1382", "58619", "43878", "66853", "56184", "69272", "34646", "65306", "59120", "14063", "55336", "62494", "68292", "32163", "23738", "66190", "12153", "29816", "69475", "67277", "68852", "12285", "47457", "10272", "65431", "48683", "27459", "43200", "59631", "62237", "55079", "34899", "52475", "39227", "47568", "9541", "15803", "12874", "30515", "23728", "1291", "43678", "6318", "15685", "49589", "71824", "38591", "54124", "30880", "23105", "11073", "8947", "21676", "23894", "37541", "23851", "23803", "31953", "66168", "65936", "43053", "16896", "37186", "19982", "48820", "48720", "52431", "62291", "27329", "19886", "12182", "57110", "19634", "59393", "25295", "9167", "4975", "62001", "59429", "4391", "72069", "72857", "52774", "45232", "49961", "15490", "27546", "29371", "32529", "22350", "3947", "16595", "60730", "7286", "33612", "35138", "55657", "31272", "60749", "56203", "47208", "10377", "69026", "43355", "71758", "61977", "35347", "72557", "11904", "2303", "31911", "64034", "9149", "38424", "54957", "39545", "59898", "3818", "36856", "60186", "48513", "44250", "54997", "71909", "4788", "1174", "6398", "60328", "71131", "26442", "10037", "24610", "64723", "20873", "66034", "588", "30180", "30156", "1632", "24307", "24606", "72221", "40751", "32116", "71411", "34435", "32713", "24023", "802", "40921", "72184", "35719", "72999", "7582", "15985", "31277", "37379", "24001", "47267", "29935", "53262", "20566", "38151", "46712", "11487", "5313", "28672", "14961", "50604", "69210", "44612", "23133", "66829", "53100", "10560", "34425", "8363", "26356", "739", "46632", "30648", "11145", "7233", "45344", "6146", "59499", "27749", "54916", "60403", "1829", "19766", "6633", "64221", "15254", "19312", "40245", "3193", "72230", "63525", "52446", "26141", "64371", "70695", "39377", "17417", "63506", "70040", "64685", "16811", "46795", "64526", "53890", "54026", "42202", "15226", "45765", "10713", "39002", "54740", "30687", "8353", "17434", "27011", "34500", "46946", "22814", "40707", "10890", "23238", "7201", "38108", "22728", "55665", "41121", "50314", "59297", "57581", "4781", "12032", "73248", "37159", "19996", "55352", "48824", "70045", "28254", "32270", "34541", "41725", "23384", "19944", "13128", "23619", "36865", "68000", "18959", "68393", "9722", "60002", "17534", "19632", "58565", "13201", "37569", "3273", "39933", "42016", "71132", "72347", "50413", "30940", "15665", "41300", "57798", "59073", "50208", "18482", "43867", "62632", "52199", "19716", "17591", "68253", "20222", "35511", "33375", "67343", "66436", "41382", "45251", "45938", "67696", "48808", "17204", "53090", "36577", "43381", "26360", "2297", "40490", "16599", "58620", "66664", "263", "6907", "5060", "6481", "19777", "40558", "17430", "45442", "4183", "69648", "55286", "13157", "23064", "40468", "31961", "48795", "34410", "58900", "58024", "8323", "58807", "5556", "24939", "59071", "52868", "72604", "9415", "54292", "37634", "71662", "71770", "50024", "21703", "72940", "3204", "30589", "50130", "44561", "27495", "70618", "22365", "35780", "2537", "71273", "32216", "21136", "19687", "46083", "64843", "31059", "44525", "55672", "64303", "13862", "44472", "57078", "6359", "16295", "38195", "65185", "42247", "56349", "11422", "6050", "66171", "63427", "44571", "57382", "37313", "40540", "45746", "52553", "146", "67306", "30335", "30185", "6127", "7621", "37608", "60204", "58727", "48912", "49062", "23931", "21420", "29985", "15003", "49023", "10140", "27194", "69023", "46192", "28909", "20194", "18792", "31145", "72822", "25586", "23120", "55677", "57534", "39453", "17575", "4113", "72790", "62140", "21393", "51927", "50757", "34265", "22692", "60894", "936", "31784", "41394", "43656", "46444", "22280", "61072", "60077", "57656", "49985", "3632", "67898", "17452", "1992", "54959", "6430", "18343", "34748", "50774", "61991", "25947", "16812", "50191", "6140", "6330", "64796", "2002", "9311", "6213", "44931", "14170", "45451", "42575", "66366", "4031", "23255", "34157", "17866", "4123", "8473", "65466", "7083", "22680", "30554", "16113", "56218", "48774", "26644", "38918", "7858", "31972", "44988", "25698", "46507", "60883", "27764", "66924", "46218", "15265", "59539", "21791", "58251", "50052", "48047", "61797", "28833", "71712", "56508", "54258", "17640", "72496", "56420", "15238", "63355", "46010", "51334", "68484", "48348", "49383", "63615", "18397", "52916", "55021", "28358", "59261", "43021", "27936", "32879", "25921", "34359", "18446", "27856", "33163", "26620", "56524", "906", "26053", "728", "11767", "32289", "27174", "47261", "6759", "63344", "29489", "58033", "37071", "16088", "51130", "35090", "52737", "45031", "61061", "10742", "28620", "1090", "30700", "19314", "61677", "36888", "19624", "4821", "9359", "34441", "49176", "811", "39052", "66693", "22745", "5179", "1801", "5422", "65489", "12771", "51629", "11902", "13563", "62209", "44099", "56876", "51143", "5231", "67789", "52771", "32695", "69664", "36886", "42092", "32165", "43572", "32368", "2183", "69498", "20229", "23046", "42770", "5789", "24938", "44692", "45988", "65777", "38585", "41472", "12172", "12185", "3232", "58496", "23807", "16518", "48337", "72059", "11943", "49785", "35735", "30514", "7983", "15048", "56118", "5264", "49917", "35340", "3041", "45172", "67530", "32590", "28832", "12799", "40621", "2810", "13018", "49560", "57617", "56721", "36468", "29020", "68251", "58633", "59106", "36897", "31266", "13446", "36408", "1391", "36862", "26174", "18780", "39420", "21494", "62996", "73087", "71129", "16709", "67721", "29543", "43928", "72117", "3363", "10467", "37499", "44891", "25169", "28067", "64259", "9555", "41860", "18213", "18403", "47093", "6088", "42521", "37977", "11744", "65839", "56914", "28960", "250", "40319", "34220", "29666", "26088", "50836", "70817", "28301", "2054", "27300", "33590", "61015", "11750", "7840", "31311", "72572", "4013", "32499", "18462", "43320", "63620", "43648", "17602", "13388", "16589", "41614", "47932", "58053", "64852", "27775", "52463", "1012", "32023", "24619", "60260", "49270", "19611", "35520", "29884", "16830", "34103", "29352", "44847", "58376", "1900", "70759", "29081", "30584", "23525", "44829", "45865", "40339", "8987", "2724", "50251", "22748", "48354", "3794", "39675", "9363", "54224", "15102", "33524", "59632", "52357", "56772", "38244", "15236", "26845", "1598", "19775", "29588", "63761", "26802", "12047", "14589", "1199", "38658", "11821", "51462", "52224", "50660", "51752", "26403", "2508", "37356", "42127", "45367", "35089", "30145", "54314", "7641", "61717", "18394", "21974", "10693", "70238", "64794", "68374", "27501", "49695", "65030", "51139", "38992", "64832", "68170", "23682", "33021", "65265", "11783", "43619", "39265", "10813", "45191", "59170", "44906", "48987", "19725", "41757", "10674", "56080", "21604", "65747", "2545", "44436", "66255", "70073", "48463", "59573", "69611", "5654", "41038", "48482", "34927", "66700", "33562", "63913", "33631", "49963", "22800", "15049", "10381", "33874", "19920", "11652", "29519", "31653", "60887", "59326", "12962", "27192", "4351", "18656", "13922", "25808", "66291", "13676", "37756", "72558", "22690", "67592", "61204", "24314", "22915", "623", "45274", "30942", "113", "65581", "65520", "2095", "3633", "37843", "7033", "47116", "7595", "50385", "54514", "66372", "46530", "25506", "8666", "52129", "5678", "48530", "65237", "58474", "63987", "71340", "41681", "29936", "27450", "8985", "28775", "14936", "52994", "59888", "62576", "45491", "18641", "52416", "21943", "13866", "23903", "21762", "60591", "8189", "4132", "8286", "9791", "56361", "32439", "55692", "2363", "33775", "8565", "12167", "20904", "68387", "15604", "37184", "53619", "27759", "143", "41299", "67126", "68707", "49470", "46578", "55435", "50694", "70518", "25285", "32812", "11533", "29507", "51689", "29805", "66419", "51467", "4916", "28087", "13268", "34983", "25470", "43785", "48002", "20689", "16320", "5329", "2843", "32413", "33464", "30911", "15326", "42036", "54741", "54059", "8090", "46321", "34324", "33582", "18303", "62706", "19678", "41841", "67953", "33393", "58791", "21465", "69143", "30733", "17026", "39822", "52349", "71890", "48110", "69605", "5986", "63753", "50023", "36294", "40772", "13458", "33976", "3615", "70075", "25948", "53277", "26899", "29015", "63285", "15668", "13982", "68198", "29303", "49262", "52069", "65956", "18753", "19936", "40968", "67992", "58436", "70890", "22174", "43362", "27927", "22671", "64974", "6195", "14900", "33787", "69967", "3877", "290", "16512", "31691", "7446", "57529", "42368", "7702", "37984", "63487", "40957", "62538", "26904", "21947", "22788", "6928", "68316", "17837", "20569", "18653", "70251", "4001", "3159", "9183", "45403", "54227", "43248", "70539", "3769", "901", "34285", "39618", "72806", "66993", "35680", "16778", "57451", "35327", "4631", "66661", "17568", "5451", "49160", "159", "51170", "55889", "3268", "60957", "13417", "53648", "28204", "41483", "44329", "9996", "45052", "20369", "47741", "25811", "26611", "43267", "66018", "31517", "36794", "20395", "15895", "6302", "26616", "65770", "46077", "49365", "61708", "29584", "23655", "2449", "47391", "71816", "59571", "61095", "50310", "66495", "61194", "47623", "29621", "26211", "50393", "59440", "5120", "66804", "1865", "50921", "45328", "29178", "46546", "67365", "42494", "51368", "9814", "7001", "22049", "9110", "14827", "68222", "15715", "72349", "48805", "72719", "59977", "42511", "25677", "8006", "17175", "2842", "23554", "46064", "31293", "67798", "61171", "67758", "72223", "15932", "59923", "22384", "5761", "5696", "31124", "40595", "34915", "66605", "16049", "7968", "28489", "34895", "53148", "51293", "38113", "24581", "57954", "16798", "70013", "12785", "2914", "12737", "4133", "55546", "41108", "29412", "44118", "4392", "44851", "911", "37696", "42863", "26812", "57264", "3610", "6632", "28918", "8416", "4489", "23801", "10479", "9844", "48367", "18074", "43469", "44914", "65226", "11058", "40104", "9697", "16495", "30303", "49413", "69227", "33112", "70541", "4131", "8614", "45993", "21876", "29892", "36552", "46053", "55090", "4254", "48072", "9026", "22064", "37282", "62766", "34350", "59419", "69391", "25705", "11590", "66195", "49411", "58494", "56515", "69156", "25591", "6697", "54138", "14407", "21882", "46738", "65880", "62189", "25712", "42311", "4799", "31541", "12533", "59222", "24574", "17611", "59450", "65800", "20332", "16476", "12885", "439", "1892", "19302", "33841", "14119", "46774", "58301", "13807", "9561", "73019", "36772", "62874", "67667", "34819", "6316", "49527", "32702", "10351", "9351", "53265", "30792", "27072", "35531", "38674", "30765", "13824", "6227", "41939", "72394", "26699", "48144", "41026", "40731", "1396", "41702", "21230", "38897", "46389", "50186", "2227", "9485", "25278", "70909", "62290", "65125", "30009", "50891", "33258", "23501", "24495", "64087", "56836", "59156", "35816", "10471", "45144", "17980", "29590", "2321", "191", "50193", "65805", "11653", "38709", "72538", "12023", "43031", "10850", "59796", "68993", "9010", "71171", "67552", "38031", "16577", "16835", "2287", "16118", "28484", "4998", "25912", "12040", "1805", "68775", "43773", "7653", "2316", "27013", "66703", "27258", "19177", "67501", "29896", "9792", "64600", "50989", "56740", "39084", "5260", "38161", "51649", "27207", "27441", "24461", "62814", "64830", "60467", "66863", "8771", "5478", "23079", "2009", "3793", "21485", "18810", "19378", "29795", "22101", "42843", "17906", "71913", "19451", "61349", "5", "35746", "32814", "58636", "48611", "70661", "16541", "40064", "4196", "39746", "24598", "49303", "55653", "60738", "35442", "34481", "58191", "28719", "71515", "35602", "30429", "9818", "45132", "67519", "29606", "42380", "17768", "52016", "6660", "31134", "66546", "27717", "59744", "13636", "41217", "43191", "19360", "13969", "8040", "65388", "23704", "10816", "63754", "17127", "58493", "49526", "8384", "57695", "49436", "46920", "65795", "60434", "71438", "14504", "63680", "63932", "33698", "8735", "67214", "54994", "32308", "44809", "46967", "60976", "52607", "13587", "33332", "5893", "15487", "71455", "54398", "22117", "62991", "70115", "55144", "22474", "63190", "3282", "6568", "10067", "42280", "18599", "12950", "67045", "65307", "55475", "56309", "14057", "45415", "4414", "23218", "59308", "13847", "70487", "69273", "39459", "35235", "39240", "50780", "40754", "31377", "19420", "66482", "21790", "24011", "71267", "64574", "61835", "51165", "62109", "55446", "71834", "35157", "8738", "46683", "17735", "69383", "27842", "6741", "19475", "41117", "4084", "36053", "62396", "54768", "44574", "49121", "71323", "44078", "58536", "52663", "58588", "9961", "56321", "47230", "44941", "53404", "3690", "31634", "37449", "25604", "43212", "8112", "54657", "59313", "11272", "54219", "68290", "20928", "34431", "34731", "43802", "68635", "35665", "25277", "19956", "71111", "10164", "39636", "23091", "61143", "42217", "53103", "303", "69640", "54592", "70565", "44521", "48696", "33642", "38788", "47619", "66662", "48765", "65609", "59431", "998", "53192", "61182", "33475", "28035", "69356", "34040", "52418", "33239", "24435", "32652", "17457", "52651", "42921", "37855", "70330", "10776", "29060", "49832", "16684", "45998", "48848", "45466", "44237", "7763", "16921", "63860", "60449", "1693", "12841", "25488", "52152", "45550", "20946", "15207", "25699", "5259", "55664", "45134", "14672", "43835", "14460", "10725", "22812", "20799", "39433", "52587", "52922", "48452", "38260", "7841", "61440", "12194", "5516", "57895", "32056", "32554", "18799", "57038", "54539", "65274", "705", "65907", "39445", "30710", "70827", "47572", "1431", "2378", "25548", "70879", "5444", "34802", "65993", "45398", "18041", "41337", "65848", "65752", "3508", "53005", "40783", "65002", "40717", "61757", "59455", "40593", "17381", "40639", "43608", "65804", "52476", "52346", "52974", "28145", "44848", "15439", "35037", "60867", "59645", "72944", "54389", "341", "10520", "55053", "20365", "21933", "31882", "38122", "28945", "37527", "59838", "64579", "11536", "69516", "34381", "44488", "21447", "70939", "17349", "45874", "46848", "34181", "37775", "7170", "63328", "73234", "15968", "60182", "65067", "4275", "52096", "29570", "57214", "13806", "58884", "49561", "67156", "461", "28575", "62", "40407", "63785", "60443", "25303", "51575", "12842", "45360", "18749", "47812", "37735", "47295", "62747", "5098", "45060", "64139", "2753", "8949", "7542", "1066", "53560", "33585", "50560", "69258", "30557", "55839", "40654", "48630", "26614", "24840", "41790", "61975", "61501", "4988", "30478", "32684", "36595", "55703", "64094", "13041", "35659", "29404", "39792", "42844", "50804", "2548", "40911", "24248", "9976", "7752", "31223", "72663", "45222", "6775", "45948", "57413", "369", "66973", "55772", "23749", "10539", "13639", "55599", "62387", "37558", "15681", "16263", "38448", "6057", "16828", "3252", "35903", "27096", "50552", "9750", "47644", "53403", "67023", "14233", "18767", "68195", "30168", "49755", "54724", "11570", "3863", "24762", "13107", "70097", "38802", "44852", "2405", "49346", "31813", "26202", "58115", "31531", "64401", "66135", "59473", "44255", "41823", "43759", "68499", "38634", "54474", "36763", "8775", "6788", "57709", "32912", "19863", "16125", "52551", "12168", "5855", "22284", "43574", "57831", "65639", "63755", "37688", "53664", "34840", "51562", "37366", "9147", "21300", "66263", "31421", "38063", "42343", "58696", "48843", "55602", "27161", "36624", "409", "44996", "65212", "61856", "71531", "10215", "60748", "17073", "33799", "32283", "25570", "52824", "22642", "58985", "39614", "53764", "71197", "70456", "16576", "43046", "68511", "21529", "63331", "31740", "48145", "49954", "63814", "3886", "67266", "27618", "64146", "22828", "67641", "20532", "60061", "52170", "50346", "27098", "1067", "46627", "59414", "42635", "18961", "46080", "11125", "32356", "71110", "69025", "67490", "57372", "64537", "33150", "63324", "51099", "37895", "10926", "47021", "31651", "52745", "6499", "71296", "36787", "39690", "15765", "30091", "42721", "70473", "15645", "46027", "3827", "5485", "18249", "33778", "53508", "3187", "24640", "10755", "71100", "16635", "54442", "52635", "5450", "41624", "1471", "71542", "62969", "50242", "5705", "16131", "55536", "52189", "39916", "49983", "70610", "44995", "8681", "24627", "6838", "36521", "23235", "16654", "15224", "41002", "66463", "43818", "67847", "36381", "31887", "7830", "28004", "20800", "41118", "18165", "36885", "71525", "62161", "5255", "33213", "48217", "25139", "58949", "5129", "54062", "44113", "34865", "36812", "21352", "21220", "14530", "60490", "63857", "66842", "41638", "18359", "48399", "13448", "31663", "49235", "23256", "25590", "67606", "18624", "17144", "45936", "33815", "3918", "620", "51961", "70421", "51637", "809", "34883", "22228", "51888", "12682", "68432", "12717", "56401", "25530", "7468", "9775", "37773", "15436", "9810", "42242", "48751", "42266", "14010", "55076", "134", "36373", "39231", "28579", "65436", "69287", "14540", "60875", "41010", "71236", "59763", "25464", "45332", "51626", "21359", "272", "65498", "48218", "7586", "26271", "38889", "55487", "43415", "11896", "39262", "42187", "63563", "42790", "31801", "59529", "23024", "63583", "25879", "33987", "39469", "67198", "67120", "65988", "33755", "55954", "35071", "57842", "23742", "14574", "44987", "59804", "31878", "44297", "29348", "46704", "50946", "62907", "33115", "5875", "17877", "56831", "12787", "39484", "37040", "19955", "36985", "41565", "21410", "5092", "17968", "66258", "8959", "64792", "33954", "21231", "11726", "52091", "5037", "29993", "67619", "55129", "70387", "44238", "48191", "62243", "43032", "72478", "47117", "7781", "63240", "41068", "11289", "37627", "52415", "46049", "18214", "48477", "36295", "54260", "61267", "37017", "8505", "7417", "69135", "34503", "28288", "43945", "11268", "62036", "47729", "33153", "4239", "49580", "64655", "53793", "45932", "35883", "65974", "52208", "42509", "3274", "552", "61359", "36054", "34614", "32620", "43595", "27279", "56790", "7554", "34111", "24803", "2910", "50279", "34382", "31997", "56674", "42194", "12065", "18132", "42967", "38304", "32297", "29437", "40561", "29268", "30764", "35764", "7646", "22279", "6239", "63920", "59639", "5209", "50890", "60198", "44817", "53021", "11660", "70547", "63988", "24368", "48076", "25568", "8558", "28581", "16242", "47171", "17665", "1209", "41042", "12148", "55295", "62540", "12560", "57444", "22214", "66881", "41763", "31358", "47773", "46901", "64489", "70881", "41792", "31859", "62820", "61482", "7101", "52835", "9307", "38177", "37212", "40085", "17949", "53789", "58373", "25274", "11310", "51079", "43440", "34841", "56056", "8667", "51344", "69554", "54900", "57847", "58786", "52027", "14424", "30943", "42518", "43952", "34509", "63335", "35963", "35181", "42272", "646", "58888", "47447", "4886", "15166", "70249", "72595", "58660", "70264", "70954", "13225", "30803", "16378", "71282", "72509", "35339", "51829", "30142", "26010", "18893", "20243", "66589", "23355", "14043", "47226", "27633", "72130", "23906", "44549", "3411", "38456", "15632", "6066", "30294", "37718", "19898", "58522", "1737", "26549", "68422", "48777", "72071", "51486", "23933", "58309", "23865", "60483", "28678", "642", "58551", "42275", "4574", "40833", "9019", "40740", "63350", "1591", "9111", "37206", "66266", "12358", "57664", "43758", "15545", "57081", "7065", "47012", "64837", "11503", "36774", "68669", "40714", "34928", "46948", "69387", "57277", "37871", "64278", "15498", "6334", "37671", "26764", "7301", "6000", "52656", "48812", "54509", "35007", "27896", "70930", "6799", "24731", "20939", "36482", "20633", "71889", "57495", "66325", "40280", "5805", "72213", "1136", "14797", "3897", "36752", "24285", "49175", "22405", "61065", "28492", "58805", "6221", "20429", "1455", "43432", "5076", "48444", "4217", "35286", "24870", "48985", "54905", "2376", "47360", "67180", "69937", "56495", "55055", "50142", "6434", "46669", "18803", "45714", "69381", "47673", "49910", "38096", "12829", "37142", "37326", "30955", "12572", "43729", "58802", "45181", "65203", "18672", "63875", "53621", "65266", "59890", "60932", "51661", "37344", "42695", "38361", "35432", "53867", "59582", "38265", "3420", "71004", "57007", "3917", "33665", "68036", "72720", "489", "59896", "65774", "6468", "63577", "59359", "54073", "38981", "32693", "61303", "26591", "4942", "21695", "10084", "20006", "31745", "41804", "43435", "52850", "62103", "71036", "25450", "18721", "68423", "4462", "11788", "15605", "71607", "36257", "71985", "62775", "26925", "38748", "54990", "54566", "66784", "62437", "34406", "973", "34543", "40922", "9806", "40631", "34551", "59385", "43549", "15178", "72475", "9640", "52038", "51312", "31405", "25402", "49071", "32653", "65140", "71979", "70466", "48188", "23237", "26028", "53411", "54369", "42779", "49802", "56152", "55296", "61515", "45683", "71196", "37414", "30924", "72175", "3149", "4822", "43287", "36506", "57863", "3683", "29427", "14252", "63283", "49195", "8599", "17687", "44257", "37411", "7066", "44050", "53150", "61696", "56550", "54888", "66821", "59190", "7057", "23232", "68347", "54924", "3539", "66731", "35918", "66530", "69954", "12240", "26272", "52854", "66800", "30115", "16679", "49849", "52818", "55207", "59110", "18648", "45646", "52151", "6281", "29680", "33477", "4213", "12372", "65076", "1696", "54253", "8682", "20732", "13292", "66144", "10436", "21275", "34116", "14645", "63873", "43149", "7373", "1110", "66732", "60900", "41821", "33386", "36208", "68915", "63241", "61019", "64739", "22830", "59436", "41174", "8281", "3405", "49222", "39014", "25380", "27101", "46120", "30197", "3454", "22891", "68899", "23226", "43685", "65176", "51028", "21374", "12274", "71388", "17303", "58150", "20782", "1996", "20333", "48328", "69086", "40216", "23990", "32280", "66335", "8115", "57814", "37757", "56227", "61507", "23754", "67413", "72792", "44007", "58464", "54968", "5549", "51091", "38094", "72619", "2432", "3954", "50991", "39563", "34695", "22617", "27140", "27058", "55461", "30434", "38371", "24266", "32129", "71176", "58398", "36286", "54559", "59597", "6471", "66338", "19596", "37008", "58630", "1936", "42566", "36347", "69620", "13335", "21591", "2568", "31012", "49874", "30364", "36660", "61584", "43839", "28331", "61334", "45030", "63054", "38564", "54288", "46301", "40996", "10586", "45709", "24124", "26231", "18817", "18071", "4652", "48956", "46686", "22441", "46069", "68849", "53826", "43760", "73069", "67098", "42479", "20681", "41929", "46589", "12394", "28418", "49265", "18495", "27326", "5199", "7610", "62076", "10670", "55847", "24850", "15483", "19158", "71226", "46651", "35326", "63606", "48142", "67430", "25311", "58725", "71313", "18920", "3907", "6837", "31302", "38575", "40452", "14957", "6225", "1361", "48894", "10456", "57736", "1768", "33010", "25072", "43706", "31665", "38613", "22831", "45093", "3952", "11850", "70163", "61298", "11648", "16475", "36500", "21323", "63614", "14088", "56729", "54987", "49999", "49780", "47428", "4693", "56516", "18541", "41433", "58564", "40364", "22665", "59719", "61286", "47808", "2607", "20285", "37317", "72763", "61232", "54719", "33798", "49649", "2979", "11222", "6385", "31250", "8709", "48485", "48863", "5088", "25637", "28211", "68578", "47612", "54437", "57266", "37235", "31729", "43874", "39647", "28695", "72263", "46975", "61456", "48400", "24152", "20267", "49123", "66227", "49336", "30853", "40655", "530", "719", "49219", "54100", "9377", "70548", "42897", "20857", "35316", "10929", "48575", "3775", "30448", "51279", "28267", "17904", "50200", "9239", "39088", "39940", "47539", "42548", "10293", "59562", "32872", "19373", "23898", "12120", "15239", "44857", "26228", "53273", "25234", "54808", "216", "46748", "68655", "29258", "36671", "517", "19756", "61159", "6498", "38533", "68399", "52687", "31071", "33860", "73111", "14226", "41020", "15800", "72128", "43398", "38665", "66904", "59397", "22026", "62663", "41099", "24432", "7716", "71071", "57003", "3048", "67116", "41663", "47656", "37969", "39438", "70352", "59613", "57355", "8965", "41298", "546", "27570", "44823", "32960", "29765", "42651", "6034", "29983", "13118", "54144", "69906", "45155", "13991", "16333", "55343", "12483", "38496", "50963", "34064", "17607", "50903", "45184", "22562", "3811", "33742", "27978", "68053", "54238", "60133", "12189", "62007", "65597", "13241", "7694", "66167", "38418", "37736", "3080", "13817", "38132", "15827", "28559", "69945", "25333", "48700", "67814", "71937", "54922", "65311", "73049", "49927", "16508", "43887", "48108", "3150", "68726", "63751", "16637", "12227", "26798", "34843", "71124", "68348", "44389", "22021", "28861", "11214", "26807", "70222", "73192", "12402", "69799", "41314", "67908", "38921", "56875", "29730", "50073", "13486", "42088", "29992", "48495", "50977", "68055", "1221", "31969", "32173", "50497", "70899", "68671", "47991", "10460", "69189", "52433", "829", "66850", "39135", "36518", "28978", "10348", "15240", "3751", "65510", "7566", "40947", "33222", "71308", "17928", "64007", "52698", "23884", "61292", "40545", "45358", "72004", "45890", "255", "56172", "35419", "66852", "28183", "29030", "61111", "32196", "3378", "66407", "51093", "39913", "42507", "72830", "52848", "61231", "6719", "51833", "48030", "72866", "6452", "22683", "16321", "57146", "21772", "32570", "41418", "5105", "23737", "54967", "65976", "63348", "19536", "64011", "3277", "13078", "52316", "20709", "3796", "14806", "40652", "31158", "16704", "53690", "66758", "16296", "10447", "4214", "25159", "64957", "20110", "21372", "11163", "14503", "63895", "39389", "17884", "30502", "52861", "23616", "27516", "53787", "5675", "34287", "8969", "16603", "61386", "23199", "43119", "27408", "36910", "22508", "68656", "30244", "38011", "38686", "29315", "60041", "45003", "59983", "44004", "46943", "67066", "40095", "65052", "50554", "63160", "3582", "28106", "65355", "27998", "22244", "20140", "1990", "29981", "4371", "53244", "40093", "3024", "7828", "14018", "49974", "16094", "49463", "30241", "50644", "68471", "23220", "39457", "17892", "20027", "45460", "1516", "64133", "22381", "17653", "58653", "12036", "59512", "24154", "64006", "27440", "63782", "40675", "71505", "67145", "26850", "30128", "16691", "20397", "6070", "38610", "31231", "16164", "5689", "15310", "32779", "49257", "53010", "51204", "60012", "52620", "69693", "27448", "54817", "54652", "5599", "38983", "51570", "24512", "41265", "9632", "55865", "34512", "60804", "4996", "50264", "9482", "54020", "33465", "49782", "19265", "4957", "54716", "25937", "34636", "21468", "68580", "67062", "37789", "17361", "50418", "70429", "24665", "54307", "9176", "24642", "61397", "23394", "27", "38529", "46574", "64808", "8882", "58004", "60427", "61336", "38779", "16561", "37684", "23917", "58242", "58638", "10119", "53252", "27462", "21649", "1491", "50814", "70313", "24528", "4412", "5408", "39556", "16104", "62111", "9519", "46383", "29462", "49119", "61511", "8419", "9205", "9862", "23788", "65756", "38362", "323", "64360", "50207", "724", "73164", "47643", "71668", "3784", "67526", "58920", "26515", "9136", "43159", "39210", "73003", "559", "1564", "58601", "17926", "68604", "12409", "37900", "60773", "72135", "1034", "37376", "45194", "60013", "44532", "13269", "162", "13855", "56195", "13134", "722", "17348", "23201", "63576", "24474", "71946", "6961", "17311", "41911", "55233", "12923", "56499", "73235", "15416", "35977", "1047", "30363", "16797", "22046", "51236", "33638", "71888", "19884", "34845", "72688", "44863", "61161", "23324", "1383", "36647", "41568", "4337", "47636", "51505", "21077", "24099", "45590", "4907", "50490", "1428", "778", "37327", "69112", "23615", "6247", "48526", "31181", "64919", "50625", "40030", "32969", "66197", "12396", "67819", "1169", "45703", "196", "46170", "37200", "38576", "53872", "66893", "3737", "47596", "65592", "52310", "6248", "19485", "16846", "69294", "55057", "66859", "55449", "61681", "60306", "71163", "61351", "27995", "71602", "42687", "38194", "60092", "49177", "11762", "39269", "58225", "60430", "70893", "56017", "28291", "22357", "29341", "22964", "28787", "29977", "8897", "45384", "52470", "24855", "59906", "8101", "12475", "13186", "62683", "65684", "69845", "66944", "59059", "18085", "5175", "42170", "14651", "465", "17645", "17429", "22283", "8824", "8863", "8491", "49834", "3477", "664", "62934", "51647", "71436", "52546", "45243", "31457", "36274", "25951", "6579", "71957", "41749", "42431", "44090", "11383", "48141", "40686", "22200", "19005", "67561", "24187", "70903", "33252", "23868", "47054", "72294", "50410", "36700", "35662", "50373", "61283", "32086", "43127", "39944", "49826", "31326", "29242", "17355", "4798", "54217", "69742", "64533", "64361", "50935", "20884", "10088", "11840", "53239", "1170", "33186", "65211", "29678", "57930", "53754", "47197", "8388", "21428", "54231", "47069", "17407", "32409", "53882", "38756", "18475", "58576", "22155", "71272", "24931", "61570", "24317", "10284", "69669", "4533", "48330", "24064", "13576", "31242", "41718", "51857", "50268", "71012", "9331", "6709", "61540", "70919", "34857", "23861", "5002", "39237", "36455", "25777", "47512", "48362", "64405", "25298", "35175", "4551", "15010", "73140", "49530", "40456", "44880", "56934", "61209", "6301", "39255", "22672", "52048", "11307", "3109", "33522", "29155", "72363", "56672", "12713", "65298", "27632", "17184", "52918", "72686", "34463", "32054", "52260", "14148", "35223", "18434", "40779", "61278", "5683", "24092", "20902", "17819", "7973", "8258", "5735", "42097", "54609", "64545", "57803", "11410", "28384", "22212", "2653", "36409", "43941", "26199", "19764", "24866", "23603", "3327", "7997", "57640", "39249", "53174", "18752", "21661", "23988", "43092", "63248", "13984", "2352", "39801", "69164", "60132", "20237", "72465", "39349", "69007", "23069", "59737", "4407", "29501", "34969", "25688", "54212", "49310", "42143", "26685", "45928", "20542", "11215", "16421", "67804", "9428", "65672", "2296", "69063", "26383", "12852", "51861", "25451", "67503", "37424", "39404", "50522", "57893", "32156", "54143", "2684", "60938", "70192", "41281", "14324", "3906", "16284", "57199", "48879", "6592", "47182", "52534", "31274", "66484", "28380", "37384", "34152", "32632", "60740", "57336", "11847", "15836", "30119", "12329", "4008", "36649", "42517", "36223", "69737", "18660", "1092", "53146", "36601", "4773", "37498", "29736", "50432", "69470", "24204", "13151", "46892", "34035", "58890", "48349", "39431", "52444", "25975", "18455", "28482", "36132", "11105", "35055", "22153", "7852", "3518", "40875", "44871", "3304", "14143", "66130", "8595", "64403", "46142", "66070", "25084", "29648", "54279", "40009", "51034", "19645", "69163", "70152", "16696", "11328", "26749", "13142", "22571", "10159", "65641", "10061", "70569", "5296", "43406", "17232", "64602", "61031", "23263", "4738", "48314", "8494", "16687", "28906", "28323", "15350", "66202", "31235", "60299", "25762", "19732", "53383", "40938", "10832", "57659", "45642", "39415", "36168", "71067", "2030", "36190", "21655", "17864", "19090", "62325", "9093", "26756", "50825", "54282", "3551", "57859", "798", "58555", "68441", "55191", "57141", "45904", "31950", "29039", "5326", "41129", "68185", "62660", "3457", "68565", "66585", "69370", "28463", "567", "56834", "32479", "72639", "5985", "63226", "2164", "50429", "44158", "30494", "10410", "24342", "8768", "16856", "22188", "58795", "1633", "45359", "1903", "39924", "38824", "6781", "4121", "7243", "60282", "66574", "1500", "55910", "58537", "55538", "53444", "29101", "13102", "48468", "70722", "14564", "50574", "15292", "72868", "36625", "29767", "39600", "30548", "17549", "40439", "29604", "71929", "25933", "11843", "23503", "1551", "6956", "70937", "53796", "16030", "54223", "27776", "992", "63811", "3721", "52219", "635", "60435", "43814", "35457", "17757", "33127", "32964", "44316", "34242", "5055", "73044", "29591", "66027", "54339", "3340", "70087", "53286", "1003", "49231", "54511", "52348", "39323", "66587", "72552", "59567", "42960", "30582", "5890", "53304", "29035", "34733", "21720", "66226", "55742", "47708", "44022", "32730", "8689", "50909", "1476", "19738", "2768", "35014", "33236", "19882", "57574", "59316", "16605", "37680", "43018", "19588", "33427", "7798", "68015", "38734", "11710", "4384", "4726", "34368", "27885", "20868", "70914", "2420", "15651", "33191", "8660", "59138", "11717", "52802", "41834", "45449", "7665", "49403", "6940", "27195", "67025", "17099", "42206", "63726", "24427", "656", "15913", "61524", "27793", "68793", "35854", "62033", "12002", "52894", "61320", "39823", "67515", "68245", "11752", "7738", "8163", "61658", "70876", "7120", "20206", "17276", "55945", "40484", "66123", "27735", "12275", "47862", "65898", "55757", "68632", "5833", "29274", "23422", "12868", "4522", "10310", "68873", "53179", "47163", "52615", "8581", "67082", "37303", "13400", "1404", "41374", "32896", "71373", "1957", "19165", "23036", "9328", "69206", "7922", "57322", "39130", "9761", "51605", "65604", "24400", "41294", "16224", "4515", "69474", "54862", "7321", "3713", "47997", "36001", "34847", "21804", "10400", "44144", "28685", "55486", "71708", "60453", "21241", "23479", "61100", "69566", "22535", "55484", "67572", "3002", "37706", "50491", "14493", "5210", "54699", "13773", "17577", "23722", "56288", "54204", "63903", "31118", "55615", "6030", "47471", "71688", "60970", "60691", "7271", "70819", "21670", "7569", "41514", "17765", "44040", "22970", "20560", "59572", "66752", "57670", "18046", "37840", "42803", "39153", "64246", "41651", "68338", "5743", "30399", "24588", "11106", "44761", "9977", "52702", "40096", "52804", "8017", "68584", "58941", "42152", "30900", "61749", "5829", "47662", "37302", "9146", "53221", "38103", "8703", "37618", "21690", "14478", "22543", "7825", "6292", "46214", "18670", "43030", "36742", "39992", "14978", "29218", "12886", "47451", "13149", "42669", "57599", "14601", "59048", "50203", "43067", "55946", "27934", "45659", "7678", "12471", "26344", "46149", "17777", "28628", "44163", "46114", "42712", "4834", "14450", "66078", "52594", "43872", "13362", "16951", "9968", "11020", "11604", "2892", "33808", "2884", "13053", "71154", "67931", "54878", "514", "5152", "1727", "42051", "54275", "15573", "33285", "2961", "29748", "66385", "22439", "32913", "50006", "41577", "22695", "6451", "55210", "59604", "39467", "59201", "27272", "71802", "4299", "9440", "59366", "21726", "73226", "7390", "58101", "39472", "69562", "16216", "54365", "63063", "30542", "50980", "58509", "17638", "32206", "12555", "26297", "69933", "44913", "23949", "33361", "63793", "69013", "54097", "23177", "30549", "28844", "72056", "67909", "55224", "61099", "58427", "45797", "27159", "62108", "2656", "6223", "23328", "50551", "8930", "61499", "2230", "32133", "13711", "41898", "19713", "53431", "27906", "70962", "15012", "6210", "68742", "4954", "63849", "5401", "34234", "9275", "54465", "44349", "39055", "47425", "15059", "67557", "52946", "23050", "11633", "29309", "44279", "31328", "54939", "18294", "8284", "36904", "13098", "226", "65051", "39371", "13056", "23624", "60943", "68340", "20198", "7630", "43539", "67744", "59237", "38611", "51310", "10223", "60251", "43993", "33394", "23185", "71445", "36858", "95", "50215", "17876", "25556", "9159", "18040", "50371", "60486", "64274", "14242", "2468", "59140", "5212", "47532", "27142", "9204", "4003", "69188", "53557", "17268", "2644", "43482", "9580", "50492", "22237", "12974", "27008", "69224", "72740", "52284", "8130", "50949", "66076", "49479", "11556", "50124", "43375", "274", "45911", "9909", "27595", "44653", "46311", "7955", "65759", "26401", "35115", "68688", "57564", "63777", "3158", "2866", "68380", "33161", "54379", "38024", "1045", "31671", "13244", "5155", "37948", "35216", "4474", "19533", "44926", "27051", "64620", "60823", "60611", "22093", "19350", "61724", "11072", "30622", "42309", "16681", "29827", "62598", "61719", "46412", "53574", "51889", "70428", "43291", "58547", "34946", "51683", "49046", "18447", "939", "49499", "59154", "27646", "72160", "43026", "69757", "15787", "45638", "61863", "47268", "25125", "66681", "54397", "35908", "6025", "33566", "21672", "29691", "58628", "17096", "17783", "71363", "72198", "24242", "67202", "20366", "35885", "23538", "35523", "16762", "25073", "37749", "69584", "67118", "4487", "48722", "8817", "30776", "21594", "14236", "18229", "20510", "5584", "7310", "65723", "44200", "16772", "48234", "41212", "61971", "38215", "48768", "11859", "57891", "29338", "59302", "67807", "71441", "53927", "66818", "22391", "10297", "305", "5882", "26239", "57626", "43841", "3105", "38501", "68944", "17974", "4590", "58343", "2313", "33850", "51287", "54175", "33380", "64399", "12778", "38370", "26645", "55192", "13518", "32147", "39577", "70400", "14144", "35830", "14379", "35958", "72154", "15519", "1973", "48533", "27137", "16298", "60452", "44808", "50047", "2152", "46886", "39900", "63633", "42013", "61465", "9355", "15525", "41404", "25028", "18542", "30953", "32789", "49078", "39168", "31990", "64427", "64043", "33558", "28289", "40915", "43011", "11488", "42191", "68729", "6974", "2082", "9058", "45178", "69138", "30790", "31588", "63459", "57811", "25369", "15318", "50863", "73239", "67444", "14550", "2106", "47373", "4442", "32839", "32954", "33777", "63436", "47995", "61362", "11728", "12134", "66562", "23557", "45769", "7104", "39663", "8422", "25974", "69137", "59050", "8628", "54444", "62400", "6511", "42078", "6350", "19262", "29259", "61198", "6939", "39278", "56350", "18477", "3444", "4880", "53202", "24720", "31958", "45473", "52223", "17737", "10896", "2595", "23833", "67738", "17250", "11386", "25798", "44821", "59247", "65490", "60005", "46740", "61773", "45809", "9358", "70705", "44076", "19018", "62030", "18069", "51521", "68433", "68861", "55438", "37132", "3285", "72636", "45780", "34849", "23277", "61786", "39077", "1762", "67687", "6777", "8851", "25715", "16733", "63976", "32272", "41978", "8830", "23961", "72494", "43575", "9865", "15540", "11972", "44378", "32747", "61857", "41592", "40573", "8687", "13385", "1124", "9891", "62443", "17921", "13133", "42849", "58498", "46154", "2649", "67648", "25768", "33369", "54252", "52549", "55417", "7929", "14944", "66381", "36278", "63017", "11786", "43561", "50541", "1465", "32251", "70459", "64218", "56323", "17450", "9466", "65745", "39125", "9222", "15139", "54352", "58573", "59379", "63468", "43674", "32772", "38466", "68266", "47343", "43548", "45833", "66017", "3117", "30441", "59627", "42219", "50426", "59303", "56924", "25944", "33981", "10292", "42589", "34620", "44345", "19620", "38454", "19615", "12289", "32742", "14068", "51174", "2374", "30595", "51929", "66314", "70860", "13156", "45974", "66380", "33633", "4845", "55845", "38975", "41549", "7423", "44777", "52211", "26915", "69574", "20877", "51393", "47477", "7887", "30111", "45208", "67110", "71987", "16881", "23098", "13039", "12314", "32576", "65135", "67937", "19058", "73090", "62640", "19718", "37119", "42945", "48114", "7565", "3600", "8657", "57477", "6728", "46811", "57275", "14113", "44014", "35537", "43276", "23549", "36906", "36387", "28568", "8792", "71846", "33026", "32267", "43076", "43616", "23087", "64208", "52603", "60596", "43481", "27211", "18243", "12850", "18619", "19080", "3644", "29322", "58361", "41907", "6478", "5193", "62840", "61266", "49389", "60487", "24867", "66489", "28439", "15199", "32523", "56956", "71416", "71181", "63980", "42130", "40383", "25354", "69101", "24677", "53554", "10236", "42052", "32294", "57542", "54126", "21226", "20855", "64524", "47017", "12907", "71517", "47527", "13542", "17540", "64847", "70785", "52908", "61380", "62618", "51565", "33845", "17594", "10440", "52051", "57489", "28449", "34972", "8653", "43978", "36829", "4869", "32889", "49339", "29714", "42568", "24544", "49914", "53663", "58000", "63158", "33679", "61738", "42336", "19871", "44861", "66003", "67700", "51406", "8094", "34479", "35930", "39892", "64656", "8781", "38135", "37582", "36715", "48567", "72977", "26580", "31128", "9724", "51790", "36783", "20703", "59404", "14699", "29405", "70441", "38073", "8147", "12702", "19495", "25063", "62974", "44843", "29184", "26309", "51811", "54354", "15294", "48815", "63066", "9544", "52192", "56451", "14747", "31262", "9194", "62059", "52677", "45187", "29477", "33739", "16821", "34803", "57894", "17342", "72725", "64907", "70346", "30980", "29118", "61800", "2174", "58310", "38568", "33177", "36033", "30140", "55426", "46534", "51044", "30318", "42686", "5064", "24313", "45882", "34790", "27588", "11535", "70141", "66521", "26100", "35578", "12672", "61745", "21677", "2087", "29372", "60094", "21957", "26363", "24094", "64406", "20616", "26124", "37795", "39092", "62083", "49065", "66796", "52078", "1175", "46124", "1063", "37150", "10632", "43087", "17086", "3069", "43883", "13686", "48434", "47412", "69343", "42440", "1118", "63186", "1294", "40139", "31787", "54958", "57131", "68802", "60537", "61418", "24639", "2194", "31559", "9993", "43800", "29887", "49984", "27487", "67558", "9528", "59441", "12693", "16458", "12783", "56092", "36048", "41585", "72689", "6041", "34439", "28928", "10178", "8392", "72650", "33579", "16397", "68480", "56540", "47921", "15135", "72819", "69706", "6715", "8493", "38636", "71775", "53899", "34786", "35514", "8396", "46633", "25626", "44712", "27410", "41871", "7183", "62352", "59935", "34937", "70176", "60366", "69397", "66194", "46694", "968", "14302", "10175", "28417", "63582", "51732", "28677", "38064", "8236", "34005", "26240", "30836", "48092", "64614", "34297", "11180", "37021", "6192", "70201", "2999", "16639", "52991", "9423", "14050", "54443", "8836", "443", "10669", "7456", "28967", "69946", "68217", "48466", "10799", "46096", "6387", "19977", "58642", "65428", "68277", "54510", "52146", "21984", "71213", "34416", "26980", "61414", "70780", "22062", "50017", "17923", "63942", "34989", "39729", "22336", "51040", "4745", "16053", "67232", "35893", "49920", "71231", "15274", "30688", "69316", "55193", "24302", "52297", "49877", "57383", "42846", "19916", "11690", "28884", "40651", "17919", "21647", "25979", "10000", "28293", "28078", "6900", "9653", "14987", "27672", "54931", "42532", "53069", "35506", "23850", "3656", "1155", "2366", "13200", "42994", "62055", "54874", "16417", "25960", "57884", "64623", "43282", "60977", "62826", "49597", "56510", "72922", "7346", "45665", "3077", "51595", "67177", "4776", "52902", "69350", "17172", "37836", "45228", "15147", "19545", "10893", "63581", "66975", "18221", "30169", "21368", "66275", "23102", "31081", "4874", "4856", "19126", "62629", "60153", "45280", "69430", "35806", "55172", "15782", "19171", "22317", "10238", "25238", "30416", "64650", "41342", "20409", "59165", "72075", "47164", "41380", "66750", "3030", "37096", "1867", "64997", "59388", "29822", "12739", "35417", "6132", "72958", "10421", "19242", "24280", "16198", "57164", "31922", "35573", "28740", "66644", "49733", "14104", "36115", "70934", "36283", "44666", "49894", "50578", "55919", "9232", "49794", "1631", "32930", "4417", "66592", "12474", "2124", "27744", "24555", "69008", "2061", "72306", "67260", "6171", "52140", "50144", "49617", "37844", "57899", "69207", "62726", "32015", "21417", "45085", "40278", "18821", "9148", "22623", "26871", "4061", "40895", "69972", "36309", "4756", "68038", "38445", "27721", "66868", "10108", "39234", "327", "39109", "68807", "34676", "52255", "72967", "45648", "24398", "21843", "49256", "34985", "29995", "42769", "29724", "57258", "38540", "40326", "66620", "48374", "47187", "33509", "36504", "11021", "37415", "72382", "59391", "28774", "30055", "3936", "6462", "18735", "46377", "60600", "24804", "34007", "25856", "26328", "53220", "69781", "699", "4310", "21257", "10309", "10916", "45745", "52473", "23401", "39698", "59535", "37884", "22402", "21873", "50629", "66880", "70605", "36987", "43077", "12985", "645", "36150", "63730", "15138", "58362", "38069", "13937", "48748", "73256", "66200", "68780", "29926", "1839", "2495", "39344", "67442", "62915", "25495", "57278", "65018", "38000", "61761", "39501", "64639", "40229", "14579", "24320", "39", "8139", "21680", "63093", "49162", "38421", "31894", "14711", "29445", "4968", "61466", "43128", "15991", "8239", "35795", "2554", "10472", "62124", "59224", "37005", "43251", "59069", "12996", "37672", "31092", "51596", "39552", "29441", "3594", "2050", "53472", "13854", "18975", "50735", "49820", "11978", "46110", "27364", "22008", "2358", "44075", "32060", "11615", "66037", "41597", "31906", "32972", "64431", "37623", "23642", "25371", "16780", "57618", "18979", "69802", "19683", "66555", "70001", "36332", "35074", "2462", "52485", "2655", "61783", "57816", "56312", "40845", "47616", "40844", "13780", "5239", "44421", "68751", "23500", "14033", "2107", "47455", "43679", "33617", "61829", "31152", "61592", "8785", "21752", "2924", "54068", "37014", "60391", "62897", "34162", "10163", "67795", "8714", "15786", "370", "33286", "66166", "41135", "22978", "11233", "15626", "46882", "56813", "27099", "8400", "36751", "53820", "20169", "6590", "6717", "47078", "59337", "35009", "41512", "45731", "4974", "9040", "12651", "12484", "29055", "36955", "32057", "3537", "16839", "68942", "36755", "60344", "71502", "34030", "21002", "69296", "54696", "11516", "40680", "31677", "51881", "50166", "38179", "55405", "63124", "5859", "68363", "20750", "34677", "918", "65179", "29138", "31185", "38020", "33476", "45819", "23775", "14260", "43386", "44783", "58639", "19059", "36642", "57764", "57518", "7407", "24865", "7587", "31018", "32843", "9165", "9872", "52891", "62163", "18117", "58485", "51506", "62548", "51224", "68821", "1977", "20326", "70708", "26318", "13823", "70470", "33194", "35380", "19402", "34540", "10155", "39759", "59220", "71437", "566", "32709", "22923", "1072", "31203", "29490", "9641", "58480", "40141", "23397", "8467", "20660", "45489", "70688", "22463", "24006", "63359", "47805", "57878", "21870", "50622", "59162", "25233", "25012", "16629", "3578", "26443", "40524", "51796", "65131", "69333", "60376", "3489", "51056", "48899", "33705", "33790", "62946", "25502", "11705", "73157", "42326", "4394", "62806", "42954", "55531", "40584", "12282", "16233", "60398", "70105", "24933", "52564", "25165", "11901", "43132", "35208", "37076", "1692", "1784", "56078", "55898", "18531", "47479", "34567", "62203", "59331", "18490", "60734", "61300", "6869", "25517", "41976", "12286", "11588", "688", "3646", "67334", "56169", "8763", "26628", "73230", "23714", "18199", "47419", "70437", "42782", "57008", "48430", "1443", "37661", "68375", "57651", "16182", "44759", "36945", "9547", "37389", "27291", "70971", "5770", "24906", "45053", "40106", "37994", "38314", "67374", "45477", "25731", "45474", "68785", "54688", "32062", "32143", "54176", "49324", "67071", "64711", "72023", "53517", "53208", "53961", "62274", "38556", "8119", "61855", "9517", "3143", "69366", "16315", "57749", "43181", "49673", "64756", "59144", "41676", "13314", "35781", "37316", "26352", "69036", "55947", "17160", "794", "4809", "56154", "27737", "30338", "68762", "12601", "16875", "34951", "20645", "3503", "53892", "48215", "68735", "59786", "36546", "44093", "590", "40506", "26998", "18454", "11970", "62975", "2362", "70582", "21460", "45736", "30616", "71490", "57791", "11494", "25200", "71393", "37999", "66699", "59750", "47583", "32426", "38546", "62622", "19513", "67751", "21616", "30457", "47463", "33759", "15994", "35728", "15773", "52435", "71083", "14696", "19385", "15893", "48449", "49157", "37951", "61979", "33020", "43271", "19187", "43940", "18230", "70959", "14384", "21430", "9536", "16548", "65302", "36843", "15325", "48958", "29709", "13796", "42760", "67577", "11889", "38071", "20199", "61994", "64122", "36220", "68747", "67192", "1988", "69288", "38718", "45046", "11466", "56426", "9150", "20588", "31301", "1164", "25877", "52707", "25448", "13964", "56132", "38083", "63392", "2597", "70349", "10481", "49079", "65504", "44732", "25075", "1296", "70393", "50580", "39499", "6802", "9249", "68180", "54693", "25227", "25174", "16133", "6151", "56949", "67264", "17856", "52831", "49100", "71714", "46642", "65890", "12896", "61502", "22955", "60824", "47651", "51589", "50462", "49197", "51262", "18840", "63955", "34194", "39082", "45047", "9097", "12628", "36567", "69919", "21293", "69265", "16880", "29231", "52787", "40265", "38679", "2713", "46190", "32347", "24538", "46342", "18099", "64740", "29002", "47420", "59994", "64071", "69822", "7234", "7418", "20749", "65069", "27429", "24812", "30747", "67762", "57774", "26121", "4336", "30656", "63166", "19655", "71338", "14725", "71720", "48333", "39611", "68187", "51906", "13751", "47328", "37774", "27801", "30633", "64131", "11448", "65037", "61307", "10489", "13260", "1984", "24166", "64654", "47038", "61338", "61886", "36720", "21942", "28716", "8182", "3290", "48422", "61329", "15183", "410", "66067", "70908", "57443", "49759", "24923", "12522", "71068", "65559", "47480", "53173", "45791", "66738", "52389", "35056", "55361", "2440", "37400", "52704", "49774", "21520", "37746", "59266", "8305", "36675", "50133", "20190", "62028", "1139", "30191", "45600", "68095", "63276", "65881", "63274", "56841", "50919", "38459", "13310", "40187", "58166", "29149", "50957", "3730", "4611", "11241", "38976", "66957", "20456", "33960", "30838", "3116", "11238", "60579", "10383", "26500", "17411", "50687", "55101", "2870", "63371", "13302", "26599", "64833", "53626", "68613", "68745", "69187", "54360", "19520", "51002", "13472", "3336", "21128", "17263", "61450", "29190", "62771", "67079", "25184", "18984", "43856", "69834", "49341", "4679", "1436", "20938", "62880", "33039", "66611", "59592", "51031", "55566", "22362", "52864", "16447", "72474", "34933", "10977", "9691", "6045", "61373", "59041", "43687", "41859", "57984", "38414", "29018", "35595", "12229", "57167", "12960", "3522", "34049", "12883", "1934", "7118", "66867", "45733", "9004", "32120", "23229", "54540", "25074", "4212", "48580", "8743", "65650", "43158", "9584", "6321", "2904", "37550", "19465", "2782", "55888", "17976", "12870", "38609", "49082", "29223", "45944", "44508", "452", "67247", "1419", "34837", "33583", "39295", "58230", "35192", "69858", "23782", "63347", "51779", "52754", "9102", "4740", "66238", "51935", "46733", "34056", "54297", "10836", "29098", "66573", "45682", "13187", "22398", "66032", "3113", "62759", "70361", "12704", "7672", "50115", "41076", "70806", "57004", "41715", "9161", "23421", "59498", "49248", "14905", "39679", "4416", "19044", "19592", "30472", "71617", "62003", "49882", "15374", "42423", "68464", "20338", "26267", "71588", "49933", "5000", "29449", "53211", "29603", "44611", "21172", "43436", "67651", "67290", "72305", "20846", "53724", "17584", "28427", "25509", "2996", "17370", "51052", "64137", "21955", "67785", "39046", "11283", "4532", "11261", "28938", "40605", "45", "48425", "26552", "63268", "21830", "39465", "37122", "62913", "67362", "40298", "42004", "52006", "72907", "58579", "26987", "42481", "21285", "49149", "45776", "41220", "33015", "44450", "35266", "70472", "27280", "57034", "58954", "49762", "40040", "45112", "28671", "45768", "60424", "16246", "40478", "44911", "13071", "44178", "58419", "66088", "59511", "28964", "63829", "57091", "70991", "42886", "31384", "61041", "70399", "2006", "1390", "28982", "31748", "14240", "70160", "34691", "59018", "16999", "72654", "10904", "61328", "212", "64305", "28544", "35705", "4995", "27796", "30840", "3408", "1076", "4375", "40835", "43041", "44963", "67379", "9704", "40854", "31343", "35325", "34376", "9699", "32401", "60933", "57373", "32698", "24111", "22018", "35190", "27905", "53485", "32287", "39997", "12898", "25778", "51405", "24249", "35484", "20306", "36449", "61852", "39198", "38920", "27889", "59304", "30766", "9265", "58569", "16213", "29011", "28570", "69201", "23311", "63966", "35536", "66939", "15390", "36724", "24530", "2863", "12092", "45805", "59083", "44174", "55738", "27874", "48139", "4322", "59814", "73074", "20807", "66278", "43319", "42640", "71387", "37056", "72896", "17740", "55840", "58382", "53779", "65191", "41720", "35887", "70694", "35805", "61429", "41706", "36780", "52298", "18003", "38535", "2395", "70150", "65576", "22192", "50136", "65350", "20835", "4752", "26921", "27034", "11724", "34045", "34976", "71008", "66744", "10438", "45167", "11367", "45618", "19915", "15361", "58924", "21915", "11578", "45951", "19961", "44452", "66622", "26624", "31697", "61547", "32448", "36604", "54635", "32544", "28220", "9253", "72748", "13626", "43809", "40042", "73131", "59305", "15087", "14679", "28977", "20439", "63799", "26246", "1052", "43933", "761", "11433", "46242", "30154", "30162", "34135", "56964", "69404", "40975", "8301", "38079", "33456", "57442", "68487", "51226", "34735", "70843", "46797", "36503", "40622", "73237", "42661", "37274", "24345", "6275", "39509", "63740", "70185", "5287", "26142", "13660", "63806", "57197", "5901", "59287", "40696", "45811", "57500", "24481", "10379", "60559", "64949", "619", "29455", "5693", "5949", "9886", "47749", "12970", "70170", "27322", "41286", "34926", "62768", "37190", "18511", "25854", "39454", "13245", "10329", "22460", "37812", "55145", "52920", "32788", "58960", "62232", "18808", "60118", "42519", "54648", "71300", "33718", "55200", "54114", "9362", "7094", "49644", "46968", "3798", "42830", "72685", "19295", "31821", "35292", "44355", "52422", "19032", "63474", "50262", "31583", "16708", "70395", "57427", "42833", "30371", "60921", "66819", "69411", "26706", "49382", "25561", "70482", "48488", "65642", "33713", "32721", "27271", "66115", "40969", "42429", "55072", "37048", "3015", "9381", "4261", "42825", "36470", "47526", "17446", "45201", "51418", "67839", "16941", "71431", "37579", "15740", "56512", "27404", "1375", "27306", "67977", "63039", "32455", "29296", "39930", "26864", "51728", "33058", "7545", "17823", "23843", "3001", "16825", "29151", "15744", "54208", "51217", "52381", "12580", "68050", "68732", "39387", "46442", "44411", "70168", "23206", "34391", "61166", "64291", "30317", "16227", "28206", "43213", "7358", "24612", "30533", "18055", "39910", "23828", "53085", "36872", "25759", "28336", "59960", "26916", "44318", "3099", "3035", "60086", "4909", "47981", "17862", "65697", "5666", "4868", "56251", "671", "56800", "61018", "604", "25137", "47728", "22847", "42373", "31732", "47418", "17541", "24956", "19789", "45368", "54778", "35704", "13564", "47815", "55064", "69633", "24785", "31561", "16190", "652", "69567", "52226", "58964", "52986", "30213", "45723", "53229", "23250", "5400", "58908", "53878", "37380", "49844", "72573", "43257", "65104", "4127", "50414", "11304", "22838", "13215", "27984", "8902", "38422", "14867", "58149", "27730", "22594", "64284", "68112", "50335", "58556", "22832", "56890", "62765", "46042", "28528", "60295", "31713", "27931", "8736", "63890", "36209", "17045", "27134", "31341", "68956", "7803", "29409", "64990", "26107", "2365", "257", "61053", "14555", "41965", "49698", "7265", "45528", "1248", "848", "4709", "49622", "67103", "14099", "64229", "28058", "39490", "40268", "52028", "44660", "70824", "17499", "35620", "32486", "3382", "25852", "46316", "59566", "71615", "20597", "40943", "1102", "29731", "50830", "38831", "17846", "59657", "38346", "63233", "72312", "63764", "57546", "15328", "42065", "45501", "68850", "52579", "18207", "50305", "39126", "13690", "14623", "43416", "69530", "70397", "69646", "1874", "70906", "7248", "58903", "50536", "47110", "20837", "10829", "12858", "54607", "60114", "37053", "23137", "61647", "35035", "47811", "5610", "27064", "58229", "24224", "54572", "4971", "18065", "24735", "4343", "2283", "42019", "20452", "63868", "47294", "67547", "48945", "6587", "20533", "11342", "49514", "17852", "31682", "44394", "23007", "58837", "50524", "6056", "47993", "28317", "54964", "13224", "40084", "3874", "66710", "22016", "39507", "29564", "50392", "23518", "59851", "67294", "19597", "54431", "72564", "26945", "15429", "11431", "48129", "47802", "25493", "57908", "68269", "19950", "24597", "45817", "26744", "50057", "69278", "26270", "61149", "48599", "23921", "51549", "34538", "58292", "29520", "61202", "17399", "52481", "7959", "49547", "38594", "35724", "52020", "47559", "12631", "44397", "62415", "29775", "60031", "66671", "43334", "4369", "17910", "58203", "44642", "54881", "39267", "61928", "45211", "15441", "5171", "13900", "34307", "37156", "70106", "27680", "7063", "18690", "64709", "52773", "29358", "34070", "60349", "54821", "42418", "14454", "15073", "11023", "23295", "57087", "61145", "33490", "3334", "265", "19484", "18999", "46135", "52500", "9815", "66845", "10792", "12029", "33819", "21731", "19864", "64290", "59057", "9742", "49260", "61074", "11336", "49772", "53176", "48503", "16071", "72197", "13889", "7312", "55116", "48823", "18627", "19399", "16686", "23522", "17895", "31785", "62661", "59830", "7434", "71028", "44308", "62855", "52197", "56880", "45235", "40015", "38781", "34360", "44628", "35086", "26722", "29560", "62807", "11275", "53295", "46281", "13678", "20412", "58793", "8053", "58460", "13185", "42270", "7422", "27112", "38077", "68615", "42970", "57723", "26412", "60902", "9899", "7071", "27833", "33625", "1779", "67255", "2013", "2767", "23322", "11078", "69702", "18591", "63948", "64273", "46423", "48091", "72805", "31856", "49113", "62169", "43130", "46470", "31793", "60593", "33686", "40612", "33494", "8885", "47382", "64603", "72329", "31327", "55035", "69819", "65941", "1617", "12983", "14336", "7263", "17946", "44185", "24667", "46056", "27933", "47972", "6262", "60381", "43597", "43365", "17054", "64885", "43094", "64057", "42109", "5822", "60577", "70535", "25551", "54463", "66437", "53737", "35772", "65473", "44184", "24965", "24520", "20088", "5991", "30749", "37264", "21500", "62725", "55739", "32976", "51624", "19517", "30964", "17922", "72397", "28277", "59502", "11174", "49843", "60414", "52790", "25596", "1163", "54325", "63204", "19202", "47959", "67880", "49721", "45654", "10343", "25943", "25710", "71967", "54340", "29156", "56472", "34147", "12805", "37882", "45687", "21926", "13874", "15601", "52205", "19839", "47894", "396", "54399", "15641", "30925", "58667", "12241", "2605", "70111", "28117", "25755", "1766", "22766", "54257", "51622", "61177", "30707", "71092", "72324", "11642", "39575", "64686", "30740", "18068", "38915", "63303", "19151", "72203", "2064", "6335", "54564", "41970", "22753", "20499", "66737", "13024", "1694", "30269", "40430", "5765", "23286", "34507", "49043", "70574", "13624", "7323", "40729", "63515", "157", "49663", "20581", "10599", "60621", "20410", "50194", "45499", "53627", "6647", "28282", "28875", "14151", "20313", "53913", "7620", "15575", "2872", "12386", "47565", "5003", "4921", "45175", "64443", "16045", "56036", "15930", "1793", "67965", "70584", "4340", "29526", "39174", "68967", "19125", "37386", "40271", "34949", "51433", "14517", "48369", "52321", "51582", "1559", "47009", "48081", "29414", "11218", "354", "44872", "58963", "4005", "45297", "45779", "54478", "58875", "24191", "6157", "46550", "65331", "34457", "34334", "54845", "46347", "58572", "18136", "59516", "59231", "19160", "68037", "24282", "31094", "53751", "30895", "23231", "54608", "33518", "56339", "45679", "7827", "52353", "44807", "40998", "24589", "21480", "44519", "43264", "57410", "2620", "4291", "57469", "16537", "37630", "53327", "22523", "42616", "17953", "42908", "11861", "38417", "57123", "32398", "21356", "10211", "54317", "782", "44550", "61957", "61265", "41169", "11576", "32402", "52492", "57611", "63883", "58606", "25905", "18095", "52897", "40066", "3839", "21847", "56346", "14258", "32355", "1149", "39388", "10138", "13941", "44804", "12081", "46212", "5877", "26098", "69619", "60191", "9652", "7856", "45785", "49636", "24079", "11269", "12181", "33040", "71806", "24566", "12674", "60941", "51300", "13860", "61212", "66254", "52238", "11802", "50344", "62972", "8789", "33004", "25350", "6147", "3022", "48424", "4560", "37288", "10160", "5881", "56095", "24012", "65661", "37770", "3625", "14858", "65732", "29053", "7475", "21057", "59920", "468", "62987", "46484", "35969", "27419", "17280", "39328", "46477", "65793", "12348", "20036", "45967", "65808", "11941", "35145", "16037", "44805", "59672", "46466", "44868", "18930", "44403", "24009", "8701", "1333", "1505", "45164", "17264", "52437", "56777", "67617", "53762", "63774", "5907", "70203", "6602", "16407", "2651", "23623", "23566", "21191", "64475", "18288", "51542", "17322", "43858", "4706", "10139", "5074", "38258", "55249", "16964", "58138", "56028", "19357", "35585", "1042", "37450", "51507", "33063", "43857", "54921", "14782", "57772", "14518", "96", "61516", "29263", "67735", "48295", "35349", "13640", "20963", "47473", "46570", "14038", "13315", "58459", "62599", "32178", "67989", "69738", "20399", "69197", "55434", "2550", "55654", "27365", "30261", "19062", "44955", "51428", "18604", "66119", "55835", "11400", "46840", "51230", "6829", "28949", "65326", "34332", "55187", "114", "30488", "50477", "67594", "61215", "36298", "298", "62460", "25416", "11107", "55994", "63020", "13582", "21200", "56353", "34466", "35720", "32379", "39867", "54261", "66689", "41973", "13116", "38750", "70617", "29158", "14490", "66897", "6095", "9789", "3341", "44063", "15635", "52505", "65573", "59720", "44424", "43665", "71281", "24473", "68890", "48158", "18009", "48235", "50565", "49370", "28405", "64587", "22394", "65294", "60830", "41216", "56793", "42232", "50703", "31261", "5411", "12508", "42627", "41249", "23705", "61714", "42447", "56716", "34428", "32131", "27398", "50848", "10573", "60839", "14953", "12625", "4296", "41468", "2804", "32551", "51171", "36086", "1717", "49853", "14977", "60836", "45472", "46944", "70509", "21397", "56107", "67315", "3183", "16685", "43554", "52531", "25299", "44674", "14159", "45742", "46533", "21785", "69121", "3479", "22271", "43620", "70281", "63647", "22342", "53112", "36428", "49779", "4450", "59751", "29605", "65482", "22786", "25457", "6377", "8418", "22893", "35739", "10469", "70702", "25861", "34570", "44604", "11574", "44427", "56253", "61884", "54860", "57910", "44720", "19710", "64878", "36076", "31825", "4053", "60459", "22203", "55858", "35813", "59215", "27873", "5270", "12741", "27620", "58425", "15655", "2685", "73043", "49269", "54591", "72821", "27548", "49602", "60124", "4482", "69132", "62893", "53802", "58615", "8100", "53608", "31643", "8806", "5232", "21232", "65448", "52652", "68698", "5867", "10526", "50781", "62256", "26811", "21278", "38903", "11362", "54168", "63456", "25258", "21436", "9090", "25057", "41879", "33525", "50464", "46584", "36949", "61426", "54868", "42630", "63956", "39277", "45418", "61700", "27899", "691", "39286", "67878", "26165", "14685", "866", "13962", "50223", "62613", "57219", "67841", "15622", "42426", "26253", "61406", "2480", "727", "42171", "43170", "18355", "18054", "11476", "3930", "4231", "49444", "884", "2312", "46371", "6571", "3359", "13217", "49556", "31461", "34198", "57995", "16436", "61241", "213", "33454", "38316", "71222", "30979", "39350", "66383", "6724", "3242", "21701", "50062", "17924", "957", "3913", "38439", "27097", "66834", "21862", "27020", "66563", "43449", "20404", "59329", "41680", "70335", "57615", "42917", "56585", "11036", "67114", "65419", "20822", "32179", "59715", "15020", "5057", "62764", "17566", "21871", "24134", "54996", "41001", "54460", "16276", "38217", "38656", "32480", "26348", "38203", "23799", "14431", "8073", "18684", "11182", "32311", "18908", "36505", "43151", "33290", "11791", "60396", "10131", "40801", "10355", "58020", "71996", "17085", "11435", "6012", "70732", "54130", "39352", "72216", "58087", "67301", "38486", "3554", "23448", "41985", "39803", "51942", "49525", "32822", "10911", "59040", "23568", "41766", "42922", "13759", "8096", "8250", "60237", "26634", "44744", "59248", "54960", "68743", "52761", "25081", "55616", "67049", "29974", "35628", "50775", "59352", "11703", "70253", "34710", "9166", "41409", "26613", "69506", "3991", "54902", "53855", "47202", "43366", "48323", "70355", "29160", "51177", "48929", "49536", "65393", "3626", "57040", "25633"]]} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn-test-split.txt b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn-test-split.txt new file mode 100644 index 0000000..cf6f03a --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn-test-split.txt @@ -0,0 +1 @@ +{"xvalid": ["int", ["1246", "11333", "18215", "16573", "11824", "8232", "18430", "2300", "12226", "25530", "5172", "22313", "14664", "3818", "2732", "21375", "19847", "1503", "14079", "12895", "2191", "25948", "7669", "1347", "14410", "14879", "22067", "923", "1453", "68", "13869", "1101", "15135", "6268", "4623", "10778", "547", "14171", "4328", "14238", "16544", "11730", "25833", "4095", "8825", "18508", "20865", "15259", "15729", "17011", "1109", "17390", "1369", "955", "18091", "24099", "23831", "16605", "8686", "22869", "12326", "8864", "6078", "21807", "13948", "13241", "8668", "13376", "23599", "20438", "17971", "1002", "16123", "16565", "9972", "6791", "3924", "5258", "15696", "17183", "21803", "24673", "25918", "7008", "22118", "9522", "25299", "18305", "20514", "1884", "8215", "5112", "21783", "10101", "6232", "11968", "10644", "15623", "16554", "5029", "4575", "19641", "22174", "9597", "21797", "7074", "16855", "20095", "10455", "4742", "22735", "9836", "8498", "16893", "4982", "19800", "21337", "17769", "17802", "2192", "18816", "21664", "24684", "23303", "19575", "12455", "13030", "2065", "21915", "652", "9252", "3870", "20072", "17511", "7928", "21004", "4217", "8773", "22983", "21379", "19147", "22578", "14580", "22181", "15446", "16503", "22596", "7072", "16731", "1582", "22280", "4832", "2423", "3015", "5341", "7513", "8482", "2844", "2331", "19665", "3030", "23439", "21923", "5003", "1926", "4645", "20129", "15871", "13751", "3438", "5804", "14666", "5518", "3397", "24861", "11600", "17835", "22167", "642", "25357", "9025", "13762", "14481", "4638", "11309", "24108", "10692", "24993", "10554", "17259", "8823", "13313", "21407", "1860", "19988", "21409", "2386", "20016", "18395", "5538", "11236", "4640", "20491", "4944", "25110", "7567", "3967", "3241", "24925", "22286", "16050", "10236", "19861", "20165", "16518", "19589", "1968", "8416", "1459", "10777", "25848", "25267", "2689", "20526", "4027", "24765", "22333", "22815", "7280", "6163", "12292", "10797", "5125", "22977", "18871", "15004", "23083", "577", "18503", "1", "13239", "1229", "4106", "19007", "10500", "19282", "17025", "11299", "17837", "246", "1160", "21698", "11563", "13763", "16763", "13639", "11900", "8797", "6764", "25553", "1488", "1776", "20168", "4393", "1346", "4013", "5997", "1813", "1490", "18354", "7130", "11743", "16608", "12574", "16709", "10449", "15382", "20401", "13272", "20181", "781", "3257", "4894", "13048", "7908", "14775", "25656", "1685", "13953", "16843", "7062", "2375", "7598", "11136", "7536", "19390", "10690", "19987", "21884", "15146", "2208", "18025", "21723", "25001", "25723", "6317", "5128", "24830", "6558", "9440", "17543", "1772", "25856", "22763", "12403", "21151", "4829", "22612", "2108", "9288", "9334", "20550", "4761", "1169", "19033", "5781", "684", "7197", "5001", "12126", "19782", "12566", "18873", "1177", "11202", "2723", "12493", "12014", "966", "12506", "8115", "21321", "20059", "15790", "20101", "20472", "12452", "14641", "8004", "17861", "25457", "25755", "14114", "7497", "1754", "4447", "17411", "7016", "24192", "14337", "3235", "21791", "8050", "12105", "7963", "21493", "10949", "7466", "17227", "12090", "16178", "18944", "21730", "22275", "18370", "5066", "25263", "4487", "9989", "8633", "8553", "16886", "19904", "1819", "19108", "10590", "22135", "19747", "23517", "16046", "22071", "20467", "205", "13531", "20057", "2180", "9127", "20539", "20633", "21340", "13281", "2443", "10640", "22865", "10599", "2549", "15191", "16941", "2430", "25991", "11468", "15953", "24942", "21879", "9527", "12809", "19343", "10525", "5114", "3284", "6372", "5813", "1861", "3409", "12067", "17603", "16460", "17110", "3206", "23112", "16830", "23577", "16414", "12015", "3126", "5778", "6482", "5330", "17480", "12454", "22683", "6499", "22451", "9118", "12618", "5797", "24693", "6818", "18726", "3571", "23276", "7323", "14746", "15493", "17749", "25668", "4655", "9968", "5691", "17204", "364", "12502", "14531", "10071", "9020", "19865", "19130", "16740", "18850", "24766", "10884", "18157", "20236", "2278", "15733", "18086", "8055", "622", "7571", "22438", "338", "9948", "24750", "25155", "25252", "15564", "14930", "24479", "8839", "11374", "602", "19571", "19527", "23665", "3938", "5293", "5555", "22939", "15388", "20404", "11080", "2548", "7595", "18094", "23862", "14682", "21007", "17984", "6568", "8474", "17499", "7375", "10108", "22552", "18359", "7568", "18933", "11464", "9264", "22422", "21498", "19129", "7477", "9289", "45", "10330", "5086", "15402", "12462", "17452", "24461", "20138", "23322", "8299", "14329", "24210", "21510", "10507", "19126", "14566", "2912", "23226", "17004", "10982", "11987", "18351", "3943", "3052", "3977", "16773", "8910", "3524", "20262", "22603", "20187", "9604", "4955", "24985", "21507", "25262", "24770", "10065", "13411", "25506", "4875", "15027", "13583", "19467", "18585", "11651", "22362", "5061", "2658", "170", "18771", "7505", "11179", "17181", "16030", "14584", "23029", "19526", "20971", "5847", "3815", "16245", "20381", "17199", "25015", "18012", "12890", "12758", "12667", "17897", "22270", "18277", "18608", "9061", "13622", "25986", "9644", "13533", "19792", "3111", "2674", "1059", "7732", "12628", "12129", "7944", "9489", "460", "13179", "13038", "13912", "12271", "15874", "5765", "2305", "25806", "8343", "8844", "23367", "14963", "18187", "21184", "14575", "1455", "12133", "7459", "15828", "20403", "9520", "24254", "9549", "13446", "7228", "7575", "20007", "16696", "10742", "4176", "24119", "25440", "537", "4625", "13005", "11379", "12838", "18241", "5762", "8385", "14333", "216", "16200", "7925", "3314", "12996", "17845", "22075", "5525", "17279", "4212", "15625", "8325", "23209", "15983", "5542", "7129", "16266", "16511", "14014", "5988", "25212", "19490", "21608", "19412", "14881", "19271", "1898", "25085", "25422", "12879", "14023", "20529", "18239", "14353", "18117", "14982", "19156", "22854", "16439", "17742", "5134", "16176", "11877", "21588", "5906", "9449", "8152", "10857", "296", "8441", "20541", "8801", "3509", "11676", "23194", "14951", "704", "2484", "15459", "23648", "22048", "14673", "24940", "610", "13344", "11960", "7594", "17813", "23939", "9391", "19097", "14800", "3635", "3823", "1650", "6882", "7101", "22522", "24825", "13032", "3277", "23105", "848", "15722", "11691", "24365", "12889", "11969", "16187", "18437", "12119", "8576", "4412", "8651", "20061", "5575", "23790", "6498", "14236", "20238", "25317", "15455", "22094", "25148", "22154", "774", "25897", "22766", "14907", "5790", "11954", "6799", "3918", "17605", "16498", "10487", "24599", "799", "20027", "6747", "14836", "8794", "5594", "12051", "822", "1608", "16916", "4677", "9175", "16471", "12125", "19367", "2144", "22199", "21200", "1639", "17715", "12510", "14893", "11251", "9704", "18732", "7113", "18630", "24633", "17894", "17419", "13644", "17476", "12353", "5374", "15546", "23445", "7044", "7393", "20628", "18257", "5834", "25450", "9054", "14503", "9180", "11550", "1371", "5333", "22066", "12595", "1451", "1581", "18158", "4763", "3735", "8097", "18514", "13424", "15964", "12591", "13159", "4138", "21069", "18940", "24334", "10106", "18122", "1747", "20773", "2916", "2156", "1969", "21828", "431", "5389", "10035", "21820", "21847", "3266", "22535", "8352", "12481", "7692", "4064", "1215", "17641", "1803", "24276", "11835", "9680", "21272", "10704", "18786", "20933", "12358", "24413", "24209", "17915", "6806", "26000", "12690", "17337", "11303", "172", "6977", "23889", "6206", "17159", "8636", "14298", "20036", "5960", "20989", "24241", "22594", "6913", "11796", "8246", "8102", "24754", "23582", "3216", "12501", "12173", "106", "5299", "989", "24817", "4282", "4287", "697", "10185", "14511", "23573", "7258", "15349", "15243", "21186", "19013", "1165", "18349", "26017", "15857", "1234", "11013", "8771", "1361", "6801", "9161", "10105", "3902", "23483", "8705", "17243", "21618", "23222", "8547", "5058", "4967", "10285", "11465", "24791", "6775", "14913", "6366", "241", "18473", "26003", "20442", "23434", "24010", "607", "16479", "605", "515", "11387", "21435", "12804", "2051", "13065", "12580", "15431", "16342", "4074", "10110", "13878", "25998", "14277", "7278", "850", "7086", "16669", "11718", "17266", "17298", "7721", "7647", "3659", "8453", "803", "14560", "11106", "2056", "16233", "21268", "24694", "18350", "24386", "383", "693", "7702", "240", "773", "7658", "9845", "511", "5354", "22487", "20420", "5833", "2933", "24284", "19410", "14771", "21261", "6837", "14384", "14544", "25693", "10844", "9516", "11250", "19361", "4179", "9024", "20873", "10947", "250", "7198", "5217", "8918", "25390", "15938", "8333", "8607", "10809", "14415", "21638", "1931", "21663", "20747", "15478", "14311", "5467", "3300", "12693", "18937", "23910", "20813", "22444", "15660", "6242", "396", "24086", "18108", "649", "20728", "6808", "25962", "13090", "21118", "11294", "9130", "6432", "15240", "23986", "12718", "24950", "21778", "2580", "25854", "7956", "16881", "1492", "10538", "3537", "23392", "22757", "12766", "8875", "10559", "13", "8879", "22994", "13076", "18374", "5097", "9187", "22321", "9209", "18162", "5010", "23755", "23675", "5639", "20156", "8631", "4743", "25129", "13068", "11815", "18562", "14901", "22662", "7205", "14369", "13785", "2905", "25882", "18050", "11939", "12286", "3790", "12509", "19956", "7427", "4131", "24990", "3301", "14792", "15281", "11484", "10006", "13936", "4508", "11338", "20906", "11660", "6449", "7175", "13790", "10024", "14573", "18754", "5701", "15409", "15031", "21817", "25244", "22478", "20012", "6776", "25853", "9041", "363", "5301", "25608", "11524", "6917", "6385", "2749", "21174", "6976", "1204", "7056", "25541", "22049", "22933", "20205", "18968", "21456", "10396", "8320", "17267", "9883", "23351", "23382", "2891", "7855", "8349", "7837", "11052", "5084", "14135", "18536", "7577", "6114", "21416", "21860", "14785", "25875", "10433", "4398", "1340", "17145", "24340", "17153", "23006", "13783", "17632", "17231", "8665", "23512", "10485", "1806", "9588", "7797", "24710", "12519", "1651", "14543", "6121", "3065", "2385", "12792", "28", "11416", "18211", "18135", "739", "8149", "4157", "17903", "14417", "14009", "9032", "18422", "1768", "4850", "11452", "10428", "21357", "11996", "10094", "4270", "6602", "6344", "11556", "4827", "17354", "14217", "18759", "16161", "13100", "6973", "8245", "20504", "4960", "4669", "24402", "5464", "13151", "8949", "11451", "17584", "11795", "21413", "1797", "12443", "3204", "2174", "20548", "15065", "3449", "9270", "21636", "1627", "8301", "10522", "7696", "4904", "2836", "10453", "15945", "21376", "12136", "4211", "23713", "3130", "13421", "9904", "11420", "5980", "11927", "2872", "22771", "9125", "12453", "20980", "20615", "18757", "15514", "13404", "22106", "14391", "9684", "1718", "16421", "7723", "18886", "21283", "21928", "14237", "2197", "19182", "23709", "13444", "15457", "18223", "9346", "907", "10457", "10361", "15171", "17280", "11178", "22264", "11698", "11339", "17471", "5600", "6409", "7359", "2688", "19092", "12421", "23272", "14474", "510", "13623", "8009", "23518", "22533", "10656", "23347", "24728", "11540", "7290", "11640", "14711", "23", "804", "16071", "20056", "22304", "23436", "1625", "15099", "10580", "17763", "16590", "10760", "18249", "8323", "5865", "8829", "16368", "6559", "4699", "5245", "7442", "14055", "16276", "17166", "4558", "1030", "11031", "10564", "2564", "20208", "9081", "25936", "9660", "5669", "7995", "25031", "19152", "6736", "17634", "10899", "17050", "13955", "21752", "14224", "17678", "15239", "22375", "10552", "24665", "882", "17737", "3889", "527", "11555", "10354", "10239", "10297", "1542", "8434", "18631", "8346", "13374", "5777", "12753", "4184", "16603", "16594", "15659", "976", "954", "6022", "19418", "6766", "3412", "1363", "24242", "25435", "18331", "6508", "1043", "11802", "24562", "16684", "21217", "16708", "1422", "195", "2477", "23664", "21173", "10839", "25322", "22343", "2289", "132", "17833", "1145", "16086", "2446", "2795", "2195", "10548", "19791", "17588", "15245", "4626", "17672", "25664", "1967", "16831", "8282", "1291", "998", "3382", "19173", "1961", "20478", "548", "3148", "5573", "13130", "1491", "17904", "20872", "19822", "10628", "23857", "17148", "15705", "4541", "3230", "8040", "6867", "7590", "15917", "17072", "12597", "23951", "11111", "23739", "18560", "24741", "1755", "24876", "10397", "22057", "6985", "19820", "2256", "20585", "22770", "12094", "21183", "18823", "3033", "10662", "4642", "3568", "12827", "17549", "8396", "25778", "24884", "13655", "11790", "3634", "22952", "13258", "24406", "3250", "16707", "18027", "9589", "13859", "5346", "1377", "11667", "13995", "18505", "3512", "18433", "25811", "13351", "9332", "15587", "11644", "8566", "8147", "7253", "2378", "8761", "8680", "12779", "3323", "12266", "19831", "13634", "8617", "23590", "4675", "16191", "6569", "5460", "4560", "21213", "13638", "12588", "10739", "2835", "7593", "12924", "18490", "25083", "20206", "3879", "16835", "11357", "24265", "1206", "15232", "8625", "9596", "633", "2743", "24713", "3705", "23464", "25290", "3387", "23360", "15406", "19631", "18285", "17240", "1950", "13963", "21304", "3937", "14801", "12517", "11523", "780", "13725", "10486", "22364", "24603", "11830", "18781", "19879", "16419", "22872", "12463", "19140", "25850", "6435", "9970", "7959", "11606", "12314", "23941", "13263", "5905", "16813", "10128", "4347", "13560", "5937", "5412", "20884", "23725", "20830", "18734", "9423", "24737", "2177", "19751", "15526", "19768", "6752", "6441", "3769", "17667", "8046", "17923", "19396", "2863", "15430", "4703", "3032", "12166", "9633", "19965", "323", "17253", "3806", "11766", "8101", "10001", "6059", "1698", "4887", "3131", "20177", "499", "18379", "5290", "16043", "17741", "24360", "3490", "21936", "18282", "9810", "9606", "17592", "6529", "4885", "378", "12367", "8616", "8425", "4326", "6145", "9511", "17807", "9821", "8277", "5661", "4139", "7489", "25974", "22317", "25306", "19389", "16088", "5049", "251", "19607", "5461", "21878", "3177", "2636", "7641", "24268", "14985", "9751", "21896", "3285", "19060", "1163", "21254", "16443", "4331", "1096", "8499", "4499", "11043", "10027", "1580", "9136", "20116", "21871", "22572", "10073", "5606", "12873", "716", "12712", "15657", "6730", "2607", "11047", "16210", "16203", "25733", "14866", "24450", "22676", "7853", "20225", "291", "129", "20862", "4467", "25130", "13188", "18314", "13025", "22328", "7758", "22859", "14199", "19534", "8049", "2229", "1730", "8648", "14995", "5666", "16309", "9239", "21203", "14706", "9140", "18797", "25699", "11980", "18006", "25844", "21796", "16561", "12461", "10895", "1355", "9800", "6378", "15207", "21252", "21169", "5897", "22689", "11599", "21582", "5408", "20895", "20755", "19123", "7395", "673", "5888", "24522", "25221", "25866", "7127", "20594", "17532", "15432", "12522", "19585", "6540", "2873", "17349", "8842", "25868", "2786", "24309", "20462", "2511", "7912", "8161", "3740", "3819", "4053", "12534", "8536", "1693", "25404", "15864", "16218", "19391", "1383", "13829", "17528", "21959", "13012", "9830", "24666", "21481", "17563", "7179", "25849", "17791", "6814", "23657", "3797", "12755", "25428", "20289", "429", "2665", "1227", "21744", "7720", "15973", "19894", "22969", "11196", "11461", "21377", "25396", "7793", "22549", "1077", "14115", "8843", "7364", "7463", "2320", "20196", "14582", "6782", "25902", "2054", "11209", "15049", "18994", "13450", "39", "22559", "24975", "12081", "3304", "5425", "18478", "7704", "9943", "20614", "8370", "4252", "2583", "6198", "23891", "4465", "5947", "16658", "14912", "11089", "20001", "16701", "11903", "20356", "8495", "9197", "15107", "8984", "7706", "4812", "13536", "17922", "14756", "10380", "11173", "1956", "24390", "24908", "14250", "16872", "12570", "19128", "14588", "11806", "7921", "7838", "17265", "2615", "12186", "20849", "9274", "23427", "11595", "25943", "5877", "11312", "3242", "9628", "13211", "20566", "15117", "21490", "12880", "1235", "24922", "11708", "14061", "52", "75", "3969", "24671", "18809", "16974", "13713", "23521", "12468", "19432", "21119", "23821", "20073", "10909", "20229", "1600", "20973", "399", "8243", "9269", "18183", "866", "12617", "15471", "23757", "23507", "24054", "14970", "13603", "13672", "24860", "10076", "6007", "13609", "7991", "13304", "23918", "17374", "11558", "25233", "4764", "1493", "20890", "25742", "6627", "1452", "6778", "20886", "2850", "1964", "20195", "19595", "2563", "24669", "17629", "11576", "1135", "19715", "13332", "1472", "6742", "12212", "4486", "6781", "4909", "15224", "14555", "24311", "19596", "4747", "23534", "1208", "24414", "23629", "1673", "785", "10476", "17264", "26025", "12424", "13396", "8000", "15594", "20814", "10141", "24153", "20618", "14549", "7729", "3660", "12464", "10893", "5266", "19750", "17282", "479", "21634", "7061", "775", "24189", "5135", "12078", "33", "14429", "12612", "15826", "23243", "21846", "15283", "3906", "2404", "19654", "15303", "3547", "10451", "462", "11744", "24745", "4111", "21536", "14270", "17694", "1384", "22308", "25790", "8056", "25573", "11409", "4923", "10712", "17157", "18155", "11235", "14244", "15439", "17700", "3381", "24240", "4030", "7007", "18698", "20408", "3006", "4415", "25764", "6963", "15014", "1791", "23401", "12558", "23927", "14491", "12199", "15547", "5954", "9477", "5413", "7218", "2525", "22185", "23880", "14763", "9178", "13371", "8629", "24584", "13067", "3337", "7428", "8656", "7102", "10722", "13173", "9789", "10164", "11393", "17875", "22318", "24231", "19583", "10907", "11134", "17071", "22297", "14260", "21875", "19652", "3003", "7678", "1659", "19922", "21598", "5611", "18638", "22462", "7312", "11525", "19197", "7977", "12639", "25672", "20142", "22150", "24077", "2953", "8728", "23824", "22157", "8014", "5126", "4932", "2454", "6177", "12796", "8837", "99", "4553", "6249", "5604", "8504", "3049", "8530", "25087", "16526", "24543", "19025", "24875", "17498", "3957", "6020", "7970", "4826", "14240", "8341", "12737", "10535", "9437", "19149", "23902", "20416", "25495", "1694", "25434", "4468", "6701", "5871", "22038", "2633", "20850", "11122", "6311", "21703", "5827", "12711", "15331", "16467", "13177", "21556", "17200", "9853", "20794", "15707", "7471", "1609", "12653", "11140", "10470", "7461", "8003", "10258", "16895", "24598", "924", "9860", "4554", "10790", "10490", "10304", "18484", "6309", "25711", "15977", "3736", "4821", "14161", "17746", "450", "19801", "6032", "18943", "11507", "23588", "5214", "17195", "16962", "467", "17541", "25169", "23319", "14207", "7479", "20335", "15307", "8544", "14307", "16715", "10541", "22941", "23231", "8561", "3867", "9284", "351", "20681", "17597", "17278", "11887", "8298", "2298", "17191", "309", "24343", "16168", "17090", "24459", "2117", "2259", "8833", "14660", "22961", "19442", "11212", "13892", "21670", "6563", "9157", "25983", "16914", "19541", "12152", "10480", "19111", "12499", "24397", "15260", "20480", "10069", "4440", "24530", "1283", "9753", "3737", "269", "12111", "8205", "15246", "12569", "4011", "12449", "10334", "8157", "8940", "18831", "10134", "6036", "13292", "14854", "3344", "11109", "3239", "1534", "17126", "2464", "18315", "22819", "2271", "3156", "9651", "6034", "15876", "18655", "13745", "6390", "6244", "15234", "7005", "1679", "10096", "14452", "23447", "24715", "10600", "5505", "21725", "379", "24023", "15016", "10068", "25947", "21017", "654", "20207", "15702", "25368", "4128", "8111", "9980", "18617", "17117", "273", "3709", "17322", "19522", "3563", "6647", "20368", "18796", "4016", "2131", "16315", "5305", "2114", "3747", "10275", "8731", "6492", "15494", "9503", "20162", "21810", "24870", "14594", "12480", "2718", "2847", "4935", "3711", "6376", "11259", "22145", "11548", "19856", "2033", "13223", "1373", "17202", "1761", "1242", "6785", "17648", "17744", "10365", "13248", "22606", "5535", "24452", "11840", "12268", "1598", "21414", "3882", "22084", "6245", "24110", "12029", "87", "7047", "22681", "23722", "8423", "20259", "25417", "17857", "19094", "13196", "11817", "21149", "3223", "23264", "25120", "13000", "10762", "19947", "3041", "8151", "1463", "23389", "2940", "23035", "10244", "6724", "1103", "11070", "7700", "2358", "21567", "15464", "8008", "14483", "9805", "5908", "20540", "13628", "9405", "13445", "17552", "19935", "14208", "2067", "11626", "23333", "2341", "18060", "4516", "25006", "20939", "15217", "16607", "9808", "13602", "21296", "10961", "6488", "6681", "25303", "5213", "14136", "14519", "21157", "14126", "6065", "7352", "19958", "9452", "11449", "7616", "20771", "18255", "2392", "23958", "2949", "21131", "2676", "7347", "20645", "4175", "11411", "17111", "4751", "22627", "19151", "9298", "742", "14375", "2864", "17196", "3375", "20760", "21659", "17203", "14708", "15840", "6800", "23156", "1521", "9002", "2491", "24517", "23553", "20080", "15870", "11668", "22537", "5654", "19321", "24015", "6521", "13818", "12688", "6085", "8615", "8034", "2005", "10278", "20635", "22656", "15709", "11232", "10837", "14764", "19289", "1872", "16936", "686", "22088", "5729", "13527", "23252", "6479", "13427", "25270", "7172", "14178", "15728", "15962", "25639", "12208", "6205", "23659", "14832", "21596", "18724", "15504", "6674", "9535", "3286", "24082", "21362", "9750", "5941", "13028", "22329", "18798", "4870", "24229", "25340", "20460", "17536", "8053", "7269", "3146", "5011", "18295", "13824", "13572", "9216", "2812", "15627", "2810", "13033", "3018", "9936", "21656", "1573", "4012", "8288", "25959", "19717", "7076", "8923", "4103", "22300", "5324", "10064", "13692", "5147", "15899", "20742", "14245", "21906", "2656", "4168", "25585", "3632", "22795", "36", "10488", "21046", "13303", "20908", "2460", "14106", "1970", "15425", "4299", "24281", "22495", "16163", "22418", "17135", "24227", "22567", "25466", "10171", "4018", "5523", "14316", "10618", "5714", "10952", "20321", "7200", "12221", "10573", "19422", "6463", "13921", "24252", "8198", "9889", "20744", "8969", "25636", "17886", "11424", "7320", "14092", "1122", "4784", "24934", "17213", "1320", "3316", "19053", "21938", "9240", "17096", "18765", "16762", "252", "19012", "9698", "22011", "15844", "6267", "9072", "15829", "25127", "12663", "3622", "16883", "9802", "9999", "3194", "796", "5044", "21569", "16866", "5361", "17436", "21159", "10129", "701", "23418", "8118", "1060", "20040", "19195", "2699", "6620", "24708", "20087", "19110", "20964", "2482", "21948", "8655", "15438", "13008", "23563", "5424", "20584", "16940", "18390", "2370", "4694", "19170", "5618", "3240", "7067", "5978", "9918", "23204", "239", "19489", "12716", "25728", "7740", "18642", "11088", "15400", "2349", "22734", "4975", "25531", "14608", "10368", "24", "21385", "5840", "19450", "24778", "14094", "18447", "18389", "22877", "23110", "18329", "5004", "20768", "987", "6415", "23217", "4198", "12676", "13335", "6288", "25706", "6961", "4348", "13321", "1508", "3118", "14152", "18081", "19810", "21278", "4948", "22644", "21840", "3368", "3096", "733", "6275", "15497", "4963", "24536", "25399", "8189", "3339", "945", "3202", "13830", "16466", "25200", "16064", "18316", "5393", "8872", "5108", "12799", "10312", "22825", "3115", "9124", "8582", "3761", "1928", "25762", "22974", "3205", "8383", "22110", "11072", "18893", "12038", "12430", "20845", "22378", "9258", "23522", "520", "5385", "23735", "20604", "16022", "2449", "24331", "6926", "9920", "14239", "17302", "16179", "6511", "15783", "8244", "18740", "20432", "21898", "22298", "19618", "196", "13918", "16295", "14050", "20819", "11131", "5588", "19010", "19903", "10550", "10289", "24852", "14187", "5159", "4260", "23055", "1159", "10374", "2641", "20441", "6592", "3523", "12815", "20711", "15173", "18403", "6750", "13939", "8145", "20852", "2134", "15296", "5776", "16497", "8647", "1795", "21544", "11168", "2655", "8158", "22079", "13640", "2000", "18044", "12823", "14690", "18000", "15669", "13397", "3884", "3347", "15923", "12528", "12184", "13298", "17205", "13598", "5040", "25246", "10635", "22036", "8813", "2509", "6783", "13125", "6919", "18662", "15047", "10696", "19620", "11164", "2150", "6995", "4098", "19227", "23060", "18761", "19849", "22816", "14624", "9896", "14938", "12977", "4491", "3477", "6711", "21088", "11822", "20425", "21179", "19143", "15426", "7324", "9230", "4965", "23277", "12352", "4368", "25249", "10342", "22719", "15968", "4543", "21025", "4312", "4578", "10195", "18550", "1196", "14829", "21227", "25114", "2412", "9385", "16124", "1074", "7962", "16521", "13209", "4813", "19853", "2524", "18007", "7149", "14541", "14751", "23678", "1249", "23513", "23961", "25313", "13381", "7186", "17630", "5138", "11944", "7247", "9734", "6412", "16789", "17754", "91", "10005", "23968", "15276", "5885", "17085", "24046", "1509", "12371", "2838", "8790", "15596", "973", "10540", "23440", "4228", "20835", "11864", "12109", "2558", "17444", "17722", "12700", "11247", "10804", "419", "21113", "2616", "25632", "7543", "5289", "16752", "14267", "24182", "1692", "1299", "16882", "25851", "2567", "24800", "25236", "2882", "9767", "4205", "23708", "12738", "16849", "13708", "218", "13040", "22904", "9303", "18951", "25765", "9160", "22968", "17708", "13331", "14008", "3147", "7350", "9895", "24617", "15744", "19794", "18284", "11716", "5951", "9677", "382", "7448", "25891", "7976", "2004", "9963", "16205", "10140", "4303", "3163", "14987", "23045", "17589", "24529", "6325", "3318", "18393", "4705", "3896", "26020", "2943", "16695", "5932", "18580", "12903", "25076", "16870", "1781", "5018", "1396", "1530", "3940", "11014", "4227", "21315", "25784", "2997", "6937", "14343", "17764", "22916", "15370", "8946", "9417", "22357", "14533", "17130", "20654", "1808", "14614", "13027", "8488", "4192", "655", "9840", "25885", "12272", "24841", "3341", "2172", "17305", "10413", "21975", "7910", "4416", "23712", "24758", "23095", "10536", "2273", "10276", "15058", "22978", "16040", "6661", "20867", "8327", "18774", "1543", "25893", "4071", "16616", "21448", "1310", "17844", "22521", "13497", "13771", "178", "17544", "15250", "17852", "25184", "13226", "13023", "19138", "9325", "18987", "12297", "11009", "17422", "20496", "3650", "12912", "25464", "2030", "17690", "17348", "10577", "18493", "24724", "2834", "11658", "9499", "2291", "24732", "25175", "19816", "11032", "1097", "22880", "18840", "22379", "19230", "24297", "16821", "25813", "23765", "5595", "18839", "2125", "3453", "23780", "16848", "13246", "17873", "19689", "21116", "8888", "16150", "11296", "10530", "4656", "19756", "9558", "13595", "483", "8690", "11063", "7750", "20345", "14576", "21609", "8878", "11454", "20564", "14233", "16665", "7680", "22160", "15556", "19065", "24184", "629", "4880", "24823", "21466", "19597", "17054", "8966", "12381", "7927", "4057", "23736", "8896", "25096", "24704", "13417", "14870", "11990", "20365", "21187", "23325", "5716", "6139", "318", "404", "5758", "14779", "9957", "16756", "1620", "11324", "16405", "7667", "22871", "25154", "7034", "19564", "11459", "22283", "12483", "3310", "9393", "7409", "23128", "18126", "7791", "5133", "14977", "14853", "5180", "25025", "14043", "7060", "20525", "625", "3662", "10565", "2708", "24460", "16141", "9690", "1233", "16611", "4188", "17035", "19154", "9611", "2206", "2981", "20200", "12249", "9685", "9227", "7079", "7272", "13825", "14986", "8856", "13257", "6240", "18989", "6726", "24677", "24256", "11391", "10526", "20726", "14364", "5197", "1443", "17961", "6261", "18602", "20445", "2352", "15156", "21623", "14234", "20634", "21331", "9333", "15486", "20127", "17192", "17649", "23387", "20543", "7924", "19283", "21677", "2843", "15201", "13970", "843", "8955", "17606", "18420", "25183", "17044", "17076", "25688", "17877", "19229", "1089", "24370", "21989", "14083", "1566", "18659", "12439", "14345", "8039", "20931", "14781", "9993", "7217", "819", "16783", "6936", "23030", "10290", "4698", "24586", "11343", "5565", "15747", "18279", "4752", "21721", "16028", "5005", "14508", "14529", "5320", "4795", "19221", "13678", "9467", "18625", "21332", "20346", "23994", "10642", "22395", "6425", "21211", "12769", "21011", "21669", "7276", "8707", "6023", "6962", "23419", "20944", "11584", "17812", "7815", "17084", "16897", "19402", "2239", "23046", "5944", "12529", "3616", "12500", "7036", "16648", "6228", "1516", "11216", "20901", "20093", "6278", "15433", "10613", "8577", "6349", "17394", "18288", "1811", "10075", "3175", "18303", "760", "21619", "5681", "5753", "6635", "9383", "23705", "1050", "15743", "18387", "16721", "13922", "22456", "10023", "11638", "19719", "638", "16207", "25367", "14022", "7897", "24775", "10104", "4065", "3421", "5899", "14578", "25373", "717", "19510", "25400", "11375", "18161", "197", "5996", "9540", "15128", "6695", "18290", "19671", "17647", "17855", "17378", "21339", "20287", "20114", "7105", "17343", "11573", "12898", "2057", "4817", "15529", "24868", "15421", "23608", "25442", "5234", "20505", "19942", "1197", "11719", "16661", "5727", "16413", "13845", "24918", "19256", "25205", "20146", "24859", "13429", "2470", "6483", "15222", "725", "18762", "12020", "25597", "4004", "6487", "25613", "2527", "12543", "12035", "21740", "19104", "10637", "6542", "24646", "17116", "21595", "8485", "5732", "9259", "14189", "7953", "20580", "24068", "23249", "12306", "14399", "16867", "1261", "14954", "2841", "3933", "6282", "22443", "17559", "253", "7231", "8927", "19037", "7902", "8060", "13288", "6154", "8729", "10429", "23614", "6745", "12544", "19605", "12023", "16182", "18646", "25027", "15819", "13657", "10866", "20826", "14201", "10016", "17138", "18552", "1646", "25041", "10818", "3415", "11110", "17475", "8882", "19950", "22299", "24813", "24333", "23265", "21140", "17250", "2103", "10026", "17106", "16918", "25238", "23955", "4913", "4404", "16121", "16586", "25309", "9944", "12158", "2582", "8525", "7188", "17772", "17268", "5422", "25625", "6423", "21138", "18728", "10194", "10593", "5152", "17775", "8375", "16162", "2409", "18275", "13256", "21868", "20713", "6735", "21352", "280", "3389", "986", "1331", "9244", "11631", "3483", "10898", "13525", "20128", "14500", "14327", "3850", "25447", "16285", "10592", "8750", "12040", "14613", "15218", "7009", "1732", "20164", "23716", "15749", "14467", "11871", "19043", "4731", "15318", "1425", "8376", "19228", "13978", "16578", "1538", "115", "19979", "9852", "9367", "15435", "25387", "20296", "22420", "21512", "14220", "16319", "12577", "20975", "1553", "25315", "4629", "5076", "24198", "14953", "20261", "21389", "20185", "24206", "21527", "21818", "14744", "6777", "11975", "24471", "16888", "8716", "20533", "24976", "24356", "12088", "21833", "2854", "13069", "1555", "2925", "4987", "6530", "18805", "1130", "13072", "24219", "4495", "14322", "20137", "4164", "20508", "4264", "17171", "10229", "10081", "11437", "25589", "21660", "20620", "14101", "12207", "9740", "19455", "20498", "20078", "9464", "24375", "20328", "12025", "5592", "14799", "9376", "2966", "11277", "11055", "6546", "22687", "5772", "6990", "14313", "19087", "23540", "4244", "22638", "22042", "6419", "4899", "25333", "7916", "20806", "5963", "21800", "2771", "21841", "19332", "6929", "13806", "19431", "10115", "23916", "12845", "9426", "20131", "2924", "24374", "11749", "16736", "25619", "13795", "6881", "8611", "14864", "24342", "19694", "18154", "12161", "1274", "23801", "2455", "11021", "16898", "19048", "23118", "23679", "15291", "18180", "9475", "5285", "12816", "18268", "4311", "20124", "12751", "25513", "24368", "12093", "4123", "14495", "13985", "9595", "22319", "18401", "21624", "3643", "25278", "22117", "20311", "2693", "3441", "22158", "16353", "4360", "20090", "21647", "3656", "4167", "8220", "21470", "11448", "10534", "6570", "1400", "9891", "20897", "24814", "7486", "2837", "18935", "11130", "2468", "20738", "6857", "8604", "6828", "12216", "20825", "18543", "6762", "11025", "19206", "14738", "14457", "11805", "13529", "15580", "1594", "11788", "9787", "23732", "24662", "12356", "7402", "9569", "13537", "14618", "14807", "5636", "22695", "23501", "15830", "11059", "7519", "3699", "5591", "15673", "3766", "8005", "25429", "8769", "20758", "8137", "17339", "6692", "16928", "24058", "18876", "2329", "5696", "10671", "23005", "8371", "24672", "5148", "8374", "18604", "25502", "18834", "22894", "21310", "8013", "744", "5242", "11724", "21899", "25550", "9493", "7125", "4261", "7030", "1178", "11878", "20268", "7948", "17978", "19946", "2589", "14835", "6609", "6265", "17712", "7401", "1412", "4080", "15638", "3511", "915", "7679", "2348", "2082", "10339", "5292", "16473", "11892", "5158", "23529", "12613", "12373", "16395", "4462", "22685", "9964", "23421", "2018", "13385", "3499", "256", "7882", "6186", "13660", "16069", "2611", "24101", "23928", "4041", "2309", "14458", "25907", "23358", "10216", "2403", "10351", "11828", "12794", "13894", "12172", "18129", "13591", "25827", "16039", "10858", "19650", "17312", "7209", "21043", "15563", "23463", "21621", "6490", "5369", "20889", "16615", "24611", "24123", "9647", "15266", "17167", "11905", "24896", "24963", "6220", "4867", "17925", "5095", "5492", "11598", "5169", "16072", "9962", "22471", "20378", "2568", "8660", "20400", "15489", "21822", "17801", "15583", "17291", "17109", "21724", "2210", "5116", "21562", "3145", "17347", "22619", "443", "14214", "7724", "653", "25938", "17428", "6815", "15738", "20106", "23330", "10316", "21006", "6258", "525", "17686", "13933", "18660", "14685", "6725", "15589", "19386", "4401", "8222", "23809", "10000", "18756", "18386", "9000", "6", "21", "12252", "5087", "13734", "24035", "23669", "1438", "15508", "20951", "24090", "748", "18336", "3772", "21707", "20947", "14717", "19970", "9247", "18832", "24627", "25734", "13175", "17561", "19767", "10666", "6025", "12575", "9603", "25202", "11941", "20386", "6132", "10097", "20909", "7703", "8733", "6191", "4371", "10874", "17154", "2162", "20836", "334", "119", "24264", "5141", "9730", "1832", "8908", "18347", "16095", "15949", "2706", "21176", "24138", "13931", "23201", "8002", "12661", "5485", "4170", "22397", "22692", "12605", "1420", "22153", "2217", "6010", "11619", "13148", "20428", "6604", "18232", "8893", "16614", "9211", "21819", "1675", "17773", "12138", "23377", "16077", "4169", "4990", "10426", "20257", "9181", "14957", "14173", "3729", "12765", "24230", "23528", "21576", "395", "3014", "7153", "19770", "21425", "22484", "23943", "9205", "20044", "25042", "2697", "14563", "13558", "17466", "12995", "8951", "15946", "2434", "8751", "1773", "23707", "11415", "12342", "6406", "3464", "14586", "21344", "11622", "11973", "15598", "13365", "18054", "19194", "8130", "4664", "21743", "9780", "7259", "16492", "6455", "12749", "19937", "4628", "17858", "8702", "10647", "199", "21106", "877", "8721", "9733", "20497", "7296", "9561", "15167", "92", "5284", "20431", "7063", "20273", "5303", "5356", "21683", "22651", "15877", "14865", "7374", "22793", "9154", "14287", "2637", "17745", "19394", "5889", "23520", "18923", "9283", "4649", "6757", "24698", "1025", "23922", "6382", "17121", "21144", "3312", "18153", "5314", "19078", "6151", "661", "24173", "21098", "5462", "14289", "14728", "15908", "8953", "2593", "15910", "5256", "12473", "19366", "23344", "6115", "25509", "15197", "22571", "4392", "8163", "3757", "14574", "9358", "791", "11483", "15343", "24620", "17932", "17179", "4149", "1068", "4028", "6637", "14455", "13721", "143", "20255", "8042", "16956", "8913", "16333", "5688", "18709", "20219", "5484", "15543", "25007", "6329", "21387", "3688", "7221", "11700", "23145", "11685", "18246", "5237", "24166", "15861", "3754", "14054", "3587", "23587", "8412", "6063", "12121", "20099", "13958", "25360", "22919", "17047", "25043", "1993", "6103", "25797", "16129", "9011", "15403", "6149", "7777", "22108", "4719", "16340", "2282", "2106", "7446", "10463", "14070", "19889", "23479", "16140", "6196", "1762", "12920", "14793", "19150", "14471", "6243", "21404", "6274", "21145", "16529", "24026", "24909", "21034", "23874", "6144", "7425", "1073", "18800", "6048", "1563", "10177", "16283", "6057", "17462", "22212", "25211", "10168", "20228", "10311", "16100", "10775", "23086", "13954", "4753", "18814", "8614", "1442", "12167", "16688", "10058", "15699", "641", "2293", "25886", "23661", "13951", "19059", "21405", "6072", "23612", "22141", "677", "23989", "20283", "2337", "7779", "9625", "2876", "7682", "18735", "10091", "5955", "6978", "20271", "24457", "14048", "2540", "19592", "12643", "12547", "6589", "19785", "13084", "15300", "24919", "19515", "16980", "12817", "6874", "18456", "12598", "18418", "19813", "19838", "5624", "2086", "20222", "1991", "24366", "8086", "24102", "4757", "2088", "18702", "18032", "1041", "9632", "17874", "17189", "401", "2625", "21966", "13437", "12045", "3440", "1336", "20060", "1128", "14562", "22909", "24640", "25272", "17356", "10294", "22021", "20810", "21454", "21333", "19165", "2561", "8737", "3909", "21100", "6260", "12474", "1406", "11184", "22671", "10376", "19624", "21155", "22241", "4573", "8810", "17736", "7666", "15356", "226", "6060", "15273", "2194", "13913", "25946", "13582", "23399", "14611", "12319", "11880", "21792", "17174", "15742", "927", "5919", "4846", "13319", "18189", "6820", "15337", "7120", "3826", "17560", "19452", "18648", "22350", "13510", "6176", "21632", "1888", "15590", "25364", "19670", "20390", "10367", "6496", "18885", "2021", "21346", "536", "8072", "18138", "4032", "20641", "16530", "650", "2076", "16060", "9215", "21125", "12047", "24440", "3002", "15024", "11287", "24067", "22346", "2181", "6601", "23589", "25634", "435", "25591", "15853", "16096", "24005", "2223", "25092", "15160", "14221", "25655", "5154", "12355", "2992", "17876", "13943", "22598", "13128", "7199", "1260", "22922", "21164", "16728", "22387", "6077", "20371", "11469", "22086", "1683", "8440", "8036", "15802", "288", "9857", "22728", "16391", "18038", "17272", "15368", "753", "12883", "25732", "15229", "4338", "20934", "20247", "22143", "16001", "15642", "2495", "18075", "23246", "25792", "19573", "23855", "19474", "8233", "16494", "235", "15726", "1316", "1957", "1951", "7052", "11511", "10798", "20534", "9557", "13910", "13813", "2889", "23251", "10439", "6608", "22005", "62", "21844", "20066", "3037", "4881", "21003", "12334", "16106", "21133", "15889", "22780", "11002", "8987", "478", "12533", "6953", "2252", "10709", "3655", "20770", "14558", "19677", "13543", "11661", "991", "11533", "25801", "15667", "24969", "14976", "4160", "2272", "2152", "1219", "12964", "3103", "3467", "10438", "9063", "23437", "25931", "23784", "12365", "8108", "23266", "10917", "20305", "7639", "6566", "13624", "14084", "25481", "20184", "20593", "17344", "25988", "18385", "9642", "15001", "7774", "9409", "10820", "2085", "6848", "17372", "899", "16278", "20197", "13050", "19511", "14204", "7801", "23225", "11725", "18852", "1771", "6323", "11117", "18170", "20013", "11215", "25512", "5167", "5757", "22085", "2410", "11124", "14944", "11445", "18324", "17525", "12250", "25162", "11438", "2500", "8764", "9035", "19416", "13377", "17185", "904", "8775", "12999", "7768", "699", "25046", "21438", "25051", "17919", "15029", "5078", "25090", "24047", "15831", "1636", "22738", "5120", "6646", "16970", "22180", "17031", "3881", "24736", "12621", "18919", "24700", "6227", "13637", "17125", "14305", "6359", "20477", "15396", "19350", "11521", "17900", "13577", "17806", "2705", "1309", "18365", "24085", "7420", "13653", "22429", "20418", "7606", "818", "11453", "11561", "9107", "1971", "17239", "20008", "6395", "23314", "1478", "153", "23820", "8368", "9112", "6870", "14934", "8044", "8998", "2866", "5117", "11239", "11934", "8399", "22900", "11636", "1147", "21455", "16811", "2235", "16172", "15415", "8506", "10822", "18783", "15804", "9292", "20133", "10087", "9495", "23134", "7473", "24111", "18079", "12259", "12183", "6334", "3939", "7626", "5343", "8116", "2827", "21449", "19961", "3880", "15028", "15719", "19365", "21226", "795", "21998", "14722", "9404", "16469", "16913", "10838", "12620", "7738", "11885", "12797", "23212", "14190", "5551", "23981", "939", "7145", "19676", "12039", "23375", "23904", "13942", "4381", "9260", "19437", "6930", "22927", "12084", "18927", "16817", "16778", "15138", "21551", "6970", "15042", "3633", "8292", "4953", "19891", "15030", "18248", "22112", "21893", "15693", "2890", "1155", "11597", "12814", "20295", "294", "24289", "25599", "16535", "3518", "21256", "287", "8624", "6640", "3188", "9179", "7548", "22730", "13600", "4720", "3657", "1652", "18649", "22892", "21739", "8493", "16015", "22652", "1616", "861", "9097", "17557", "21390", "10088", "4070", "8418", "13373", "5990", "17718", "13502", "2301", "22895", "820", "7772", "23523", "23804", "24349", "21552", "20412", "20544", "8840", "10482", "21222", "21627", "10308", "5602", "15261", "4484", "4350", "603", "20481", "2909", "13231", "22617", "4844", "7719", "9009", "1489", "19776", "11522", "23147", "14715", "13364", "6893", "11629", "24806", "22586", "13662", "4051", "19784", "1011", "3406", "26004", "10551", "8431", "16331", "2853", "9321", "23361", "9855", "13131", "9875", "3998", "9220", "19044", "18201", "17077", "25499", "11938", "2251", "5094", "18077", "6958", "2923", "9786", "11028", "20276", "13843", "10815", "20083", "14523", "19265", "4822", "15560", "12671", "1631", "4430", "17797", "18535", "2590", "21643", "24080", "347", "14679", "19276", "19351", "5697", "14654", "19237", "20658", "5801", "2965", "6517", "10400", "7365", "9697", "886", "19653", "2003", "7744", "11516", "17414", "4234", "16655", "12709", "16785", "6147", "19531", "17949", "16452", "12414", "3361", "25900", "23215", "23833", "24727", "13024", "12456", "9318", "1256", "3295", "21298", "17180", "20140", "11865", "17881", "3384", "17652", "8585", "23054", "6610", "10084", "17530", "10190", "7672", "3612", "14349", "9600", "13496", "12524", "21031", "2748", "16946", "21937", "25990", "20104", "5803", "18496", "23992", "13654", "6172", "4067", "19516", "433", "14076", "1395", "13051", "20265", "11376", "14116", "6547", "25578", "5071", "3172", "15384", "21942", "19185", "18844", "17725", "1715", "7903", "11784", "4286", "23281", "6080", "21648", "10266", "16254", "20417", "20756", "6017", "2052", "11243", "1912", "13610", "20966", "4549", "7652", "8262", "18411", "21016", "7677", "53", "3821", "9815", "25796", "20899", "8588", "14465", "6033", "24403", "12073", "7457", "13220", "7187", "19224", "22866", "15392", "25346", "18755", "5208", "18240", "23152", "18716", "1548", "15428", "10650", "4254", "23101", "14297", "21644", "22980", "15221", "17921", "8300", "16037", "3260", "16845", "20375", "24453", "7920", "402", "17757", "6387", "13097", "25344", "24196", "13251", "6528", "2686", "16786", "20571", "10654", "18100", "13124", "17657", "23701", "19745", "7971", "24143", "4501", "4024", "3714", "11408", "25815", "9090", "12150", "10625", "21694", "54", "2091", "8652", "14078", "2610", "8305", "6410", "22460", "23570", "7863", "23087", "17214", "3332", "5848", "5028", "2857", "5608", "15680", "23013", "23051", "13696", "9434", "9804", "21189", "1115", "22250", "23102", "25759", "9906", "11971", "9551", "11761", "23680", "15233", "19773", "5842", "16188", "18156", "453", "15740", "14166", "10158", "9123", "14320", "5770", "14213", "15970", "21491", "6093", "22845", "25160", "16706", "20358", "19744", "22492", "23240", "18843", "8583", "25462", "22477", "3270", "15095", "14073", "6454", "17212", "2813", "5137", "18371", "20894", "11930", "22374", "11863", "6319", "20436", "1333", "19867", "8289", "21509", "16388", "9723", "15807", "19740", "24815", "17527", "17226", "7898", "19125", "20900", "4607", "15338", "15676", "4362", "18471", "13984", "6883", "14358", "24177", "10160", "3794", "9842", "9565", "107", "16084", "10965", "5163", "6509", "7601", "16463", "6102", "19949", "18945", "14940", "7663", "474", "11966", "7363", "23111", "609", "17460", "4332", "18024", "12153", "4226", "24560", "18856", "5142", "10514", "14998", "24059", "805", "7884", "19825", "19369", "14550", "25157", "14897", "10198", "15887", "953", "23498", "10911", "7437", "22016", "12714", "20829", "10604", "23116", "17043", "5693", "5902", "20495", "19120", "25507", "25443", "10750", "6946", "24888", "13212", "10418", "25603", "5400", "24298", "23486", "3813", "20611", "17469", "26014", "449", "19705", "7931", "17868", "6636", "2760", "15100", "7210", "23604", "18328", "25916", "23032", "2380", "25957", "5035", "6181", "25751", "20522", "24428", "2596", "22640", "4174", "25624", "20476", "15578", "11246", "16574", "14056", "7508", "6213", "1174", "13010", "8691", "3878", "14988", "22073", "13495", "17220", "25494", "17033", "18541", "4289", "912", "13868", "854", "6591", "17217", "24549", "18837", "5670", "6533", "17006", "6787", "15062", "8378", "20720", "4224", "18583", "1267", "20423", "2350", "17697", "6537", "14749", "24169", "11315", "14734", "15760", "24688", "6302", "3222", "4636", "6914", "25452", "22777", "10139", "25253", "17404", "3763", "5490", "15282", "4351", "19960", "24795", "15701", "21247", "177", "5820", "2994", "19569", "2886", "19178", "3796", "19176", "22111", "14159", "15293", "22930", "5747", "9045", "6832", "2961", "8718", "20326", "9799", "24039", "3504", "24443", "1591", "23114", "7367", "19272", "21028", "21684", "7004", "7171", "21085", "19492", "13336", "1351", "16433", "22682", "19805", "19481", "10359", "6613", "10338", "3825", "21355", "18125", "13431", "13291", "20694", "6026", "2316", "19330", "12312", "13190", "3510", "12096", "21446", "19200", "5645", "20636", "9183", "25803", "8778", "21680", "11688", "19656", "10608", "25800", "2638", "8805", "18148", "11849", "19933", "20176", "23131", "910", "23490", "25012", "18884", "4094", "9610", "10603", "9028", "26006", "4422", "14639", "914", "17866", "11386", "1352", "11545", "9576", "11221", "5852", "17789", "3181", "2761", "6684", "22144", "17853", "11278", "24788", "4161", "21985", "7872", "9823", "16906", "19413", "9580", "22282", "5104", "15913", "2532", "16171", "14206", "3514", "22822", "13891", "22773", "23107", "21666", "24185", "10392", "19002", "11816", "7861", "16922", "20053", "3602", "17346", "11006", "8417", "12278", "17484", "9775", "23040", "23864", "6440", "11035", "16437", "6379", "8985", "24120", "20115", "7195", "18434", "16803", "7514", "9451", "15723", "15453", "1841", "9568", "16101", "15732", "19735", "13216", "4888", "23037", "14261", "1013", "6128", "22353", "12416", "19543", "21143", "9759", "21558", "5786", "19082", "7901", "22465", "9462", "995", "13235", "8216", "3553", "10723", "3887", "21097", "14147", "8204", "15888", "3139", "25048", "583", "3244", "15904", "17951", "8345", "16698", "18633", "2202", "24777", "656", "16960", "1280", "7886", "6106", "2436", "16973", "166", "3689", "21933", "20851", "15064", "21813", "13999", "21051", "24398", "17246", "20147", "6472", "18343", "8786", "5297", "13819", "21393", "15308", "12007", "22975", "14468", "18151", "21353", "18472", "23567", "20696", "22893", "25952", "8793", "19399", "18504", "15963", "10607", "18325", "16128", "4647", "11750", "22743", "902", "9438", "22172", "15774", "7180", "4908", "19114", "25736", "6216", "10208", "25666", "23853", "3363", "20761", "1597", "25351", "21391", "14181", "25890", "14", "15280", "579", "6826", "23254", "16989", "7878", "22995", "21920", "20591", "20994", "8976", "22997", "5390", "12820", "12884", "13289", "8543", "8400", "25726", "8124", "2372", "20210", "22511", "6011", "2187", "11890", "24346", "12085", "6723", "9356", "8904", "4901", "3550", "17916", "19796", "2644", "15689", "8532", "17827", "25182", "9531", "15764", "21342", "20729", "16875", "25777", "13414", "16073", "1519", "1546", "6624", "6746", "18317", "15328", "24644", "22539", "17524", "9927", "13490", "23346", "4156", "6046", "24263", "8841", "22643", "12497", "13136", "19633", "16348", "2111", "14542", "7834", "23579", "5707", "22921", "293", "22393", "913", "4108", "16059", "1264", "1668", "9917", "19799", "22151", "14107", "4814", "20387", "12106", "24618", "3611", "4411", "5271", "9351", "23861", "5355", "13204", "3369", "2284", "694", "22058", "7806", "2758", "6829", "17669", "2074", "595", "24999", "25106", "15289", "13722", "6596", "18744", "23379", "9869", "19270", "15991", "10862", "3760", "18830", "20630", "5516", "18778", "13174", "1798", "11926", "16564", "10348", "8693", "23200", "7467", "6790", "18477", "23169", "20924", "17595", "23505", "3309", "14637", "24431", "21374", "6082", "19005", "18367", "19449", "24359", "2083", "21633", "21062", "14154", "8944", "13349", "1015", "18256", "3567", "2862", "19509", "10853", "8905", "9760", "14450", "23793", "25107", "2533", "1082", "24914", "10113", "12641", "14124", "20325", "967", "5936", "12802", "3541", "19542", "10328", "7715", "24838", "23336", "4441", "24616", "14081", "25987", "3545", "24498", "24482", "9150", "18267", "8410", "25760", "11659", "9418", "22060", "11889", "5916", "22485", "12669", "3039", "369", "14650", "12060", "22633", "11143", "15076", "16485", "14822", "4864", "4463", "21525", "7585", "21786", "11094", "19706", "10888", "19248", "5430", "385", "16236", "247", "7655", "2535", "10226", "12341", "14828", "17518", "5904", "2552", "13839", "736", "21765", "8035", "14905", "23090", "18859", "13586", "13243", "25683", "5281", "11771", "908", "5399", "19673", "22396", "23435", "23753", "10459", "24055", "1586", "14026", "25776", "12521", "15851", "18172", "21297", "1460", "15798", "7609", "3710", "22083", "15075", "3846", "2537", "3502", "13441", "11970", "779", "20531", "15717", "16919", "1063", "21001", "18008", "10434", "17668", "11518", "1653", "22296", "13893", "8099", "3454", "7396", "17662", "25342", "18906", "21360", "15009", "23402", "20551", "9788", "18995", "2785", "10513", "1506", "12406", "17324", "20291", "23892", "24083", "477", "17355", "2822", "23004", "20801", "10935", "5414", "17913", "19478", "17479", "23819", "12980", "1102", "21280", "15038", "5048", "15758", "3730", "8560", "6600", "13171", "9366", "7285", "11914", "12389", "23273", "20679", "11412", "25284", "16450", "21790", "20869", "5184", "6081", "5869", "22600", "16055", "24625", "5067", "22500", "11314", "3097", "25141", "5746", "24518", "14315", "11138", "11352", "15897", "21722", "16496", "13311", "5441", "17516", "1474", "4474", "19713", "7549", "3685", "2808", "20272", "13612", "21474", "12835", "16510", "16447", "5748", "10188", "581", "1014", "18743", "6759", "121", "4444", "13145", "23258", "18729", "8402", "12311", "12080", "9986", "25286", "21037", "16592", "18904", "612", "22428", "4616", "13667", "9089", "17870", "18687", "8109", "14357", "22863", "9854", "1764", "10869", "6658", "20866", "2642", "19644", "23493", "6900", "14691", "15334", "286", "1533", "12066", "24725", "22937", "22408", "20552", "17273", "11116", "2669", "9068", "15538", "5744", "25232", "13584", "5352", "11271", "1273", "7032", "10828", "21384", "11813", "25563", "24362", "24856", "10072", "18506", "11731", "14396", "6328", "6840", "24441", "9267", "22855", "13804", "3726", "18970", "22755", "19045", "14788", "25588", "16438", "17388", "5928", "8353", "3661", "22029", "24364", "22690", "15815", "15712", "8715", "3215", "22585", "25297", "17284", "21129", "18745", "21605", "14218", "24836", "25772", "10560", "906", "9884", "16442", "5260", "10506", "528", "23769", "4521", "10926", "9515", "23823", "4769", "10293", "18112", "18738", "23155", "6466", "22966", "18811", "12202", "6045", "3124", "17421", "13262", "19789", "2764", "12070", "9314", "18113", "4294", "21070", "7327", "20493", "7883", "7310", "5080", "3168", "23878", "6168", "15255", "14165", "801", "929", "18882", "9790", "1036", "2163", "7042", "2799", "8608", "15934", "24267", "4301", "14361", "5047", "23010", "1275", "7184", "24513", "21823", "10833", "9537", "163", "23859", "21224", "22466", "17542", "10946", "4534", "9231", "23178", "3020", "21063", "15010", "8818", "2242", "2334", "17969", "6184", "10976", "5754", "24148", "22722", "8119", "16070", "21395", "2995", "14824", "24588", "3377", "16023", "14112", "4977", "9747", "13744", "14117", "19135", "3839", "3282", "11706", "15190", "13361", "15238", "7492", "8045", "15154", "9225", "9638", "23751", "18511", "6740", "7909", "23847", "15139", "24947", "17351", "15793", "16947", "18929", "16105", "2338", "16329", "22588", "18946", "22065", "10992", "8858", "4369", "25971", "3697", "22072", "67", "5107", "9567", "16742", "24475", "869", "21439", "10824", "23932", "23078", "11844", "7261", "11405", "1000", "5006", "5220", "23734", "802", "23365", "1372", "24657", "12198", "18209", "9285", "10302", "21518", "19660", "20313", "1417", "12411", "9864", "18677", "1891", "6188", "4193", "18203", "21997", "19375", "297", "7018", "23811", "18462", "18626", "8122", "10189", "16345", "23688", "8677", "11654", "6040", "18523", "8993", "11388", "23233", "4225", "8217", "5836", "21215", "1661", "2927", "1241", "12756", "20888", "9329", "2351", "23306", "6868", "23154", "9277", "13750", "20646", "11099", "21489", "23869", "10655", "19235", "1392", "5945", "21639", "22171", "8527", "2322", "25206", "18683", "24145", "19169", "12813", "21282", "24894", "10854", "25515", "13264", "9593", "18068", "16793", "12721", "14895", "4833", "4823", "4937", "7133", "15674", "5391", "1997", "21546", "23963", "11346", "15683", "1054", "19576", "8866", "22679", "15479", "14796", "12013", "22089", "6836", "17093", "3965", "20576", "8240", "4672", "21005", "18635", "6812", "10954", "24175", "15302", "7983", "5483", "24619", "12028", "10972", "9171", "838", "13240", "8063", "21110", "1061", "16689", "26001", "22782", "8502", "18216", "7638", "10699", "21530", "6355", "21706", "14920", "18737", "6375", "12761", "6625", "14896", "5013", "220", "1936", "1736", "2788", "4734", "16724", "11911", "5709", "10092", "17485", "4561", "20450", "24446", "20405", "12236", "3861", "9052", "17594", "17015", "10240", "4197", "16147", "6979", "6352", "17871", "2679", "24682", "10516", "1137", "14688", "8571", "10649", "12072", "21773", "11876", "12768", "5470", "7697", "9115", "8150", "2842", "21002", "12061", "25928", "16303", "741", "15091", "6885", "12665", "38", "416", "2407", "11537", "22225", "25123", "23814", "10496", "911", "20175", "25371", "8444", "1168", "20004", "18070", "23193", "17292", "6289", "18043", "5177", "25810", "6984", "23703", "22232", "17784", "11318", "11292", "23666", "24121", "10852", "12540", "6326", "3527", "13500", "4061", "23882", "10329", "19716", "1203", "9897", "18322", "15751", "1446", "18414", "16533", "25059", "11442", "4641", "5480", "11305", "23132", "5386", "23998", "20818", "23282", "551", "20015", "873", "21205", "7151", "18715", "17981", "20678", "23158", "8619", "10902", "22748", "932", "1643", "9093", "6536", "17490", "10945", "25144", "18229", "17994", "5832", "771", "1132", "18694", "7049", "3586", "11242", "24690", "6719", "6780", "7524", "22260", "20979", "2818", "13455", "21917", "21865", "4916", "20430", "17285", "9973", "11829", "9119", "9320", "20538", "17290", "400", "3802", "12518", "8696", "22574", "12224", "24499", "15484", "11559", "1138", "25691", "25704", "8541", "17141", "2553", "5257", "25992", "4585", "6543", "5090", "10327", "16221", "1330", "7400", "8570", "4998", "2070", "23849", "17082", "6219", "23620", "1525", "17867", "21829", "12459", "12702", "9887", "11274", "24988", "21236", "5255", "22002", "25748", "8250", "20563", "16002", "9476", "14024", "19611", "23366", "11176", "15312", "18629", "6813", "22013", "25055", "7313", "5907", "4397", "1728", "18474", "21482", "6167", "14992", "10333", "22589", "4615", "4130", "20638", "25716", "16011", "1852", "14389", "16193", "12589", "4596", "23630", "17198", "5468", "4105", "22271", "21770", "18185", "8480", "5181", "23575", "20847", "24474", "10401", "4778", "20466", "6116", "2986", "18983", "7867", "21160", "8816", "8272", "17133", "21480", "22509", "25775", "10310", "10807", "17781", "23192", "11741", "11392", "20399", "15352", "22044", "23670", "20946", "6895", "22765", "8968", "15304", "9502", "980", "2214", "15059", "17946", "9560", "24409", "19580", "6576", "18903", "6298", "14935", "5100", "1250", "4470", "7600", "24124", "8938", "7163", "19635", "1288", "17621", "25319", "13372", "19131", "15754", "21805", "6281", "13270", "12350", "14225", "11426", "1496", "14678", "22448", "4295", "7646", "9618", "16090", "9075", "23912", "7849", "12395", "18147", "3085", "3299", "22050", "18486", "21065", "25549", "23082", "6694", "10074", "24948", "1091", "3849", "13799", "24768", "4349", "6717", "2627", "11323", "3956", "776", "9237", "8059", "6534", "24130", "14128", "14398", "1449", "11114", "20079", "2042", "3109", "16223", "21745", "12877", "4711", "2011", "11978", "6804", "1005", "25524", "23298", "4446", "4972", "18176", "22320", "12423", "9888", "16767", "9939", "19761", "10708", "8330", "24168", "10251", "23813", "14472", "19913", "10848", "21210", "13597", "11011", "6842", "19602", "3245", "162", "25996", "20570", "7423", "20215", "23475", "11326", "11466", "25713", "15469", "444", "22459", "5882", "25887", "25300", "8487", "4984", "16606", "6583", "25766", "25173", "17083", "2974", "20822", "17818", "18270", "12863", "13974", "8321", "22464", "25518", "3346", "24225", "21373", "9322", "20949", "2232", "1742", "22754", "25156", "18722", "1744", "25921", "13453", "1996", "5935", "3256", "2806", "25825", "2503", "13472", "24878", "8279", "4879", "13782", "16375", "21094", "19714", "3636", "5926", "19193", "17474", "23544", "7974", "13285", "22284", "3298", "9394", "2710", "24705", "11809", "11747", "7695", "9309", "25779", "6290", "13034", "6312", "2071", "24696", "7517", "7391", "24819", "12048", "15198", "6088", "17221", "6236", "22349", "14042", "2888", "15898", "10440", "7760", "23689", "11791", "24100", "15510", "2508", "20453", "20731", "18766", "10355", "1810", "24962", "14967", "14146", "1887", "23652", "8556", "24021", "2956", "4469", "8231", "10916", "16755", "20018", "5106", "493", "19642", "24816", "16799", "11363", "20569", "19568", "523", "2526", "14339", "19790", "2130", "17599", "13457", "4653", "22023", "13736", "19476", "11160", "25883", "15795", "16080", "3579", "20778", "3810", "3496", "16192", "18776", "1550", "12140", "7387", "18391", "20170", "15491", "23285", "19622", "16127", "14758", "11592", "5557", "7572", "6545", "14304", "12678", "4884", "2581", "21345", "22707", "5800", "24746", "2072", "18035", "23021", "2020", "22211", "9670", "11846", "6944", "21925", "130", "9755", "7417", "23409", "15928", "13031", "24537", "19144", "21255", "6629", "17790", "19242", "24380", "4544", "14278", "1123", "18815", "23860", "6477", "18911", "21263", "10028", "22303", "24527", "21180", "5358", "24721", "1729", "10373", "4803", "21306", "5774", "4563", "4621", "6358", "18407", "13968", "10879", "21837", "5572", "9965", "9942", "8779", "3975", "4329", "3720", "23398", "10372", "2496", "21059", "5081", "14635", "18846", "22090", "19473", "20108", "23067", "5232", "14052", "4201", "22753", "174", "7249", "8106", "24691", "2648", "22576", "24384", "15634", "1125", "1949", "21450", "16599", "6448", "13140", "22536", "16617", "4599", "12457", "15926", "24851", "10162", "2238", "21044", "25977", "11783", "18862", "2903", "25363", "3955", "25374", "2554", "17223", "597", "9689", "10545", "18721", "5610", "1035", "21086", "1308", "10539", "3486", "24109", "24589", "7383", "23704", "22301", "6150", "7914", "18424", "19100", "23108", "18591", "14948", "10319", "4802", "8610", "23077", "20962", "11564", "2201", "210", "19506", "3561", "21962", "16910", "10582", "22463", "15284", "12740", "25602", "10959", "7444", "7416", "5189", "3228", "21600", "10130", "5647", "5350", "7617", "14735", "17100", "3325", "14138", "4296", "16361", "2221", "16146", "14292", "6925", "1766", "19655", "25198", "23470", "6866", "1583", "18868", "8857", "23412", "17136", "21901", "19299", "3792", "18327", "11873", "5958", "11132", "25637", "23656", "21924", "7139", "3578", "7182", "23350", "12942", "3488", "7735", "22883", "22827", "1958", "1896", "13315", "23452", "18442", "25235", "24250", "25168", "22505", "5278", "24626", "10685", "17635", "17098", "14255", "19828", "10915", "24949", "17080", "18564", "16386", "9049", "1628", "20927", "11883", "9190", "10460", "14167", "26002", "4933", "21541", "7287", "421", "13872", "21880", "12229", "16113", "24098", "19103", "7814", "24638", "18512", "12235", "4056", "25237", "11958", "25325", "2757", "17242", "22189", "11680", "23500", "9046", "14780", "25554", "14891", "13113", "18594", "11457", "16458", "11406", "10978", "16557", "22716", "4836", "3556", "24822", "16262", "17335", "20256", "6954", "9977", "25954", "10237", "19349", "25115", "10832", "4874", "24097", "7730", "24232", "21335", "12466", "22862", "20902", "24150", "6751", "15327", "9387", "12032", "7329", "21897", "14698", "1213", "14983", "23671", "20643", "17986", "11104", "16999", "21750", "2049", "23009", "13091", "9571", "9163", "16568", "6617", "22806", "13509", "7569", "5864", "16551", "14184", "13707", "147", "3106", "11610", "23049", "5174", "23038", "16628", "24078", "19061", "14917", "23729", "22875", "24114", "8022", "19566", "7112", "12019", "20277", "142", "24064", "18208", "665", "5321", "14016", "9933", "24421", "25592", "11037", "8191", "9490", "9461", "23817", "3908", "7741", "7054", "13011", "22609", "15033", "3458", "11981", "23020", "14724", "23451", "18959", "17020", "24733", "8777", "3807", "1731", "21599", "9352", "15316", "9639", "25013", "3647", "17206", "23457", "25241", "19555", "18458", "5634", "19726", "6835", "6203", "11265", "521", "24106", "10967", "24435", "21513", "3598", "179", "1632", "4985", "12283", "19348", "17917", "4947", "23466", "9384", "22198", "15690", "23618", "6841", "11722", "11078", "18861", "11123", "23807", "1657", "4689", "17905", "7332", "18029", "8919", "25717", "8508", "5042", "315", "7104", "15812", "13308", "23979", "10578", "18306", "14156", "23396", "10896", "1124", "5494", "9390", "10827", "3208", "3392", "4216", "761", "10431", "6120", "18791", "14737", "12302", "9981", "587", "22352", "13671", "997", "18047", "13036", "12842", "18206", "17219", "25526", "9203", "9065", "5316", "11421", "8689", "14169", "13063", "10111", "4089", "2809", "2577", "11837", "16461", "1759", "4178", "25673", "4306", "13971", "2241", "22498", "9534", "10211", "23726", "3717", "374", "6071", "21716", "18656", "14257", "20977", "17701", "1686", "18190", "11422", "3161", "10931", "19755", "333", "7612", "16996", "13187", "13268", "5054", "3274", "3155", "7998", "8755", "12396", "25468", "5127", "1529", "21842", "11434", "16824", "1081", "12760", "15610", "21410", "9801", "7645", "6901", "12376", "2016", "21650", "8364", "15595", "5212", "14861", "4506", "10908", "16878", "16988", "1084", "813", "12563", "18174", "2762", "17645", "4681", "4062", "25288", "7394", "831", "8589", "9725", "15161", "13553", "17366", "5190", "6271", "18984", "8770", "19382", "2852", "24709", "2502", "22242", "5835", "3007", "6793", "711", "15127", "1892", "16894", "15187", "15803", "19079", "1062", "2646", "10924", "21682", "8146", "23064", "7539", "15247", "5521", "16925", "15842", "23832", "2560", "7717", "9720", "687", "19480", "10056", "17750", "12397", "8395", "825", "10054", "18059", "19512", "3613", "13575", "8082", "14085", "9273", "5651", "9542", "9984", "15506", "21545", "13053", "15880", "13689", "23326", "2984", "25857", "8021", "22946", "555", "11477", "16151", "10799", "1149", "2304", "21430", "12969", "16977", "2824", "508", "24550", "14095", "14997", "11720", "4630", "23164", "21673", "13919", "22262", "22506", "17982", "16165", "21228", "6178", "9411", "25449", "9626", "24769", "7124", "20357", "7399", "4590", "24218", "2266", "20737", "16570", "18426", "1090", "3987", "13924", "19862", "7660", "2169", "12723", "3594", "23444", "17369", "13121", "9726", "7918", "12832", "4365", "10826", "6630", "6173", "2617", "10997", "25116", "11195", "11252", "5014", "13705", "22046", "2129", "19700", "13029", "3799", "3949", "249", "15772", "16373", "5056", "18643", "6685", "23776", "15105", "20482", "5267", "6773", "22677", "6669", "18175", "19723", "13467", "15866", "22634", "23356", "7584", "25544", "9497", "13695", "13198", "10126", "13208", "17408", "4115", "25406", "181", "24559", "3193", "23782", "25358", "15336", "11649", "561", "8038", "17924", "16787", "10555", "1279", "21657", "17656", "3355", "19334", "15202", "3104", "9975", "3022", "24062", "5812", "7267", "13037", "13700", "798", "25424", "9605", "20627", "18652", "19593", "19634", "5659", "26012", "930", "17627", "12772", "13688", "2089", "13767", "17625", "17611", "17303", "20297", "14017", "12989", "4513", "13166", "2619", "17957", "24234", "15868", "21940", "2660", "11191", "3895", "25061", "3054", "21038", "2255", "10170", "17522", "17839", "16138", "7305", "8290", "1388", "4044", "5443", "14831", "18087", "24034", "4849", "17056", "21789", "7950", "6807", "12893", "20786", "16562", "476", "20082", "8069", "23805", "13876", "2983", "25231", "9514", "22237", "10299", "11378", "22616", "3817", "3313", "23015", "7992", "23728", "8253", "5635", "4420", "5216", "25158", "12810", "8235", "14877", "8373", "13753", "14703", "22813", "22384", "637", "18590", "6182", "15299", "9619", "25104", "21053", "20981", "19681", "5616", "2609", "20306", "11414", "135", "1557", "9302", "19637", "6089", "21877", "17005", "1995", "16877", "20631", "15896", "18139", "2327", "3248", "15566", "7843", "7107", "20260", "13768", "3187", "19015", "14174", "8037", "21883", "10010", "10829", "21757", "3921", "16289", "13403", "4514", "9212", "22915", "2529", "21024", "25045", "22590", "7480", "11897", "15070", "11620", "24063", "19421", "16024", "25937", "17884", "6474", "8181", "12374", "18182", "22370", "23286", "6918", "25459", "13483", "22179", "25028", "15155", "19779", "19318", "15988", "3386", "11475", "7642", "9763", "18292", "21460", "18003", "17513", "681", "20362", "17127", "20812", "25433", "15737", "2725", "17809", "2113", "14696", "3666", "3816", "12610", "8424", "16703", "12975", "14569", "16305", "19985", "20084", "14681", "16322", "12594", "4203", "7597", "9131", "21336", "23353", "16066", "6400", "3184", "6421", "12962", "9166", "25339", "25807", "7698", "10176", "17711", "3434", "1461", "2022", "5938", "18858", "7906", "18972", "20740", "18037", "19127", "18941", "2050", "25314", "25167", "5837", "5895", "2330", "5782", "9144", "22808", "18775", "2551", "6932", "6389", "8516", "17168", "5360", "628", "16243", "11888", "21999", "8989", "11542", "18640", "21238", "10254", "893", "8602", "18607", "4189", "7339", "19209", "19309", "15248", "15540", "24675", "15264", "24301", "408", "12104", "12213", "16455", "11142", "13947", "1202", "24445", "19038", "24249", "16214", "12357", "10025", "4831", "23785", "23304", "24307", "25694", "17759", "2115", "16776", "1817", "5704", "9241", "3932", "14028", "516", "12659", "10154", "6680", "7737", "1672", "19383", "7191", "16861", "9226", "20173", "8889", "8339", "26024", "15377", "16975", "22006", "11623", "24524", "11617", "18121", "18588", "138", "20843", "20558", "17329", "23674", "2613", "8090", "19722", "14993", "9952", "13172", "7771", "3401", "7242", "10658", "1502", "11467", "16448", "7300", "23914", "13284", "18163", "9665", "7939", "16148", "22009", "18842", "23622", "10689", "13871", "21427", "3100", "25546", "12011", "4200", "7232", "6810", "10031", "15169", "266", "671", "10102", "22979", "17258", "14900", "25919", "13391", "17073", "9360", "25020", "7275", "25164", "21120", "20278", "7116", "20398", "2945", "10437", "8943", "8824", "1305", "16381", "10737", "14442", "10228", "2550", "7289", "12914", "12858", "1099", "12774", "11587", "25065", "10180", "1726", "12957", "10131", "23607", "8554", "1092", "864", "21082", "6012", "19883", "17786", "24310", "20323", "17118", "19320", "17094", "1972", "7733", "12399", "175", "10398", "2326", "2504", "18964", "11153", "21630", "17578", "15944", "22310", "17022", "6585", "11208", "10805", "25858", "16507", "21689", "594", "24573", "1881", "12141", "19080", "18515", "23559", "22371", "19040", "11811", "11260", "25746", "25768", "6792", "6330", "23933", "22206", "24332", "12853", "19869", "19507", "17067", "12705", "6148", "3422", "18448", "19843", "25840", "25068", "666", "14784", "8979", "20183", "25577", "13476", "3485", "20653", "6231", "11789", "25163", "15811", "18220", "13566", "15490", "8594", "14937", "8791", "9620", "16", "21641", "1527", "8974", "3074", "17162", "24314", "7876", "9070", "487", "10697", "11504", "24714", "806", "14383", "19465", "7284", "3801", "15056", "550", "10528", "19423", "9415", "14003", "11280", "11936", "20385", "12440", "7373", "13119", "17850", "18231", "13851", "6898", "21748", "3102", "16500", "4423", "441", "8465", "8007", "13552", "20844", "10305", "1855", "10561", "25581", "16109", "12971", "4524", "22163", "7233", "13448", "22309", "25702", "19499", "18015", "3048", "12545", "22778", "7058", "4547", "21581", "2659", "984", "18339", "22929", "8285", "2376", "20301", "22169", "17161", "168", "7168", "17384", "11404", "6397", "11693", "16440", "18790", "16198", "3179", "15474", "11249", "7328", "17122", "17523", "23159", "22240", "13330", "2158", "18276", "8079", "10145", "5175", "20796", "20815", "7544", "4265", "15131", "21914", "22501", "9170", "8524", "12425", "12571", "2599", "4324", "10843", "6001", "13701", "15214", "10340", "8248", "7997", "24036", "24790", "19155", "18693", "21775", "15954", "11652", "13167", "23774", "14392", "3150", "7360", "4785", "16265", "12606", "1154", "7714", "12496", "25186", "10570", "1979", "4276", "16612", "20903", "24091", "6308", "18597", "7132", "14969", "889", "4433", "22720", "14989", "25598", "5686", "9792", "3327", "11479", "12576", "18769", "7643", "814", "14741", "12717", "2892", "13194", "20089", "3640", "6639", "6707", "24244", "24857", "5625", "24462", "24647", "4413", "4808", "4624", "176", "5248", "13327", "1313", "14314", "2424", "8994", "7121", "20842", "21593", "24202", "17060", "13800", "6239", "292", "13896", "9506", "4702", "14411", "5062", "21932", "25542", "10055", "18373", "5161", "15472", "2178", "25831", "6049", "19336", "1433", "3237", "1999", "3095", "20923", "25814", "19462", "728", "4202", "16932", "8687", "22195", "13834", "7604", "7281", "16853", "17698", "23879", "12527", "23334", "1587", "17277", "16376", "5372", "9200", "10958", "2755", "16638", "2267", "11874", "22680", "1800", "1607", "7464", "5517", "14317", "12361", "21444", "16390", "15692", "6224", "11149", "14581", "7621", "4941", "3113", "2566", "5033", "17586", "15601", "7487", "5845", "7860", "25196", "186", "13255", "1332", "24155", "15254", "25418", "16394", "6598", "19551", "8507", "8347", "25771", "10371", "19279", "7607", "23397", "19414", "1410", "6858", "9104", "11733", "3868", "3862", "4069", "17429", "15417", "18071", "24904", "11756", "21853", "10706", "14118", "14015", "9120", "23291", "3370", "10956", "1511", "5883", "25191", "14379", "19826", "24939", "19832", "11578", "18289", "19438", "14918", "357", "18460", "20583", "25302", "1254", "727", "20355", "14680", "18186", "10153", "1947", "13606", "25276", "21561", "18965", "6110", "2417", "2768", "4172", "20579", "6051", "264", "8644", "6117", "9570", "9261", "2024", "14231", "5660", "8403", "12143", "11674", "1763", "12960", "24151", "8983", "24233", "19984", "20103", "18833", "2355", "15323", "11712", "8599", "13715", "11672", "25445", "16857", "21198", "709", "17029", "24755", "18363", "10332", "8107", "11948", "2437", "13475", "7509", "22878", "4199", "10903", "16107", "6090", "21679", "24418", "19246", "1329", "23643", "24540", "24635", "21162", "24477", "22401", "19132", "1180", "3834", "14819", "18720", "22493", "18383", "19905", "13761", "23305", "5867", "2497", "10769", "18264", "14674", "13774", "14355", "3213", "3917", "7564", "5779", "25659", "1774", "18925", "683", "8855", "12058", "12378", "15678", "15801", "10581", "8274", "11283", "9546", "25607", "7410", "3122", "9378", "10353", "21039", "12405", "22898", "24065", "24607", "853", "16784", "13161", "13794", "22107", "24935", "7662", "9491", "16155", "25533", "10971", "3290", "15150", "20433", "7845", "11884", "5149", "3292", "7081", "23898", "8586", "24685", "12041", "20723", "553", "7603", "2902", "4323", "10615", "7098", "13337", "21185", "17375", "875", "24719", "9935", "8140", "11302", "25719", "13580", "20446", "1984", "12159", "6324", "13180", "4859", "11086", "18921", "24347", "2188", "12748", "6044", "7705", "24074", "21672", "16679", "21109", "2002", "10920", "3480", "8192", "12069", "4313", "12223", "11133", "12691", "22454", "11148", "2486", "16807", "20676", "20918", "20828", "10970", "15890", "16560", "6927", "25062", "10562", "13697", "3672", "12393", "2145", "12625", "23818", "11473", "339", "13614", "1959", "10941", "10206", "13507", "20837", "11737", "21090", "21463", "7392", "12209", "5567", "14577", "2798", "10232", "1032", "7419", "18731", "6880", "21084", "24271", "2704", "13439", "17732", "12003", "6350", "12217", "8926", "17463", "24895", "9673", "25551", "13972", "5741", "9186", "4461", "19298", "2225", "14524", "16992", "3644", "25265", "15375", "1716", "4530", "1867", "18877", "16556", "22194", "15998", "4566", "4266", "22991", "1184", "8928", "3741", "8545", "11285", "9578", "24339", "22546", "25484", "12368", "16029", "20684", "22625", "23786", "10702", "25942", "2778", "19917", "16403", "6586", "5698", "9098", "25898", "4152", "10420", "8548", "14528", "2186", "12168", "11875", "8724", "22957", "9766", "5274", "11841", "19679", "1567", "25586", "5089", "2363", "4007", "18297", "4993", "14837", "4457", "10752", "9708", "10937", "4755", "974", "18254", "1079", "24139", "11797", "21983", "19304", "14974", "20135", "3151", "9224", "9774", "5695", "9848", "16240", "17770", "3554", "9038", "24358", "22344", "23721", "20674", "23549", "8792", "1494", "4921", "19373", "8207", "12967", "23747", "7725", "1847", "24843", "9782", "3036", "15920", "5075", "20859", "3997", "2080", "3525", "4845", "13808", "8500", "8873", "11022", "9271", "25250", "21982", "25899", "1799", "18247", "8533", "19446", "12146", "22294", "13685", "20556", "13642", "6507", "13548", "2654", "9764", "5136", "8267", "1929", "1391", "389", "11214", "19056", "10988", "3664", "11470", "18881", "238", "12269", "5073", "390", "22746", "12896", "22703", "12635", "2766", "13217", "21597", "14018", "5514", "23646", "17562", "3508", "11535", "25864", "4277", "15367", "2406", "15567", "7657", "12287", "23906", "10086", "5946", "25423", "8579", "3837", "4704", "745", "8113", "22263", "10499", "17987", "6015", "6942", "25888", "22674", "6430", "21986", "10375", "2367", "2935", "22792", "2062", "496", "5541", "8894", "19953", "12294", "466", "20574", "21692", "10241", "8903", "7379", "16234", "22873", "21128", "14643", "21093", "9195", "4379", "14772", "12244", "20000", "10800", "21681", "6718", "6158", "20710", "8672", "11946", "17247", "19429", "15516", "4685", "17424", "20857", "4586", "25834", "8874", "21437", "19090", "21735", "23806", "1199", "24889", "11039", "16111", "17883", "2559", "922", "4715", "3328", "20632", "6256", "8753", "25320", "8977", "7583", "11571", "16892", "6122", "11045", "16465", "3092", "24304", "4180", "11355", "19553", "4374", "534", "23915", "7050", "11931", "8555", "18914", "11869", "3788", "1212", "22133", "5335", "22414", "19500", "23096", "25606", "18699", "16490", "19584", "2807", "19959", "5511", "24917", "10700", "8478", "16944", "17776", "23647", "22124", "6621", "1980", "14235", "2711", "15974", "6161", "15265", "1193", "7431", "4008", "7819", "10", "3431", "24490", "5510", "3220", "16634", "16476", "19241", "1626", "1284", "23297", "19167", "19231", "12710", "18897", "24566", "10050", "6753", "20621", "11229", "10066", "21421", "16347", "146", "1593", "4125", "2220", "6683", "7502", "24970", "2043", "21408", "15810", "2007", "18622", "25981", "22351", "21073", "12800", "12811", "20042", "15373", "5802", "968", "24729", "12137", "17468", "1471", "16008", "15313", "15571", "11181", "19166", "3195", "9990", "1357", "19969", "13967", "23684", "14609", "19544", "11982", "23482", "21781", "2695", "770", "14700", "21458", "3789", "10987", "22706", "2112", "11579", "25536", "8884", "15123", "1870", "9925", "2160", "800", "9204", "10255", "12876", "10957", "11108", "13645", "10271", "21607", "231", "19971", "4643", "11773", "10109", "8071", "22520", "20651", "721", "13981", "15568", "10468", "5298", "22389", "8569", "8811", "4816", "18685", "12320", "2989", "10043", "5496", "7366", "10802", "3192", "17622", "23800", "5046", "3590", "11112", "15902", "3546", "20823", "2391", "12370", "10037", "17244", "423", "9666", "24211", "6222", "6986", "7177", "17571", "14273", "13478", "10527", "12165", "12428", "22718", "9982", "18137", "14377", "5566", "11721", "1207", "20051", "15873", "18084", "1401", "9067", "3582", "13249", "8780", "17321", "6908", "20750", "11100", "12338", "15394", "23438", "946", "9878", "15369", "9811", "19738", "11557", "1026", "18993", "19840", "7057", "22784", "2720", "22787", "11016", "11774", "2675", "17765", "14393", "19967", "24681", "15324", "20559", "4096", "2097", "18976", "5613", "23165", "2340", "10458", "4581", "4181", "15771", "19472", "5957", "15317", "5515", "21191", "1164", "6297", "4777", "22789", "3068", "4839", "18962", "11498", "7087", "1118", "13447", "12918", "18408", "4085", "19023", "13593", "7629", "2166", "20360", "5288", "21601", "8188", "12006", "16966", "4075", "4568", "12689", "23796", "5370", "7093", "6649", "14352", "2811", "1869", "16991", "18376", "25837", "23768", "3981", "5495", "8126", "6915", "14899", "21715", "1787", "23227", "25304", "10936", "691", "2357", "7091", "25999", "881", "2512", "11358", "10118", "9914", "23137", "25579", "19074", "9039", "20309", "4952", "22861", "11336", "8455", "345", "6255", "19854", "5979", "11717", "11770", "459", "12557", "7157", "14504", "23106", "5329", "6884", "20749", "1251", "10962", "4114", "11367", "18092", "5254", "15055", "24141", "18600", "563", "25741", "10283", "22234", "8626", "14065", "17823", "7773", "8917", "11575", "15600", "22482", "11373", "15269", "24140", "20119", "4259", "19968", "20824", "23315", "7436", "22553", "19525", "4592", "3597", "17501", "14927", "5784", "16889", "13096", "16031", "9945", "11480", "8854", "23887", "16248", "23686", "14790", "10836", "14604", "22610", "16879", "8512", "12001", "2845", "25269", "25035", "9040", "6269", "3199", "20549", "13358", "9837", "15918", "17751", "18213", "14962", "10606", "19434", "21188", "10022", "17959", "24867", "11262", "13224", "21259", "23450", "5119", "22055", "3311", "12498", "9664", "17760", "17872", "14449", "21445", "9141", "24982", "9916", "4118", "13114", "15736", "20863", "12549", "1403", "21442", "4889", "5432", "2480", "19264", "782", "11814", "9465", "22762", "21121", "19207", "23119", "5063", "16791", "8821", "7292", "4378", "9077", "10980", "8401", "18253", "13997", "24877", "16271", "24989", "283", "18922", "23572", "17762", "12042", "8170", "10934", "18104", "15220", "7535", "14168", "4388", "11925", "4066", "1697", "9903", "14766", "11293", "322", "10624", "9735", "1990", "17139", "18753", "3733", "25695", "7355", "11493", "15985", "22388", "23993", "20587", "5436", "7701", "24329", "3413", "1237", "7155", "184", "3423", "19001", "22024", "23905", "4762", "23300", "3132", "7709", "17705", "7989", "11077", "8519", "8", "9879", "23798", "24412", "2490", "10257", "7297", "7043", "1394", "13674", "7142", "18760", "11780", "2781", "21112", "11902", "15087", "13668", "12433", "940", "3926", "3913", "2584", "19900", "24526", "9976", "8398", "8446", "22540", "23468", "6038", "23937", "10901", "4185", "18863", "7746", "8637", "25326", "5816", "15452", "14338", "16732", "14975", "3706", "4866", "10605", "15824", "7348", "17929", "22243", "16346", "11785", "12862", "24327", "20338", "25610", "232", "21478", "267", "16093", "9304", "20575", "13694", "24552", "14804", "24483", "6485", "7059", "10331", "19469", "97", "13060", "20907", "12869", "20", "8449", "9938", "23511", "5814", "12998", "15093", "10007", "8452", "17780", "14599", "1367", "23413", "8936", "285", "8963", "25687", "20010", "6531", "4267", "9727", "8742", "24574", "718", "24500", "21528", "4425", "22034", "6142", "5805", "17066", "25847", "22035", "298", "1689", "10787", "17547", "12022", "25362", "1816", "2919", "16418", "7255", "1243", "14941", "21497", "6043", "24931", "5868", "9389", "16834", "19220", "10863", "1917", "10981", "15277", "14505", "14906", "13739", "10814", "26018", "16943", "2793", "21909", "16775", "2219", "16781", "15978", "18340", "6614", "9001", "3869", "25011", "21253", "13917", "15521", "17575", "14423", "18647", "22405", "8234", "6442", "10172", "8303", "8156", "12214", "24335", "16990", "10801", "23975", "16694", "1722", "23488", "2136", "20201", "23459", "23182", "24302", "16280", "362", "10122", "8518", "8992", "22981", "5540", "3771", "25117", "3624", "15253", "8201", "11331", "11821", "16699", "11067", "9824", "24469", "21635", "12535", "24641", "11364", "19251", "9374", "11007", "7553", "883", "295", "6234", "12082", "6862", "11901", "24345", "14787", "12739", "15005", "7808", "4700", "11565", "5487", "17107", "4190", "12822", "1565", "16734", "21465", "18979", "18342", "5692", "5854", "2182", "14172", "25474", "22490", "21629", "23727", "9529", "24061", "17478", "14506", "15140", "22249", "2971", "7637", "19841", "1595", "12952", "25525", "4612", "3296", "25411", "18978", "8745", "20329", "21704", "15839", "18236", "18689", "6876", "10046", "10150", "22935", "15860", "16241", "8257", "24797", "19708", "23710", "24944", "12492", "12870", "1740", "14488", "9793", "12736", "14414", "18149", "905", "7726", "3947", "8706", "25021", "11092", "21563", "25969", "9202", "20370", "6541", "1501", "10303", "7913", "8796", "17017", "8709", "20048", "7874", "10725", "6099", "20763", "7881", "22499", "11859", "16921", "6461", "25140", "21539", "7803", "22513", "20840", "15082", "212", "12000", "12351", "24071", "4497", "17101", "3395", "23036", "19081", "16833", "14707", "2672", "10865", "13752", "19454", "7846", "5381", "23954", "24400", "20389", "18801", "9562", "5427", "17464", "19439", "22228", "865", "20340", "4373", "187", "2418", "6226", "3120", "1120", "10314", "2519", "9737", "11569", "762", "17488", "8929", "16735", "7021", "1915", "19345", "20171", "23515", "21882", "3859", "18335", "6092", "6886", "14701", "9585", "19398", "7770", "14631", "18090", "1108", "3681", "618", "17576", "6443", "24286", "16377", "17317", "20043", "15379", "22076", "14160", "20123", "10387", "8454", "21534", "1725", "13394", "20922", "22686", "1720", "19890", "2328", "21136", "12532", "25583", "24134", "24547", "3291", "9134", "14516", "7520", "19267", "217", "3319", "15320", "11436", "1919", "14006", "12315", "24582", "25377", "16631", "20688", "24849", "16298", "7787", "1579", "13477", "13718", "15153", "23682", "13523", "14469", "21972", "1225", "23952", "12696", "16750", "12431", "14030", "24007", "19342", "4134", "14755", "786", "21758", "13341", "22530", "21475", "3585", "12301", "1051", "1475", "21193", "13352", "1835", "13517", "15792", "4100", "15703", "19662", "25153", "11919", "22093", "3268", "1085", "12230", "7755", "18352", "24968", "2307", "11311", "11560", "1760", "21064", "15525", "9156", "2118", "15937", "6552", "22434", "11205", "3615", "578", "12788", "13578", "20479", "20783", "6351", "1536", "21653", "17618", "4693", "7023", "3708", "21687", "3062", "5153", "15706", "19083", "2546", "7456", "13154", "11596", "18357", "19535", "316", "11893", "7115", "4049", "21403", "8365", "12955", "18064", "1680", "13837", "8886", "25535", "15048", "15805", "17842", "24247", "25658", "765", "10219", "290", "12886", "17878", "14258", "9349", "25735", "5017", "20904", "23355", "2400", "1605", "24381", "25929", "10558", "9457", "3712", "4218", "9893", "19538", "15262", "5818", "3219", "3993", "6803", "17768", "16896", "4619", "10676", "15130", "12169", "4366", "25743", "17323", "4562", "12052", "7106", "6910", "6246", "22359", "17068", "1599", "9693", "21501", "10225", "4666", "4435", "888", "1017", "17615", "19", "15809", "5442", "24038", "10149", "16453", "25910", "17", "10235", "16838", "17236", "18592", "5863", "23838", "16000", "19666", "25439", "11572", "16235", "14517", "2620", "6875", "13810", "13245", "13758", "7900", "17811", "23843", "4153", "5339", "14636", "12050", "12387", "245", "2247", "1737", "25724", "24630", "15483", "24187", "11836", "8939", "9306", "20562", "21343", "16434", "2401", "1191", "21484", "6100", "4679", "8141", "9717", "24341", "14759", "10519", "9798", "11127", "3402", "21572", "21080", "24546", "19991", "23542", "981", "21286", "4983", "7385", "23947", "10125", "15142", "25425", "18918", "5851", "19724", "12646", "14269", "137", "11377", "13946", "25788", "7368", "22545", "25488", "2319", "4236", "19727", "15626", "11441", "18614", "20055", "6465", "9765", "24395", "1702", "24886", "15628", "5156", "23162", "24456", "24464", "18546", "20250", "25470", "16632", "4419", "14892", "20096", "20154", "24901", "9777", "24396", "16718", "19184", "19020", "14739", "3694", "3765", "20354", "9128", "25865", "18619", "15762", "15386", "21628", "26013", "11201", "9368", "23653", "23966", "9582", "6965", "5539", "16404", "2313", "20191", "2602", "14656", "18466", "7235", "23229", "4483", "23079", "17441", "16489", "7303", "3197", "9121", "993", "22087", "14522", "3259", "13213", "1604", "10751", "16335", "18022", "22382", "4509", "994", "20034", "6599", "11714", "25547", "2435", "7340", "13570", "6955", "25940", "18252", "2015", "13055", "16814", "2395", "23605", "7097", "24391", "13059", "14230", "1842", "2429", "19288", "13123", "2662", "24820", "18855", "12098", "10210", "12841", "24129", "79", "1389", "14802", "574", "22860", "22714", "9956", "14849", "1078", "15644", "9442", "12489", "7341", "1113", "22419", "18686", "4847", "14192", "13290", "19582", "10366", "23568", "23221", "25679", "15126", "112", "714", "24484", "25737", "8379", "20109", "2280", "17458", "21918", "6514", "18676", "12075", "13482", "16230", "22829", "23429", "19039", "17262", "25923", "4533", "18319", "21746", "13221", "2814", "24735", "3174", "1669", "198", "13534", "12908", "1989", "16629", "5291", "13254", "413", "16771", "18419", "23424", "25329", "3809", "19218", "9007", "13142", "17049", "18866", "835", "11663", "10963", "6795", "4060", "20223", "11417", "25281", "24915", "24372", "19405", "3439", "19275", "14557", "21166", "19613", "20151", "17001", "5880", "23088", "5383", "17828", "15843", "16523", "116", "3385", "3890", "7822", "1983", "2265", "1992", "21902", "22601", "6462", "17727", "22515", "20396", "25187", "5533", "14961", "11530", "2803", "12586", "22406", "23538", "9905", "16559", "7754", "24144", "3970", "19293", "24519", "7460", "934", "10223", "12191", "15088", "10478", "9564", "1963", "12587", "144", "5715", "7126", "25669", "5069", "25566", "15882", "2598", "3445", "7521", "7972", "19669", "21486", "24491", "21420", "6659", "16307", "24656", "14671", "20586", "20772", "5239", "17758", "18107", "13890", "2433", "4249", "2868", "12865", "17972", "16047", "18879", "7085", "16195", "15635", "17778", "21195", "20149", "5677", "7291", "1666", "12237", "3348", "6445", "25074", "23865", "18098", "21502", "3639", "18803", "16604", "9448", "17912", "21953", "6301", "24958", "10845", "16459", "17838", "24079", "18265", "21658", "11898", "1656", "5679", "21753", "5796", "5726", "10729", "7742", "6391", "3649", "24204", "4247", "5878", "13717", "2767", "6959", "8653", "20916", "13710", "1743", "14590", "15645", "19675", "4912", "14585", "15650", "3734", "18002", "25145", "16900", "7789", "1758", "1624", "22100", "21777", "7338", "18178", "1655", "6891", "11961", "2317", "12400", "10175", "5987", "2211", "26009", "18824", "19311", "25594", "19870", "14623", "20443", "10864", "5571", "214", "14695", "22392", "9029", "24699", "12303", "12130", "18167", "10718", "463", "13683", "22796", "22268", "9481", "19226", "19951", "10052", "1640", "2299", "5279", "7665", "20094", "20795", "22667", "12554", "16997", "4658", "5974", "2312", "1623", "14719", "20457", "21825", "20285", "23654", "19915", "24081", "11206", "13152", "11060", "4285", "7122", "13884", "10424", "23198", "2290", "14440", "3630", "2734", "9233", "21691", "25036", "8952", "12280", "7857", "10391", "12759", "9617", "10701", "8901", "26", "10922", "23752", "19268", "18102", "12444", "10557", "19709", "2058", "14412", "21262", "14462", "23400", "23822", "5031", "12317", "11263", "21372", "6805", "19146", "16856", "21939", "903", "25663", "5569", "22534", "19068", "1311", "7611", "4485", "12906", "10747", "14416", "11726", "619", "8820", "11186", "7510", "12783", "13847", "8782", "23207", "982", "9021", "8433", "16982", "202", "2661", "22976", "12699", "19175", "1277", "8271", "23097", "6921", "420", "23320", "6809", "9646", "18263", "12162", "6830", "2356", "18821", "3691", "18969", "3073", "12132", "19636", "14271", "22274", "1765", "4333", "24509", "20353", "10529", "25982", "14296", "23655", "10406", "10212", "17492", "23376", "2805", "8131", "19731", "4781", "9773", "5053", "4713", "15993", "690", "11546", "8676", "19430", "7752", "11380", "5459", "23332", "18854", "1423", "5282", "9778", "16850", "21718", "6200", "6002", "2881", "18718", "15515", "23381", "22068", "3864", "6285", "25769", "18668", "19098", "25086", "22650", "1358", "7877", "8391", "13348", "23839", "6550", "8258", "23063", "3668", "160", "22347", "2402", "6851", "14659", "10085", "5609", "22805", "4255", "1748", "11319", "4278", "23863", "20014", "20974", "11769", "9071", "15518", "18381", "25408", "5687", "8622", "16601", "10588", "13561", "8741", "22658", "11264", "18058", "15759", "13143", "15347", "11370", "3189", "19990", "13085", "10298", "11389", "14777", "6860", "6279", "1782", "18657", "15177", "2240", "9323", "10080", "1319", "12539", "13463", "20167", "7314", "12447", "1385", "18258", "7727", "6758", "7812", "7650", "16681", "24410", "9427", "464", "11632", "2691", "15226", "13503", "17295", "1042", "10691", "23830", "5793", "15502", "23473", "23289", "15060", "13675", "25532", "3722", "2557", "21426", "1859", "7670", "11361", "3562", "1895", "3831", "21863", "3001", "23744", "7173", "1696", "24927", "25985", "24553", "13601", "15930", "16137", "15499", "560", "15006", "16920", "13199", "7161", "7315", "1052", "25245", "22547", "23292", "18971", "12777", "10677", "10443", "18733", "3320", "25099", "15555", "22354", "17692", "23639", "14424", "21905", "8932", "15539", "9769", "22984", "6899", "17410", "11984", "21299", "2573", "14964", "20324", "19378", "5750", "14867", "23957", "4804", "16089", "24955", "11129", "302", "18099", "7743", "3473", "243", "15387", "11778", "8868", "19331", "24667", "1152", "25455", "21477", "7946", "13328", "18853", "7077", "8193", "3718", "827", "5109", "6130", "8169", "18482", "4676", "22739", "428", "21434", "8935", "18299", "25331", "24712", "20577", "18181", "20537", "6743", "17333", "47", "311", "10505", "25378", "14547", "19975", "23208", "21316", "12786", "12812", "16777", "8598", "11526", "12074", "14247", "6938", "20249", "4068", "6879", "25482", "1482", "2707", "21324", "16654", "23074", "2215", "9256", "9843", "15166", "7219", "21380", "9702", "17829", "25039", "25651", "12789", "8921", "18046", "18494", "8673", "5419", "6894", "25681", "17885", "22931", "15528", "18451", "20429", "9950", "11362", "19688", "349", "24623", "15716", "321", "18751", "70", "24844", "15548", "24780", "19322", "9926", "4632", "20363", "1602", "10378", "15532", "4143", "9369", "9278", "15393", "22276", "18936", "14077", "7661", "14596", "14888", "19093", "18089", "5196", "25584", "13456", "4654", "9791", "12383", "3297", "7778", "23840", "20464", "8315", "18450", "19407", "17286", "1065", "1415", "1738", "5956", "3107", "25192", "14509", "15967", "5251", "12391", "19520", "2371", "2578", "10167", "21099", "3396", "1840", "7708", "25064", "22064", "5068", "24907", "1622", "10859", "7973", "19627", "9876", "14019", "1752", "11145", "22843", "23791", "7451", "10682", "3833", "12102", "17590", "11169", "18421", "11489", "17846", "18572", "9713", "24176", "24679", "11268", "11211", "13343", "5405", "12744", "23773", "12507", "26023", "10344", "25454", "5192", "16154", "21368", "20244", "6469", "22928", "14444", "8685", "23150", "17190", "13749", "14527", "23759", "13508", "22254", "16826", "22043", "12782", "23407", "11020", "13720", "15023", "12640", "24133", "6850", "12477", "16917", "13652", "19250", "17574", "1469", "20252", "25880", "784", "18611", "8646", "17496", "14437", "25569", "16360", "24394", "5447", "23100", "3716", "22314", "3429", "11359", "6314", "5247", "4319", "14196", "10695", "14959", "19277", "7504", "3968", "5445", "8999", "23176", "24743", "9794", "1862", "15072", "14597", "7376", "17376", "5268", "1231", "8132", "15329", "14552", "4897", "23616", "24003", "16662", "41", "10670", "14089", "1634", "19051", "3750", "4505", "1126", "20553", "24782", "7092", "20698", "3367", "25365", "10684", "19948", "9108", "4124", "22529", "4732", "5030", "6068", "8838", "22713", "10549", "4097", "21087", "20532", "18673", "140", "20735", "7455", "11808", "15080", "25740", "12117", "8890", "7183", "24648", "24610", "24228", "12629", "8291", "21850", "8754", "8083", "17580", "4974", "15735", "14840", "17803", "12479", "15845", "13117", "11279", "7304", "25054", "19939", "16972", "9967", "6424", "25485", "8634", "15925", "24902", "15035", "11151", "1181", "9424", "17186", "2731", "16760", "16643", "13165", "6672", "17717", "3916", "3218", "5860", "17457", "6141", "25", "3349", "2576", "25354", "14025", "11782", "3621", "5444", "10772", "12536", "10584", "7356", "12485", "16907", "13957", "3521", "9459", "22445", "3046", "8477", "14540", "24325", "17327", "11076", "22973", "13786", "18222", "15012", "8885", "6101", "6434", "1111", "16324", "5193", "12885", "13350", "4637", "12938", "2473", "11831", "6811", "14405", "25018", "25121", "15986", "1662", "7206", "16639", "18152", "17558", "4003", "25014", "8688", "2773", "9099", "4173", "15670", "25975", "1257", "5831", "11494", "23149", "25823", "12009", "14045", "7749", "4356", "15616", "14222", "13156", "2531", "7354", "8171", "3343", "17747", "10201", "9784", "7941", "7270", "23385", "19876", "9559", "5526", "8390", "18228", "25460", "11855", "7608", "23487", "16212", "14858", "17027", "9392", "15200", "15620", "17360", "25538", "3531", "2279", "3024", "9486", "5026", "5364", "22201", "20721", "23430", "15359", "24534", "20799", "8862", "9608", "4606", "15581", "12438", "21492", "11503", "25979", "23547", "18988", "21081", "5146", "14814", "11580", "1769", "8892", "4009", "25219", "20410", "810", "13511", "14760", "14783", "4414", "20246", "14108", "19035", "24028", "15148", "3829", "15002", "13399", "20992", "2110", "13210", "16646", "12318", "11491", "13058", "25201", "4436", "24615", "13987", "5449", "23552", "10282", "6721", "12567", "1349", "24308", "19124", "7001", "17437", "5981", "22400", "25956", "3780", "7945", "13873", "16580", "10781", "23141", "15420", "10419", "6527", "12505", "14382", "10766", "21814", "20882", "15186", "19308", "25571", "12735", "12002", "15346", "4520", "18136", "10889", "2714", "22305", "8366", "21077", "25410", "9004", "17070", "1886", "12927", "17036", "6957", "15040", "21843", "13310", "13002", "13043", "1253", "2147", "7769", "11798", "24426", "24105", "7811", "23456", "15416", "14638", "13238", "16184", "9749", "1856", "3436", "19698", "19295", "13687", "5456", "2027", "24259", "12958", "17864", "11427", "4297", "14955", "24689", "3576", "1413", "1368", "3238", "21839", "17149", "18498", "4399", "190", "22630", "8211", "14747", "9446", "20730", "7748", "16432", "22517", "6833", "20157", "23613", "94", "15463", "4830", "22543", "9776", "2530", "1354", "8828", "549", "150", "9831", "22562", "5050", "16825", "3905", "20085", "8746", "11012", "16018", "24417", "16208", "13740", "643", "21674", "19294", "1710", "9265", "25060", "5788", "8421", "301", "18105", "3629", "13993", "24707", "13906", "7685", "692", "9612", "2447", "17158", "24045", "9577", "5113", "1112", "17102", "19508", "4163", "22906", "4204", "6693", "21018", "9337", "11515", "12375", "17389", "10369", "7307", "14193", "22943", "10653", "13904", "17752", "6975", "8744", "1185", "2667", "24320", "5914", "25643", "233", "10019", "3249", "13077", "23143", "15731", "10362", "19374", "12177", "13232", "14818", "24826", "6520", "17706", "15079", "25352", "17270", "5909", "25038", "3989", "1343", "13885", "1038", "19558", "7854", "2772", "13018", "4683", "22737", "5207", "10746", "10735", "21855", "17935", "16166", "2176", "23446", "22114", "3108", "25821", "6994", "6096", "15591", "22732", "872", "22074", "3459", "15482", "21168", "10197", "24585", "21457", "11767", "22518", "20192", "18204", "7170", "15593", "10665", "20968", "9594", "6207", "1116", "9758", "7573", "12239", "611", "608", "7246", "171", "14485", "23181", "20032", "25026", "19612", "19897", "4835", "12349", "6343", "25791", "12246", "16264", "16685", "20046", "6644", "208", "3227", "275", "794", "11517", "14925", "12624", "19158", "8865", "13021", "13184", "21625", "23295", "19163", "455", "24883", "4610", "4940", "14203", "8798", "20005", "17358", "18538", "5198", "25674", "19623", "1239", "25859", "18261", "10336", "11497", "620", "25836", "11713", "10855", "25079", "13934", "8661", "3500", "13923", "20705", "501", "6676", "1441", "22015", "21981", "21943", "18711", "20364", "22641", "9135", "24521", "9335", "21294", "20141", "12451", "12394", "15536", "10152", "7403", "17097", "8125", "6201", "21993", "25327", "1466", "14198", "7295", "23827", "5862", "10200", "21419", "7226", "17598", "5985", "17310", "21041", "22655", "26016", "4086", "10619", "24930", "4914", "18218", "14087", "8714", "14039", "21571", "13996", "7430", "24033", "21741", "8743", "20777", "16092", "6947", "8164", "4427", "12542", "24734", "1156", "7273", "3200", "12565", "18219", "15125", "8538", "4588", "10878", "7201", "21172", "9960", "10472", "24739", "12429", "9345", "21371", "918", "1922", "6394", "6333", "25294", "12175", "25295", "6263", "5479", "21667", "14848", "18528", "21519", "22876", "5396", "6016", "19187", "20248", "22731", "1684", "22637", "7458", "23956", "25958", "6924", "20293", "4036", "3474", "900", "4037", "10741", "16190", "18558", "21921", "8024", "15467", "25978", "11347", "5317", "16483", "10474", "24785", "24592", "23406", "5463", "23851", "18730", "23122", "25098", "18023", "22593", "19519", "4240", "11400", "23496", "8947", "9985", "9545", "2738", "25876", "16795", "9331", "18542", "17018", "7632", "11908", "5623", "11366", "24621", "24258", "23760", "25872", "4345", "457", "20379", "9826", "14433", "8490", "1690", "25808", "23949", "2988", "22326", "1039", "19875", "2515", "25930", "25514", "22129", "7334", "9637", "21225", "15893", "4717", "1988", "5178", "22146", "23719", "1871", "9543", "4873", "25323", "11046", "4776", "885", "16891", "16579", "10510", "18308", "19552", "9194", "10546", "8800", "21506", "3534", "22776", "8212", "4891", "21170", "10120", "7541", "25989", "25661", "13643", "15931", "2432", "15212", "25147", "13469", "19934", "2750", "15604", "14125", "7840", "305", "3425", "24661", "21874", "22287", "2063", "10583", "6587", "21851", "22544", "16449", "14743", "7938", "22453", "14127", "14714", "599", "8384", "15210", "8970", "15203", "15480", "9176", "19011", "23983", "6457", "21945", "21867", "3191", "17661", "14640", "626", "15213", "13370", "17385", "491", "25397", "24094", "5656", "257", "12561", "11621", "19024", "18938", "19066", "5876", "20834", "18847", "2203", "12064", "8701", "2294", "10504", "25642", "14617", "4293", "8481", "2677", "11735", "8249", "20194", "15786", "11746", "17869", "5984", "10846", "18792", "17696", "16801", "9165", "3468", "6854", "25677", "10912", "5501", "16302", "3856", "23758", "6377", "13186", "7478", "25955", "17840", "18613", "14720", "11976", "20704", "12658", "24444", "1364", "20474", "21359", "9051", "4288", "73", "6673", "4101", "5641", "5767", "2702", "18707", "7316", "17798", "20150", "19245", "14388", "12118", "21592", "5007", "2339", "3455", "8420", "1613", "23854", "24628", "3745", "15410", "21313", "19445", "25964", "12631", "12010", "21495", "4472", "1479", "23057", "4794", "23224", "25773", "18641", "17928", "22818", "18133", "971", "6698", "3855", "10711", "15106", "10995", "7835", "20126", "9640", "13524", "7858", "9803", "8183", "9403", "10318", "6523", "13260", "8990", "13346", "7481", "24154", "21364", "8016", "9361", "2099", "7627", "17733", "3489", "16663", "11819", "25749", "1777", "327", "2345", "136", "24567", "20606", "8462", "6595", "24156", "7065", "7055", "22852", "26027", "2230", "17064", "12831", "5627", "5378", "6156", "8534", "19141", "24149", "14486", "3548", "5934", "12241", "22253", "8922", "7967", "10435", "22336", "7490", "18773", "7795", "8479", "23808", "13875", "16985", "23144", "8196", "1944", "24321", "12910", "14554", "14144", "15984", "5411", "11787", "1141", "24794", "10325", "4842", "12298", "2647", "5455", "11068", "25057", "538", "23146", "15183", "16788", "10543", "2196", "7537", "19034", "9941", "14371", "24162", "2874", "5345", "14219", "14105", "11040", "18750", "16470", "17676", "3021", "16640", "16397", "3152", "17461", "19122", "5098", "20227", "10173", "13883", "2954", "24115", "17019", "22390", "7766", "9357", "13956", "23896", "9724", "18513", "21279", "23621", "19057", "20725", "5173", "4539", "25053", "4597", "8814", "20753", "23924", "11682", "1190", "10323", "4343", "24379", "1031", "22797", "11000", "12120", "9456", "14000", "10576", "23378", "13324", "16730", "16238", "15401", "3667", "12541", "22040", "22192", "11534", "20599", "3136", "24070", "846", "22248", "16246", "3045", "24277", "24353", "14130", "21091", "4241", "4116", "6786", "10622", "9544", "21538", "5915", "18901", "18665", "16159", "18975", "18018", "14251", "2579", "13664", "19368", "16532", "2908", "16081", "3121", "22911", "4604", "7949", "18320", "14847", "6972", "14672", "11655", "6638", "3087", "8112", "12257", "10326", "17439", "8075", "23531", "7673", "22119", "21577", "16183", "24084", "3596", "19954", "3915", "15586", "4336", "2124", "15894", "24282", "13900", "22424", "139", "23124", "7075", "10404", "4896", "19426", "11365", "25500", "10817", "19570", "6004", "8941", "11256", "3972", "12993", "8897", "11683", "17550", "6655", "24472", "13367", "24040", "7500", "20898", "4302", "10542", "9900", "8756", "15053", "7690", "6615", "23219", "5603", "23845", "24493", "9185", "8094", "25285", "336", "24726", "8458", "18872", "14149", "17902", "20049", "9583", "9762", "12973", "21386", "21900", "16491", "18770", "25119", "1303", "5509", "15292", "8430", "19842", "23940", "20029", "8041", "21328", "6476", "13449", "24057", "2932", "1585", "5219", "22824", "9326", "5598", "16407", "5322", "3058", "20754", "14727", "22147", "14536", "18675", "11091", "20961", "20214", "7298", "15184", "23263", "15757", "11485", "2740", "22570", "12719", "20775", "23301", "10274", "12135", "10871", "25135", "20716", "23369", "8286", "13716", "1098", "1874", "8331", "12915", "20342", "15818", "10523", "7762", "20470", "9949", "3874", "4633", "22507", "12864", "16290", "19836", "10281", "4855", "2035", "25185", "24578", "18584", "1750", "8605", "22996", "10849", "15614", "25128", "16253", "20347", "2343", "25004", "12369", "1775", "5409", "17450", "17500", "6616", "22807", "25271", "7495", "10768", "13205", "22342", "24786", "7022", "2955", "13852", "14266", "6380", "6665", "17721", "13379", "1601", "19420", "22649", "13555", "1664", "13106", "20708", "12729", "20344", "25432", "8581", "3306", "4883", "10238", "4355", "20766", "1287", "13841", "8960", "19302", "833", "7688", "14709", "12274", "23023", "12195", "3433", "8973", "13563", "6896", "15613", "10890", "4150", "24565", "2479", "123", "2485", "9937", "11024", "7361", "5099", "1615", "14062", "17820", "10381", "2248", "21964", "1584", "2605", "7529", "20642", "20821", "14058", "11928", "5685", "23485", "3727", "4405", "13690", "10388", "113", "20373", "16726", "9293", "23001", "18748", "14583", "21194", "24388", "25493", "20665", "2037", "1901", "25376", "19685", "13181", "22142", "1574", "14021", "13075", "19925", "1687", "22581", "2856", "5036", "24179", "11913", "2198", "23537", "12590", "14629", "18878", "7213", "16173", "16216", "24658", "17755", "2459", "22341", "8850", "1374", "6108", "6489", "24280", "15906", "1071", "7780", "15068", "13484", "3684", "17382", "1008", "6671", "15286", "9732", "23148", "3051", "21012", "12970", "11896", "9510", "10469", "10757", "23829", "12600", "3090", "3752", "3481", "13858", "5706", "3522", "19572", "20745", "10770", "5892", "325", "23803", "17173", "1056", "13471", "25100", "4040", "5160", "11552", "10182", "11609", "23469", "14981", "22421", "851", "3086", "9168", "3988", "17887", "11476", "19164", "20776", "18531", "17483", "1863", "21401", "16765", "18234", "9217", "566", "6190", "2025", "2790", "15579", "22554", "11093", "20302", "16186", "12551", "5166", "20388", "2759", "1134", "20677", "16540", "21354", "19930", "8178", "8177", "5828", "19211", "258", "25665", "21432", "25945", "4865", "1745", "21559", "898", "24318", "25393", "6367", "24757", "14902", "84", "17328", "25596", "15817", "1966", "11456", "23535", "25282", "15424", "17810", "5841", "1421", "16378", "12190", "23393", "15326", "23687", "19704", "9192", "2212", "18980", "14408", "4407", "17675", "13261", "19581", "10597", "10939", "23003", "12304", "8784", "5428", "15981", "13648", "18565", "1194", "17152", "10823", "24622", "12077", "17209", "19259", "14281", "23031", "4171", "23907", "12446", "21307", "22831", "20499", "8466", "13522", "16197", "5789", "6709", "2440", "1379", "12243", "18661", "21973", "2269", "3996", "7094", "7353", "58", "2652", "6452", "20861", "7588", "11957", "15102", "4945", "22870", "12324", "5736", "15039", "12926", "10471", "1100", "21772", "24978", "10973", "21866", "2441", "9110", "849", "18540", "12293", "5475", "13742", "15063", "3983", "72", "8683", "15223", "5756", "20607", "22591", "6357", "2681", "9509", "25676", "24569", "7776", "25138", "24104", "8012", "13201", "2517", "4384", "314", "15475", "6941", "19944", "2077", "22187", "5948", "4322", "15948", "6974", "19004", "23585", "9305", "22426", "22132", "20494", "7033", "17289", "8392", "21305", "12652", "13576", "23930", "14669", "20590", "3210", "23835", "0", "3954", "7322", "14921", "14286", "17670", "19788", "6902", "13702", "7824", "13499", "12347", "17601", "21930", "6133", "5968", "13259", "13743", "15652", "2543", "15727", "712", "10612", "24880", "23166", "1072", "18082", "22182", "366", "18825", "16536", "21756", "18140", "8806", "14812", "24744", "24702", "22954", "2716", "117", "4551", "919", "22216", "19712", "8356", "21347", "25953", "9262", "573", "6579", "17666", "2777", "11647", "565", "332", "11959", "2595", "6338", "10020", "25383", "211", "19428", "12568", "2444", "4627", "1751", "12412", "4005", "20269", "14463", "25142", "23730", "23720", "8415", "9687", "12115", "6292", "2680", "18551", "1334", "10803", "3196", "23586", "16454", "10493", "22010", "13353", "21568", "18120", "7952", "1380", "22856", "22698", "16751", "10975", "5712", "7", "15776", "10209", "13103", "10951", "23592", "3724", "3448", "15448", "10683", "16149", "18476", "21381", "10819", "10875", "9881", "15533", "18056", "6053", "22669", "24213", "4772", "11444", "12460", "16539", "3214", "19560", "24408", "4525", "4083", "12360", "16828", "13807", "3209", "13432", "15881", "25463", "22899", "20990", "15073", "25251", "11670", "10147", "24913", "9355", "10674", "20458", "2253", "13487", "22583", "16851", "9340", "23142", "2364", "10868", "17256", "10719", "22538", "9655", "22711", "3974", "15495", "6675", "14386", "2920", "10136", "12346", "7579", "6437", "8981", "4110", "24476", "6660", "18719", "867", "10279", "10116", "388", "9413", "3178", "1018", "2894", "371", "18586", "12113", "19804", "12372", "11443", "3930", "17796", "22193", "10643", "20088", "5859", "24761", "22839", "19364", "2369", "16585", "6138", "8711", "17585", "11199", "7377", "22077", "25774", "19626", "15550", "12245", "22102", "21968", "22631", "19834", "9683", "16051", "11608", "4962", "9169", "17058", "19588", "3678", "9785", "12747", "10705", "23294", "15995", "16903", "16294", "16995", "6923", "14464", "1572", "25277", "12418", "270", "3743", "21732", "18080", "16690", "4459", "8226", "1436", "15357", "19358", "11188", "25394", "9095", "3487", "20625", "10733", "2146", "3901", "7450", "25593", "3000", "12390", "6481", "10993", "16291", "25698", "469", "21092", "10999", "3872", "19736", "18118", "11447", "20597", "9371", "15351", "8934", "14595", "19297", "4475", "3533", "6280", "22404", "12934", "1224", "20361", "14665", "11033", "10246", "9866", "23631", "8337", "22577", "24147", "19009", "11642", "11419", "16350", "14886", "21075", "6643", "3980", "18364", "8537", "12142", "7947", "25587", "14600", "19488", "4208", "19192", "23792", "4122", "4595", "63", "3577", "19599", "10782", "7103", "25166", "21686", "615", "16780", "10767", "1849", "19786", "16321", "6594", "13092", "209", "16676", "2006", "22030", "23673", "1327", "14265", "3593", "1151", "5694", "14876", "1480", "14645", "937", "9196", "25356", "23467", "23815", "19183", "14627", "4432", "15081", "14418", "20572", "15796", "16657", "12251", "4243", "1578", "6041", "16597", "5725", "3140", "13017", "149", "12336", "9044", "1526", "1338", "22302", "10998", "5810", "21958", "5921", "12645", "7371", "13877", "9249", "4697", "10758", "11496", "14991", "24429", "12651", "25473", "12491", "4339", "9158", "6728", "23746", "18530", "22857", "6009", "21737", "13712", "18019", "25132", "19054", "1020", "12784", "2465", "7485", "9709", "21955", "15848", "1162", "2094", "6551", "3929", "23484", "3110", "8058", "3331", "18076", "7406", "23583", "191", "3800", "15143", "6503", "1033", "12227", "6427", "21806", "7644", "12422", "25384", "5201", "25503", "14628", "6237", "24052", "9721", "18341", "7591", "5942", "9315", "110", "6909", "6872", "5601", "16366", "13835", "15577", "13929", "15746", "17330", "24214", "10940", "9324", "23130", "6821", "9524", "24473", "17815", "6056", "9885", "7764", "19444", "7635", "7240", "1596", "531", "23797", "7128", "14904", "21117", "21802", "19517", "9614", "417", "16768", "20982", "18802", "22584", "23117", "13726", "9483", "6524", "19107", "13927", "20948", "1271", "16181", "13228", "24987", "4792", "5394", "25786", "695", "9100", "10230", "15980", "4090", "15228", "7763", "23066", "5971", "3258", "1304", "9431", "18321", "24678", "18534", "2393", "6342", "15227", "26029", "18953", "16610", "2673", "12585", "2122", "6878", "11499", "12247", "12992", "12174", "23842", "15378", "17113", "9649", "5143", "21303", "12103", "657", "13004", "3679", "17430", "16284", "10187", "17281", "3812", "19482", "13699", "13528", "3403", "14351", "17898", "3278", "22168", "11940", "17491", "14889", "2594", "330", "14534", "272", "3857", "17306", "7302", "19340", "12991", "1114", "9908", "5507", "17703", "10948", "13617", "242", "11527", "22288", "10166", "394", "20212"]], "xtest": ["int", ["14770", "6565", "15089", "2846", "5662", "22261", "24405", "23543", "24322", "4535", "11276", "15204", "1228", "20941", "4029", "3170", "12941", "5262", "14816", "16041", "2165", "9382", "25884", "17540", "13309", "5760", "16281", "11539", "21338", "16772", "1647", "17620", "1812", "18654", "6365", "22799", "24468", "14973", "4442", "5528", "16693", "24924", "22416", "25630", "25350", "9422", "20413", "2467", "20402", "5241", "19453", "2128", "16332", "12556", "18651", "4930", "10721", "15315", "9768", "19829", "22402", "20582", "4120", "8028", "7582", "5070", "8442", "15350", "22708", "14859", "20619", "21617", "24014", "22025", "9525", "720", "8335", "4907", "13952", "9149", "2941", "9482", "15045", "8552", "10142", "14821", "2167", "9091", "18763", "8065", "13966", "3786", "1953", "8899", "8435", "18413", "20304", "5060", "1660", "22441", "22183", "19806", "21890", "25927", "13793", "13244", "25682", "10464", "659", "20967", "6845", "16766", "19962", "8870", "21264", "598", "11177", "5717", "17103", "3342", "4104", "3610", "21447", "21049", "6392", "17682", "12909", "14999", "14931", "16534", "8128", "7570", "16705", "25040", "25498", "6582", "25381", "1512", "15663", "8807", "5739", "16727", "15137", "8826", "11369", "13512", "22235", "22694", "1270", "19734", "21517", "15199", "4002", "14153", "22801", "16924", "6486", "16758", "317", "19802", "2040", "17680", "7716", "3203", "17740", "16790", "18410", "689", "13778", "10791", "14348", "7053", "1868", "14012", "18808", "8759", "17151", "24031", "5590", "8419", "2612", "17472", "10124", "1001", "1558", "13815", "20266", "16550", "1226", "22881", "19341", "19477", "9827", "2618", "11862", "23284", "15942", "16251", "6202", "8907", "9301", "10632", "25575", "15797", "14133", "18067", "9517", "7831", "19882", "19387", "14745", "4691", "8670", "9839", "24181", "11645", "9172", "9494", "9339", "1880", "2999", "2151", "20706", "7765", "22494", "6087", "16167", "15136", "18691", "2390", "9143", "98", "20282", "7522", "4315", "12897", "18749", "13853", "14632", "13866", "13462", "19018", "9635", "8849", "3886", "19338", "15097", "14490", "5823", "1328", "23279", "1575", "3059", "1232", "22761", "244", "1230", "2090", "17009", "9056", "24767", "12063", "10133", "3279", "10301", "12262", "12407", "15249", "20624", "15808", "15520", "19379", "11755", "20136", "16097", "10687", "17658", "12099", "3953", "21543", "3713", "25555", "12988", "21979", "21529", "18955", "6504", "12868", "9400", "10630", "6416", "18073", "24575", "7512", "61", "15051", "25407", "5959", "22956", "8460", "1876", "15686", "3201", "8202", "9350", "5277", "15752", "8678", "6765", "2053", "21848", "5559", "20039", "23311", "11998", "4359", "8708", "6949", "15152", "17445", "22316", "372", "23632", "13150", "11601", "6951", "22948", "17533", "22910", "25935", "15411", "20536", "18344", "23799", "25471", "16794", "20885", "16620", "3651", "5491", "2996", "8029", "7674", "23043", "18768", "20937", "11738", "15145", "14857", "156", "24539", "1913", "19762", "19177", "12836", "20853", "20998", "19008", "17365", "14670", "2730", "23837", "6817", "21325", "960", "12192", "14860", "1255", "2466", "18438", "1920", "7558", "24811", "24624", "18483", "19692", "24865", "4639", "23327", "17975", "2858", "13532", "5626", "11924", "2342", "9330", "6218", "14372", "20242", "18078", "22491", "10224", "4006", "16033", "4989", "10673", "8043", "21451", "22645", "1058", "5596", "25744", "4300", "18836", "7203", "7343", "5675", "15129", "6838", "22849", "9630", "6098", "674", "19354", "18039", "11851", "9979", "8942", "17165", "2421", "13147", "3183", "19680", "10256", "23199", "8954", "25075", "24995", "12981", "17639", "7254", "12976", "22105", "12968", "16774", "12728", "15020", "13827", "259", "11175", "8603", "185", "22532", "340", "7084", "11284", "3540", "21701", "17406", "9023", "20535", "19316", "9681", "21780", "14841", "13016", "1012", "21876", "16211", "3053", "12947", "17074", "19863", "25961", "21329", "11301", "1437", "18103", "20555", "22472", "2946", "7424", "11054", "2651", "11008", "4021", "23115", "22846", "6370", "21646", "23771", "16277", "6111", "14432", "22307", "23558", "8464", "11739", "19664", "13632", "25501", "8081", "392", "25437", "18812", "2286", "24511", "7659", "4250", "23395", "6283", "2821", "13052", "7542", "3061", "20199", "23499", "14284", "1914", "6677", "23275", "1143", "23136", "16984", "2010", "21712", "18457", "23383", "1348", "21913", "589", "14067", "12830", "22306", "20656", "16832", "18055", "12520", "25805", "14459", "18101", "16852", "13146", "11139", "3135", "7917", "18568", "22733", "18313", "18045", "1315", "17664", "25003", "20929", "11074", "5052", "7969", "9907", "15681", "13797", "22221", "13888", "11154", "5203", "13747", "6118", "18415", "3701", "22833", "23772", "20690", "25715", "1879", "8408", "13405", "11090", "3515", "21036", "18883", "17927", "21123", "12264", "20237", "5210", "7176", "1285", "25648", "4082", "25730", "1826", "12538", "10070", "12555", "15257", "2046", "23938", "7411", "8812", "19329", "8557", "19281", "19529", "2915", "5891", "11119", "10477", "15000", "17208", "154", "16103", "17804", "12310", "16122", "8781", "6560", "19457", "7398", "10039", "19301", "5795", "3141", "10044", "4460", "12916", "24048", "3732", "4251", "14676", "16517", "18198", "18539", "5764", "14088", "21785", "19287", "14120", "5873", "25540", "8087", "6666", "14394", "1981", "126", "14882", "8358", "19887", "81", "14762", "16369", "29", "13538", "9234", "20337", "25177", "23969", "21532", "16504", "5240", "8197", "7252", "13443", "7412", "14434", "7531", "17980", "16004", "925", "5103", "9667", "4386", "8340", "15916", "21487", "18915", "1894", "10199", "15921", "6008", "1911", "5700", "23946", "16286", "8213", "4652", "77", "19312", "1362", "14470", "17398", "13629", "871", "1831", "4238", "25181", "5244", "1873", "18695", "22890", "3703", "25171", "2934", "11308", "18348", "22365", "11956", "10713", "630", "11394", "21127", "6211", "14274", "928", "1440", "17413", "18443", "14612", "13061", "12", "18952", "18615", "23541", "15008", "5367", "11204", "15662", "13520", "4650", "18510", "22095", "13732", "6553", "17091", "18767", "21078", "20741", "4409", "11671", "19073", "5728", "6662", "3410", "3452", "55", "6859", "10774", "2608", "23872", "21904", "685", "5458", "11015", "5194", "4957", "9458", "23274", "5973", "769", "16400", "13856", "21949", "20224", "3682", "826", "7767", "12949", "1918", "11842", "23288", "9856", "20530", "4954", "8809", "194", "18892", "5752", "13039", "20578", "1306", "19077", "6187", "8517", "23754", "13408", "2249", "25420", "3476", "9828", "10205", "1263", "18913", "6223", "19086", "21524", "8224", "4271", "2621", "3911", "808", "1699", "22339", "25197", "25209", "22992", "25291", "5043", "21951", "3759", "17211", "13886", "3159", "11843", "10448", "20803", "13438", "9512", "7378", "6526", "12727", "8822", "9092", "7739", "21754", "14929", "18706", "25521", "2792", "13949", "7017", "24941", "10416", "23564", "9492", "23391", "5088", "12607", "18128", "20006", "4165", "85", "19728", "25402", "19766", "20198", "22914", "24407", "9479", "20732", "6652", "18502", "535", "5488", "4117", "20243", "6414", "6769", "21610", "23352", "25427", "3011", "23919", "9820", "13780", "24717", "5663", "12514", "10107", "3591", "13066", "20241", "12674", "7136", "10788", "18703", "5783", "6396", "8587", "13193", "10217", "17941", "4895", "14420", "21661", "3625", "20991", "12204", "12791", "4187", "2900", "3101", "7090", "17308", "11291", "22560", "19099", "6404", "20589", "23239", "5964", "14312", "8682", "19486", "23624", "25727", "17991", "24636", "7747", "6611", "8736", "10063", "6086", "16215", "24041", "3356", "522", "23033", "17368", "6042", "17510", "15468", "14276", "2726", "11036", "6869", "4058", "5273", "159", "13850", "14616", "25056", "23766", "18610", "133", "15537", "2039", "14873", "8093", "16673", "23909", "9648", "21137", "1458", "319", "24113", "20751", "14310", "23206", "755", "12562", "17679", "19346", "19711", "15582", "21555", "13080", "13094", "1793", "13207", "24049", "17175", "21032", "7452", "17514", "14752", "9872", "4556", "18051", "10715", "15175", "2585", "8695", "11923", "4394", "852", "11272", "6999", "10703", "3979", "15972", "17996", "13275", "3891", "1756", "12763", "3116", "25763", "14134", "25321", "21594", "23489", "3565", "23836", "18368", "2244", "7986", "12844", "8098", "1724", "16501", "1075", "24827", "4493", "3695", "17607", "17021", "6911", "8182", "24593", "14684", "18193", "4464", "12163", "3865", "25254", "23186", "3675", "7691", "23261", "18346", "15061", "22097", "2797", "25190", "8727", "14619", "3081", "19858", "901", "13491", "21511", "1276", "15969", "7707", "10247", "11618", "6109", "1784", "947", "24793", "24096", "12504", "13045", "15862", "8437", "14980", "8451", "1927", "2450", "22811", "25949", "16809", "2095", "16656", "22868", "4077", "25438", "4047", "25638", "13141", "3470", "1903", "18404", "23737", "10572", "1322", "15151", "13342", "22469", "13195", "5306", "16587", "22602", "60", "9748", "24203", "8261", "22373", "24568", "7957", "11694", "24002", "15306", "21852", "4437", "23903", "13899", "2565", "14293", "11463", "22879", "17252", "13267", "18571", "12743", "21626", "20748", "19764", "14477", "5644", "14794", "25369", "10032", "14572", "24415", "703", "20461", "8550", "24273", "18636", "20871", "13390", "3478", "3505", "18366", "19016", "23022", "14815", "9505", "20316", "12095", "15838", "1211", "22721", "18974", "8501", "11627", "20521", "6475", "9148", "11018", "20880", "14282", "15588", "8640", "18887", "89", "16737", "12170", "21547", "14520", "24847", "983", "9890", "15115", "4744", "23828", "15086", "20456", "4779", "5917", "17115", "13164", "3067", "8852", "23011", "14598", "7693", "16844", "13110", "20178", "13755", "2717", "21566", "4402", "19752", "6948", "10364", "18917", "6678", "24747", "11999", "17918", "16160", "16194", "6362", "5853", "22510", "21895", "25842", "688", "18789", "2353", "4661", "11135", "13959", "3725", "17999", "11023", "2747", "19064", "9752", "24245", "17614", "8980", "16911", "5749", "17787", "13297", "165", "14328", "20284", "11665", "19440", "7523", "111", "16952", "13838", "1704", "21651", "25914", "5164", "20334", "19355", "24807", "21516", "956", "11335", "8703", "8730", "23626", "11662", "18688", "5021", "438", "17334", "14713", "25692", "24238", "4088", "22835", "8278", "1481", "14834", "14589", "25298", "4428", "2478", "2060", "23715", "24137", "3739", "15440", "815", "17693", "713", "23942", "506", "13976", "22579", "21500", "10601", "25646", "11669", "24834", "9162", "14093", "3198", "6229", "21270", "21441", "14438", "12323", "1439", "23974", "7434", "9466", "13727", "2826", "25709", "3994", "16722", "12892", "12935", "24892", "18049", "13305", "6834", "4135", "8080", "992", "1814", "23211", "2116", "9711", "23610", "14209", "2751", "7610", "2387", "12882", "22398", "188", "21861", "11", "22847", "1976", "1095", "5379", "11679", "13822", "15971", "8851", "18333", "8492", "21309", "25580", "13821", "21540", "3078", "5032", "7014", "10732", "8134", "7465", "2179", "23495", "19885", "9114", "23565", "19406", "14395", "23232", "15512", "7346", "436", "12603", "1520", "4926", "10773", "16078", "3243", "7864", "4771", "16328", "20675", "22356", "18061", "16374", "9460", "15658", "9521", "14211", "18645", "719", "21994", "23635", "16481", "3951", "5775", "6194", "11860", "21108", "10870", "13618", "12945", "22864", "15997", "22604", "18242", "3493", "19540", "5678", "5998", "5165", "1265", "25628", "7968", "22292", "16520", "19049", "6688", "16514", "7501", "21317", "5328", "16816", "6407", "17160", "19191", "20155", "25933", "4674", "24159", "25793", "15481", "17643", "13882", "24996", "15342", "25627", "11282", "20987", "19848", "23602", "22999", "19730", "22896", "4791", "25818", "19658", "15553", "25131", "23856", "4507", "11807", "13088", "11180", "12295", "21838", "23184", "1021", "14072", "9132", "16074", "1405", "17583", "16336", "12279", "1045", "16729", "13848", "13356", "22566", "17631", "9242", "5261", "24246", "4257", "268", "16935", "17041", "3075", "6126", "12476", "24467", "19815", "25690", "9969", "15718", "19823", "23554", "11508", "19720", "11490", "20878", "8070", "16087", "16761", "20965", "7547", "4806", "6295", "9074", "15462", "9294", "25721", "11049", "1298", "12670", "21675", "7825", "12944", "19859", "1318", "20058", "15782", "8988", "18627", "15901", "5966", "3707", "5617", "21727", "20189", "5471", "9931", "15734", "10495", "1674", "4918", "8995", "25005", "8679", "20779", "11413", "11327", "1093", "556", "14875", "8606", "20452", "752", "4476", "25203", "20234", "11349", "24496", "17326", "8436", "17147", "15361", "4421", "11150", "25701", "6634", "19921", "10370", "6491", "18997", "7580", "3079", "21907", "16306", "24279", "23738", "18712", "9971", "11082", "23514", "16908", "21130", "19754", "5188", "8239", "12940", "24419", "21494", "2899", "2931", "11648", "14702", "11095", "1465", "12410", "11163", "21589", "13064", "6556", "4158", "19994", "17459", "13596", "14104", "17440", "13189", "25617", "11538", "15124", "12633", "23420", "6510", "11446", "10627", "18495", "22256", "14268", "18202", "24275", "1192", "21244", "4768", "6055", "11554", "17401", "18727", "12672", "21124", "19548", "11615", "8632", "12548", "14946", "8089", "23557", "14731", "10932", "10892", "7788", "5318", "10918", "16317", "16493", "18924", "679", "20935", "15355", "10930", "19377", "7349", "9892", "21363", "25510", "3236", "13316", "4741", "24997", "11402", "7011", "7019", "8846", "11240", "11861", "16119", "265", "14515", "14496", "19019", "19771", "12359", "15770", "13087", "7615", "4583", "14040", "11146", "18326", "4706", "4976", "4943", "17671", "452", "11120", "350", "2321", "15816", "14978", "200", "3551", "4917", "23625", "897", "8621", "22963", "12913", "16829", "16044", "7651", "17207", "13412", "342", "352", "2171", "3793", "1153", "25023", "10631", "19941", "13410", "21483", "6199", "13621", "3767", "1324", "1428", "25296", "25215", "20953", "12679", "19314", "14082", "8568", "21349", "14915", "6346", "10989", "18040", "18820", "16670", "5583", "24531", "21055", "10537", "6134", "21956", "3491", "5489", "6277", "20469", "25582", "14387", "15158", "2987", "20100", "141", "11666", "90", "2123", "17307", "20764", "4210", "23335", "10531", "16890", "7342", "19069", "15867", "14587", "7318", "14279", "22917", "13564", "10117", "6495", "44", "11759", "22279", "1297", "24840", "24350", "17779", "1802", "16075", "1048", "6064", "20950", "16808", "2635", "14129", "12656", "15542", "23868", "23571", "25528", "16745", "11307", "5057", "15541", "10876", "13237", "10738", "12546", "2830", "13772", "22217", "6525", "6433", "17028", "12059", "9988", "8432", "25259", "19533", "8237", "5597", "21503", "17851", "4862", "5903", "13057", "2575", "13120", "17937", "1119", "8765", "14380", "13540", "19678", "3604", "24508", "8067", "13301", "10991", "12321", "13962", "17377", "10159", "17425", "20652", "7483", "1559", "23562", "23781", "20331", "19763", "88", "5224", "9783", "14163", "5857", "20216", "3027", "9584", "6066", "17993", "12984", "310", "6436", "18360", "24501", "16744", "10349", "4308", "13865", "20132", "9022", "5886", "10407", "1210", "3417", "24470", "24541", "25448", "18544", "21719", "21029", "15242", "9498", "24174", "10148", "18928", "6291", "25097", "15940", "24842", "9779", "8411", "2209", "24642", "5702", "6538", "17880", "4033", "17958", "17819", "4766", "20415", "2059", "7559", "5522", "6388", "1087", "11200", "10616", "7988", "9338", "7753", "9048", "25889", "19403", "11340", "7930", "13250", "840", "10048", "17729", "25366", "14482", "22238", "21246", "19795", "11383", "8739", "19319", "10873", "22775", "13252", "2048", "24637", "8105", "18902", "14362", "3559", "25069", "17888", "16027", "7118", "24818", "24324", "7382", "3383", "13550", "6798", "2645", "7620", "20914", "6331", "9308", "8227", "14397", "4072", "21705", "11678", "22452", "1939", "7031", "13314", "22945", "19846", "3985", "20515", "7372", "21565", "25146", "2928", "13607", "22769", "20963", "23359", "24427", "25621", "12990", "14502", "20339", "5558", "11042", "19448", "18561", "4591", "10784", "24073", "23328", "1022", "6131", "14439", "18908", "23984", "49", "25894", "9706", "25071", "20997", "11709", "2939", "10454", "23019", "15947", "5872", "23237", "14621", "13203", "23591", "11732", "22345", "11574", "1376", "24207", "1629", "8068", "1409", "1735", "18093", "25761", "5730", "17288", "8613", "11684", "9714", "14810", "7325", "3028", "8296", "15144", "5940", "19874", "20312", "17477", "12441", "21709", "15272", "14419", "14911", "11329", "16986", "4496", "19244", "16625", "9915", "6816", "16547", "16337", "9819", "20254", "24378", "22266", "18713", "20827", "2869", "19022", "16257", "6447", "14263", "10698", "24594", "3617", "17730", "2396", "13139", "21798", "8283", "2189", "23248", "24336", "3408", "5553", "12097", "16456", "20263", "14363", "24998", "1293", "11203", "17184", "25283", "24808", "7518", "25960", "23991", "14874", "17509", "20047", "14949", "6124", "5121", "22031", "16644", "6313", "4023", "2078", "25629", "17112", "14571", "862", "12157", "5420", "23985", "10483", "19491", "7935", "9341", "18217", "16038", "17822", "2170", "15305", "13754", "3885", "1335", "16341", "14466", "15443", "6225", "15996", "2270", "16247", "7964", "25520", "15631", "13347", "12404", "9017", "6369", "25159", "9312", "6982", "19550", "22338", "7407", "4898", "18378", "8902", "1830", "7082", "17650", "15077", "13418", "5417", "13930", "14325", "10259", "24389", "1805", "24488", "10165", "15437", "6035", "5581", "19914", "18111", "14838", "25568", "1611", "15643", "25266", "22255", "5111", "11126", "21553", "23526", "17487", "5929", "17364", "5223", "16674", "10249", "3572", "19739", "8845", "18905", "24664", "11097", "9859", "10532", "10456", "15599", "11589", "14553", "19855", "21206", "16428", "5874", "10994", "368", "21248", "4939", "944", "18545", "393", "13693", "2008", "1994", "22512", "2867", "21944", "17825", "9113", "18841", "23967", "3211", "1714", "16553", "22430", "20876", "10660", "14280", "18072", "22715", "13757", "12332", "12632", "10641", "16588", "22052", "15846", "14856", "2574", "3637", "23180", "2481", "6522", "5129", "7274", "2833", "16797", "21908", "5755", "9533", "22768", "24266", "3287", "10135", "24764", "19759", "24060", "7560", "24373", "4213", "19910", "19397", "20789", "7358", "19381", "14942", "12861", "17342", "16202", "12210", "21148", "22033", "6192", "25839", "564", "22246", "18875", "3599", "24299", "19317", "14979", "14603", "22932", "5658", "16714", "7614", "660", "19943", "9229", "11300", "8168", "14111", "7996", "17379", "8426", "21076", "14740", "2784", "8342", "6095", "7013", "24753", "2098", "8386", "14385", "23890", "22431", "16680", "13766", "16252", "6393", "2969", "20459", "1865", "25976", "25925", "9817", "24821", "8297", "13498", "11781", "8138", "21733", "18446", "24695", "21126", "19646", "18710", "839", "5771", "4456", "15633", "4454", "20468", "14699", "11852", "22436", "25194", "19408", "8382", "12953", "11295", "12946", "23627", "13684", "23050", "7265", "25754", "12148", "6341", "1500", "7475", "1739", "21422", "16581", "2893", "1176", "20449", "23763", "475", "14176", "8184", "3892", "2632", "9816", "17843", "5503", "2921", "9834", "11870", "3507", "6590", "14071", "20930", "16552", "12282", "5342", "5235", "4136", "7496", "23480", "6383", "12327", "682", "569", "631", "25311", "22531", "19105", "24653", "13355", "5051", "11932", "1076", "750", "4584", "2631", "13505", "19419", "16486", "22907", "777", "8722", "22458", "9575", "11727", "4571", "15501", "20697", "22091", "8422", "18461", "18123", "18225", "15451", "11710", "14288", "20117", "8438", "3961", "13454", "10383", "4026", "5648", "398", "1269", "482", "789", "6305", "1564", "23312", "4582", "4936", "9408", "16318", "1258", "22291", "8565", "24561", "8329", "19121", "17395", "1610", "5545", "10923", "12706", "9718", "17637", "22399", "19269", "2505", "15413", "8789", "19216", "24609", "6217", "2285", "25714", "3252", "17608", "16098", "7734", "12982", "20926", "3330", "24959", "5933", "6897", "21330", "1117", "787", "11810", "11207", "25049", "2472", "25504", "14460", "8835", "12494", "19497", "9137", "25280", "11544", "3373", "15668", "4059", "14451", "24748", "18435", "969", "19743", "9096", "24319", "20838", "14956", "21163", "18860", "24494", "9966", "18752", "19871", "14492", "16764", "10933", "4310", "8719", "18020", "17793", "3466", "7683", "4063", "24960", "18291", "14038", "3345", "25204", "2904", "12904", "13798", "42", "13102", "96", "6069", "8871", "23084", "8031", "12261", "20896", "1047", "13378", "23384", "10143", "1289", "122", "13676", "20153", "18132", "18981", "2141", "12034", "5961", "14318", "10813", "9412", "22891", "19598", "21976", "25622", "1326", "14830", "5549", "25523", "19693", "13282", "18700", "236", "21111", "23343", "23944", "25091", "10231", "16513", "2901", "13022", "25908", "9235", "20609", "3262", "2257", "17140", "21369", "18380", "9601", "7384", "24416", "1589", "24208", "8698", "21749", "19977", "16231", "4014", "8030", "21859", "21165", "17616", "5735", "14406", "4668", "12516", "11748", "12016", "15935", "23187", "15639", "17241", "20230", "12839", "18142", "16399", "5989", "11616", "9536", "9410", "1216", "1136", "19647", "9344", "8218", "11977", "15422", "21300", "5668", "16372", "21190", "14047", "17057", "2728", "12057", "9586", "9429", "13086", "12636", "11328", "25392", "2204", "9849", "6761", "12854", "15886", "8817", "7634", "15721", "2804", "19821", "6572", "9034", "14373", "8247", "6417", "5200", "7449", "9940", "10983", "12189", "16358", "11867", "25752", "16741", "25419", "13278", "7826", "5881", "2671", "11723", "3526", "10743", "21652", "13738", "22056", "13842", "2399", "8830", "1476", "17892", "504", "17977", "16209", "16810", "11633", "20026", "3627", "13895", "17276", "14697", "4609", "19600", "4479", "14885", "375", "3920", "20384", "20351", "16273", "5676", "1568", "24303", "5227", "8241", "25080", "15641", "4229", "11368", "21281", "23976", "11360", "17756", "2012", "8659", "2634", "18616", "9102", "13789", "16519", "14182", "23532", "2263", "5502", "21858", "6555", "12363", "23749", "13214", "700", "12703", "10146", "20068", "1801", "12807", "860", "23743", "24929", "11061", "5967", "13569", "24951", "5913", "9055", "12307", "22524", "23093", "5228", "9086", "7507", "3359", "15654", "11220", "2034", "6070", "16297", "276", "5552", "173", "896", "23345", "8847", "8362", "12296", "6335", "10183", "8674", "17380", "19909", "20475", "4872", "17579", "12482", "13407", "21265", "4439", "8127", "15975", "25631", "10518", "22526", "8937", "9736", "9850", "14493", "19258", "5950", "18813", "4448", "25904", "18865", "7025", "9572", "14478", "14813", "8324", "10269", "25152", "24858", "10004", "5855", "12685", "22101", "559", "6657", "12299", "20613", "2137", "16965", "22836", "6320", "24981", "14326", "1921", "15936", "5734", "12979", "8406", "7731", "16082", "7555", "282", "6273", "8186", "6411", "16934", "21520", "14757", "8114", "15907", "7311", "22970", "7223", "6864", "9145", "9006", "4330", "10410", "18932", "22885", "9902", "1217", "18150", "20647", "12776", "10728", "16282", "23092", "9016", "23257", "23085", "8447", "19116", "18492", "15205", "11512", "20715", "9682", "3340", "731", "8381", "10213", "4262", "18192", "7164", "18985", "9396", "3841", "20011", "23170", "21142", "11742", "20393", "11757", "16288", "1427", "6593", "18119", "14302", "17572", "18212", "13317", "24355", "19204", "11320", "18398", "3114", "1828", "18639", "24831", "25539", "2534", "7003", "13003", "3143", "15569", "18114", "18031", "22361", "9807", "4542", "1829", "18784", "19524", "19089", "15884", "13380", "8804", "13728", "22366", "5353", "25458", "11832", "19401", "22502", "8047", "24243", "12211", "863", "19606", "11238", "5807", "14431", "20785", "25058", "23636", "1019", "254", "13135", "23901", "2746", "3471", "19931", "20805", "12339", "6332", "17783", "24933", "16798", "15933", "20668", "1386", "21367", "23428", "11858", "22191", "20887", "23408", "1603", "20424", "8521", "25487", "1448", "3860", "8916", "11906", "4665", "18747", "4144", "10515", "3034", "12655", "24224", "3494", "7429", "19210", "18992", "164", "9695", "3091", "12240", "3854", "14210", "20182", "7802", "17087", "20617", "8664", "17600", "17251", "11337", "13960", "9094", "2045", "11038", "6849", "16424", "10710", "3289", "4081", "6827", "1577", "512", "729", "18416", "9662", "21522", "23980", "1719", "18977", "11951", "18991", "19212", "17623", "20054", "5649", "4801", "23717", "12382", "12961", "12021", "15649", "19385", "20640", "23848", "3272", "20660", "18986", "18127", "22197", "13867", "10964", "9248", "4372", "1173", "10609", "12866", "21042", "1978", "4354", "56", "22949", "10356", "15855", "12426", "575", "13649", "10835", "1848", "4959", "15066", "16054", "18250", "3628", "14748", "10621", "16692", "4876", "18439", "3479", "9082", "2360", "8859", "15777", "6185", "19159", "20359", "2489", "2507", "13546", "4537", "6024", "5986", "3910", "4364", "18817", "6573", "454", "20484", "12225", "23190", "14903", "1187", "21801", "16026", "18116", "20693", "14259", "23973", "2384", "9291", "23491", "19353", "17003", "13225", "8236", "4868", "5434", "22116", "4046", "12354", "12049", "21894", "8669", "10427", "6696", "20143", "16457", "4245", "16169", "21161", "16431", "25855", "4631", "8528", "13879", "732", "23497", "19333", "7628", "14202", "22277", "12713", "13229", "22925", "23694", "2233", "9206", "5310", "2414", "2926", "20121", "9142", "15496", "2001", "157", "6956", "20439", "9397", "16841", "590", "1717", "4723", "19352", "13112", "22887", "14725", "23372", "8223", "158", "2087", "18525", "1223", "4671", "24186", "9221", "11166", "18179", "13219", "12316", "9279", "10395", "12559", "7100", "11955", "8628", "2829", "13435", "13677", "16468", "16020", "14131", "16596", "4523", "4754", "14059", "12203", "1426", "24092", "16415", "17856", "21728", "16157", "11431", "23745", "18432", "766", "23724", "25334", "19359", "7870", "21521", "17059", "11983", "20995", "9663", "9715", "13419", "4132", "20376", "19238", "15121", "21965", "13724", "5499", "16139", "16753", "25970", "13659", "23432", "9287", "20936", "25804", "13983", "11686", "13604", "6054", "6458", "19973", "24711", "17357", "24296", "772", "8313", "1778", "25287", "14150", "11051", "25386", "17940", "19872", "20545", "21181", "16287", "3426", "19042", "10740", "4142", "22564", "1312", "11231", "9547", "3742", "13413", "1705", "7890", "6251", "8172", "17665", "25874", "2120", "5893", "20856", "6327", "15982", "16868", "13277", "24095", "19306", "23850", "18013", "18518", "5629", "22039", "5529", "17119", "17417", "19906", "13944", "14068", "3083", "2973", "2411", "16213", "5365", "14753", "11228", "320", "17766", "12122", "15841", "5211", "9010", "14693", "11918", "19989", "8260", "11857", "2066", "3442", "2494", "23323", "25172", "7117", "21514", "12684", "23881", "18280", "21468", "8735", "15865", "19787", "10860", "9432", "5879", "20666", "15788", "18171", "13389", "8760", "12681", "11079", "5199", "9573", "13296", "12664", "7445", "5304", "540", "6654", "14145", "20332", "11635", "12929", "12731", "14242", "9822", "11351", "15017", "9738", "9441", "13307", "17296", "19305", "568", "21612", "23660", "6401", "23121", "2544", "22988", "22955", "2038", "5334", "7279", "9425", "16343", "5162", "20958", "17336", "3519", "3063", "15314", "21504", "13567", "7848", "14425", "21815", "19547", "7002", "13916", "4221", "19290", "12348", "15832", "8991", "20667", "13823", "6316", "2216", "4273", "308", "13481", "22222", "11390", "25878", "2968", "3008", "1677", "19852", "16201", "8472", "25963", "19503", "16354", "10808", "14157", "4725", "20033", "24237", "19466", "12107", "7216", "16710", "7839", "5206", "1399", "15215", "5451", "5319", "15390", "18011", "17952", "11219", "15531", "857", "18302", "920", "23103", "15687", "18637", "3777", "8338", "14179", "1247", "7152", "26005", "3971", "11702", "13765", "411", "21970", "1882", "13200", "5884", "4620", "4735", "24164", "23113", "23545", "25819", "20317", "12829", "7782", "12851", "5655", "20883", "23460", "3273", "10444", "15858", "17545", "25361", "2183", "3671", "25572", "17767", "228", "23241", "13083", "15575", "19601", "16079", "274", "5023", "24017", "406", "6153", "5205", "18663", "13698", "7421", "2629", "13909", "9005", "23349", "13518", "2849", "15691", "4001", "2471", "16804", "21287", "25795", "12907", "14593", "4925", "4408", "9454", "21764", "6169", "24557", "4964", "1245", "7515", "1148", "8762", "17383", "14730", "19920", "18425", "5395", "15429", "4091", "14994", "11474", "4690", "5450", "13901", "7264", "821", "5887", "19578", "9421", "18166", "21992", "25037", "25067", "4721", "2950", "13062", "15436", "25802", "7214", "1272", "9188", "21642", "19929", "7817", "5074", "20623", "3044", "13911", "13779", "1127", "9658", "19106", "18934", "2245", "11997", "25601", "20733", "8623", "11758", "2287", "23175", "12255", "11942", "18899", "25968", "24804", "18259", "11505", "24659", "16363", "12110", "10263", "1617", "25689", "23354", "5493", "23185", "3914", "14692", "22647", "10806", "1844", "6755", "1541", "22788", "25832", "1067", "13737", "4109", "24952", "13276", "1129", "23502", "11762", "15859", "6304", "8061", "10610", "21766", "12005", "9657", "13182", "17409", "19850", "25944", "10617", "20294", "3543", "18338", "5015", "2359", "18310", "21249", "17734", "24220", "1671", "7426", "21207", "2815", "25543", "18867", "2622", "9797", "10501", "7775", "18062", "6633", "19496", "12123", "22417", "2910", "403", "1712", "2347", "15165", "9899", "24506", "10379", "17416", "2683", "10494", "13234", "18160", "5969", "11487", "9574", "25559", "7899", "2991", "17777", "17521", "12821", "7192", "25389", "16380", "16981", "14275", "8273", "11603", "7462", "6710", "21947", "9661", "15544", "11418", "17515", "24649", "16158", "15527", "7443", "7511", "8354", "21889", "4248", "14687", "6428", "22061", "14484", "1935", "12837", "19878", "19179", "21763", "16930", "14299", "25335", "23411", "22231", "18960", "20719", "9501", "18851", "19137", "2562", "2026", "1818", "14034", "15730", "18479", "14200", "11165", "15114", "3180", "17194", "11478", "17012", "24001", "64", "24317", "299", "10910", "6861", "13542", "8503", "1908", "25645", "4327", "8950", "21662", "15924", "10423", "8535", "22646", "8144", "25108", "16099", "125", "12828", "16365", "10764", "13387", "24760", "2377", "8710", "23404", "20490", "2268", "17976", "25722", "21960", "5719", "22164", "26026", "6183", "19868", "4602", "24590", "9930", "12956", "5102", "18725", "7894", "8175", "7596", "19982", "8006", "16618", "7844", "8697", "24383", "23826", "9147", "10765", "10841", "16068", "16899", "22867", "4999", "4788", "19255", "18954", "2499", "16085", "20792", "23230", "8877", "18589", "3963", "22265", "3626", "6446", "8895", "17163", "25310", "18069", "8259", "2335", "9313", "19328", "16847", "19095", "13287", "1845", "24663", "3820", "8717", "23884", "1345", "8316", "18412", "284", "19645", "1633", "6997", "2541", "18598", "22592", "4529", "15748", "3023", "2193", "15046", "12525", "18973", "10099", "20978", "24809", "16256", "23283", "10950", "13440", "15915", "7248", "17053", "13730", "18165", "16863", "12601", "2014", "11066", "23008", "19513", "20610", "17799", "14661", "9627", "11334", "21794", "18233", "25032", "18488", "8734", "14570", "6399", "20816", "5453", "4996", "2536", "20369", "676", "14121", "11812", "14498", "10882", "11125", "9607", "24292", "21891", "12634", "23262", "13748", "11057", "15104", "17831", "13855", "2092", "14890", "18453", "1941", "17099", "23256", "4195", "11073", "8819", "13973", "6539", "4035", "23000", "25088", "23948", "25024", "20868", "14044", "25491", "21526", "3513", "13049", "1146", "20695", "12487", "5970", "21542", "19363", "21010", "25242", "17995", "4230", "16436", "3064", "856", "4087", "5439", "24801", "16754", "5025", "14336", "19088", "19460", "4614", "12043", "9030", "6631", "5179", "23634", "11689", "14798", "10779", "12762", "5763", "24964", "24720", "16782", "17300", "6123", "11549", "59", "13986", "25444", "23691", "11833", "18005", "20463", "18278", "5531", "4902", "20251", "1635", "19549", "14197", "14916", "1741", "3040", "13116", "7796", "10049", "21954", "1457", "4246", "909", "20158", "1815", "9548", "1514", "7656", "22159", "8225", "6907", "1378", "18134", "4856", "10095", "12062", "24348", "1188", "22385", "12775", "23471", "11350", "10508", "734", "21398", "6663", "5186", "25151", "3138", "11105", "2787", "5821", "10394", "24221", "14366", "10234", "17863", "5975", "17063", "3894", "23047", "20912", "16666", "430", "21971", "5856", "18806", "6300", "24363", "16704", "7865", "8252", "11321", "2041", "7556", "24885", "6505", "4726", "17143", "9743", "15834", "7618", "17979", "21531", "19236", "414", "4848", "3475", "6843", "11653", "25951", "7288", "19096", "2237", "22670", "8654", "16593", "24382", "2153", "15535", "21969", "4309", "8363", "10352", "3824", "1614", "24315", "15785", "21812", "23073", "7676", "21061", "21440", "12281", "2100", "9280", "11152", "10306", "6960", "16446", "24991", "15671", "12764", "17974", "20235", "18480", "7999", "5476", "3928", "3952", "12787", "10218", "23157", "1943", "24392", "876", "1590", "18452", "19621", "23965", "21033", "1016", "4737", "4618", "15181", "14525", "14334", "25758", "7331", "16630", "23210", "22367", "21319", "8803", "3529", "7687", "3836", "12825", "23953", "14191", "16274", "988", "14350", "4015", "9079", "3948", "25072", "437", "6549", "3105", "2413", "13920", "15101", "9436", "25345", "10659", "15704", "20492", "18559", "5825", "7020", "3773", "1975", "23580", "17879", "25612", "9407", "20800", "8154", "6179", "9624", "23053", "12647", "13513", "21103", "11019", "10350", "20790", "6708", "4231", "12144", "22134", "10098", "1094", "15724", "12139", "22474", "20159", "15209", "17985", "14374", "18739", "3324", "22161", "22497", "4997", "25228", "16591", "5377", "18159", "24928", "14300", "25537", "20377", "17318", "2952", "1700", "8572", "15391", "25063", "14347", "10034", "11988", "17774", "11604", "6690", "25066", "13982", "22960", "23476", "13620", "10669", "20598", "18788", "9587", "22993", "6398", "5708", "2487", "7885", "21196", "15380", "12856", "18682", "5519", "10925", "23007", "5806", "4528", "16003", "18224", "16509", "9088", "14648", "7333", "18445", "19134", "25703", "18245", "22092", "4242", "16577", "12741", "22127", "8312", "3232", "4503", "6873", "12145", "18221", "2998", "19957", "10842", "17039", "17738", "21690", "22176", "19757", "22136", "19672", "19494", "10734", "2601", "24013", "10163", "634", "1733", "6502", "5870", "4824", "19000", "7686", "6513", "22162", "16012", "10442", "10921", "410", "20727", "10062", "15007", "9686", "11776", "8749", "18272", "19109", "16884", "25739", "13631", "22369", "11570", "11904", "17948", "509", "8176", "8397", "17613", "16912", "7071", "11113", "25830", "10761", "22470", "14869", "23767", "25479", "14754", "20275", "25398", "12154", "24833", "2175", "2463", "1144", "21808", "4973", "4659", "14732", "21388", "10403", "4775", "17443", "3176", "25207", "11933", "25817", "4994", "22752", "1900", "13044", "2264", "23996", "8609", "24824", "10904", "21267", "22203", "6988", "18375", "2297", "12288", "25033", "5397", "13626", "15739", "20905", "25915", "24943", "5093", "3964", "11692", "4305", "356", "6939", "4383", "21963", "6368", "23195", "10563", "8387", "20921", "7736", "23386", "7984", "20940", "5898", "20512", "7800", "13474", "2142", "24651", "14647", "15768", "17748", "7438", "4545", "21152", "23191", "8836", "17142", "16083", "767", "16006", "11543", "15951", "15883", "23858", "13158", "23414", "507", "7024", "8965", "4663", "11198", "16862", "23267", "20501", "19818", "3294", "7271", "12939", "15206", "6083", "3435", "23405", "10363", "18033", "14806", "3358", "12401", "14795", "20954", "14742", "4526", "17187", "11289", "23236", "11354", "19339", "13095", "15074", "17899", "20662", "10986", "494", "14850", "4466", "6749", "11628", "24525", "20067", "23388", "19690", "14996", "15252", "7562", "17423", "10421", "8139", "12054", "22131", "9414", "9027", "18251", "24197", "24516", "25785", "20511", "7454", "4284", "16232", "22394", "16939", "5642", "15385", "13814", "451", "17505", "22335", "5055", "9008", "472", "708", "15952", "11541", "16733", "4417", "8645", "17481", "8092", "21462", "19649", "22582", "7169", "3686", "1807", "24903", "12031", "23770", "7035", "7028", "13506", "1049", "16007", "21767", "20513", "8627", "19835", "2628", "23970", "1638", "7554", "4253", "22800", "23259", "22439", "17013", "9554", "19514", "2079", "19032", "4992", "5371", "25861", "13293", "9164", "3992", "21873", "3723", "23597", "13719", "8280", "13599", "6157", "18795", "8726", "22990", "24449", "21606", "15", "21835", "18956", "24051", "4215", "18799", "14430", "19233", "12871", "17743", "4613", "14551", "10929", "6544", "19778", "5718", "13889", "20038", "25787", "11705", "19145", "23649", "7871", "12857", "3407", "11257", "6209", "20650", "3127", "16134", "23339", "10789", "3782", "964", "20996", "17008", "2942", "1820", "19468", "22175", "14726", "24687", "15460", "10614", "20240", "5269", "11226", "8658", "20440", "18028", "1866", "22209", "21824", "16126", "21731", "18431", "4740", "341", "11704", "25279", "12180", "3648", "10753", "20022", "5498", "25570", "4893", "14319", "15362", "12046", "12878", "25480", "10620", "18191", "23651", "3700", "2354", "12151", "12219", "894", "5808", "14625", "3804", "18529", "6679", "20603", "22828", "17222", "5798", "2897", "22020", "13144", "7040", "25920", "19753", "19205", "10969", "8095", "17674", "422", "8511", "22218", "20986", "15021", "3404", "12086", "14852", "14990", "3293", "22461", "4317", "2591", "21260", "21068", "23950", "1411", "15990", "22905", "14712", "21235", "9538", "13706", "10589", "3321", "6214", "12875", "15517", "10720", "23016", "4443", "24839", "24354", "14579", "21040", "15922", "8681", "7923", "17314", "24016", "5513", "4718", "2650", "2914", "5155", "3986", "6030", "23596", "18949", "6720", "20367", "9083", "10544", "20319", "12079", "14256", "21717", "16541", "17709", "16945", "9015", "4434", "10771", "9590", "9450", "17293", "23260", "18570", "19433", "18948", "12746", "3093", "22082", "4739", "15364", "24323", "8597", "5745", "17896", "24722", "12734", "13530", "1681", "17731", "8657", "12757", "3414", "6119", "10192", "19657", "21322", "5110", "2234", "15319", "22340", "7932", "22621", "4565", "6286", "25230", "5631", "20988", "11553", "10727", "16272", "10103", "25275", "24157", "19603", "10227", "5448", "2877", "5034", "446", "702", "13163", "7134", "6580", "14908", "24387", "19133", "13608", "2438", "10277", "13271", "593", "5962", "18262", "5497", "21412", "8578", "24142", "324", "2121", "9276", "13665", "8264", "22628", "13286", "22897", "7089", "5949", "447", "6444", "22051", "11170", "14642", "1418", "8915", "18358", "18723", "23779", "3019", "7215", "5259", "9019", "19614", "16383", "14215", "15695", "6564", "17182", "225", "13731", "12037", "277", "18926", "763", "15646", "8066", "23929", "9922", "6019", "9634", "14097", "19995", "10850", "9182", "11703", "20984", "6847", "4767", "16622", "25150", "9833", "7653", "13155", "19980", "8720", "11075", "22295", "21654", "6519", "18016", "23576", "6164", "21239", "15622", "1323", "11827", "20703", "948", "3973", "12270", "21229", "16948", "19809", "11158", "461", "12232", "7245", "25912", "11085", "1857", "8229", "19873", "17989", "14950", "6252", "15404", "2148", "7794", "14649", "19036", "936", "15096", "22467", "5124", "18469", "18195", "22220", "21402", "18497", "4714", "10386", "6021", "16842", "3549", "22022", "22214", "5930", "8997", "18669", "5738", "19545", "5809", "12638", "5238", "3427", "4307", "12083", "6943", "21736", "14845", "17362", "1505", "18829", "8306", "3492", "19936", "1701", "14602", "8057", "20290", "15606", "21192", "4186", "17467", "19360", "17313", "7879", "12313", "21616", "22178", "18423", "12922", "15374", "3394", "10786", "12750", "22832", "4603", "25476", "11965", "20920", "21461", "12265", "20673", "22247", "2963", "793", "9359", "22756", "18296", "1788", "4986", "10717", "10462", "16542", "20911", "16711", "13709", "7351", "7239", "5252", "24606", "14965", "2258", "22503", "6420", "8473", "17681", "20700", "7236", "14109", "13461", "24723", "17939", "16065", "16370", "2922", "18449", "12682", "11141", "12579", "7237", "18394", "6478", "25590", "6028", "15360", "12649", "8887", "11155", "8162", "5811", "25644", "9615", "16871", "19993", "8450", "10393", "6303", "25522", "22629", "17269", "2851", "20270", "23685", "25395", "17687", "5943", "3936", "353", "15371", "24660", "21230", "18428", "22121", "24882", "21472", "12309", "2887", "22312", "10623", "23422", "22323", "878", "17092", "11316", "10202", "4220", "3457", "21974", "23756", "7842", "9347", "13138", "21350", "16938", "6689", "11064", "18085", "13323", "13574", "6284", "21067", "23027", "15366", "10567", "18818", "11519", "303", "24338", "17716", "4141", "8159", "7648", "11403", "14359", "14733", "591", "24563", "22103", "24330", "10492", "8788", "19234", "25009", "22415", "24287", "18537", "16641", "13338", "13366", "21881", "12470", "21570", "7711", "22184", "11381", "10648", "16769", "48", "19604", "17910", "18567", "25133", "13846", "8010", "14186", "16880", "18574", "22660", "19136", "7880", "5366", "2722", "2274", "14090", "5000", "12847", "8360", "7224", "2733", "17128", "15943", "9886", "16352", "12573", "15939", "23637", "4981", "10405", "10436", "11514", "8209", "12745", "18169", "1353", "17473", "24752", "9761", "8964", "823", "11594", "13215", "3144", "15562", "5105", "3642", "13691", "2295", "17489", "7911", "5426", "9026", "9552", "24846", "10585", "11050", "427", "8961", "9199", "2451", "20009", "18030", "3186", "9631", "23913", "8551", "5532", "7141", "17340", "16747", "24030", "10286", "9579", "24076", "14960", "1532", "10193", "9671", "554", "20179", "7503", "12859", "24158", "24217", "23052", "11460", "13178", "8867", "16094", "4770", "9523", "2415", "12273", "16668", "16652", "23138", "3162", "24613", "2381", "1940", "11193", "24938", "20746", "2624", "2522", "5309", "12919", "1780", "22693", "23718", "11197", "21978", "11233", "16873", "6125", "23870", "11290", "23945", "20427", "18194", "7545", "8523", "14453", "5815", "6361", "21358", "11356", "15725", "23072", "2780", "14797", "617", "19347", "23578", "3557", "11048", "17684", "12329", "12697", "15407", "16475", "21586", "14972", "17095", "23617", "12850", "6597", "24558", "7194", "13333", "15094", "15122", "10385", "86", "5398", "15487", "10291", "381", "9622", "8496", "22019", "5130", "22293", "32", "23997", "17556", "9678", "1789", "24871", "3334", "7625", "10664", "16820", "15632", "24053", "109", "5410", "4552", "25372", "15608", "15189", "11696", "24650", "23080", "16957", "8266", "5270", "18679", "22944", "21688", "19479", "12997", "25176", "18596", "20288", "20648", "15376", "21158", "13046", "1294", "24128", "3066", "1779", "9955", "21122", "11646", "4478", "10905", "10002", "22383", "5534", "20600", "17216", "14884", "17134", "935", "12560", "3017", "16417", "11353", "23740", "9310", "15950", "722", "5202", "13134", "18931", "25174", "43", "23787", "19999", "16320", "4797", "13295", "15365", "5910", "15274", "23692", "4320", "2427", "19161", "13312", "17848", "13773", "9184", "22664", "4183", "10943", "24992", "25965", "12650", "3416", "4840", "24670", "9946", "25892", "19927", "545", "12720", "9485", "1444", "6364", "4570", "14252", "11630", "3607", "11286", "22623", "22699", "9710", "19916", "5402", "25047", "24261", "412", "16567", "2701", "16408", "4869", "21293", "4750", "11001", "3904", "16812", "23900", "9153", "3822", "13099", "664", "14952", "3758", "1009", "11937", "13594", "8078", "18409", "778", "6686", "17123", "6129", "19699", "1209", "23546", "15294", "10891", "25255", "5976", "19071", "6248", "8848", "13299", "15090", "17105", "23714", "12450", "7710", "9018", "13393", "11004", "9882", "11728", "23662", "19261", "24433", "21115", "4678", "10517", "14170", "24579", "18678", "4396", "24312", "3469", "1977", "14538", "20527", "12089", "8294", "6360", "1278", "9223", "10521", "15836", "11920", "16227", "17504", "13733", "5824", "14264", "890", "880", "17965", "15172", "3781", "5787", "7895", "17704", "20081", "11167", "14404", "11137", "22804", "16451", "23268", "23677", "3746", "12486", "23416", "22412", "12185", "15381", "4099", "6980", "1027", "25574", "17363", "21508", "12781", "636", "9375", "17176", "22125", "23960", "19063", "13988", "2013", "5406", "18547", "11495", "25380", "6578", "24781", "17507", "10196", "20782", "23598", "1545", "17554", "458", "2948", "21747", "13516", "8559", "22032", "16933", "14923", "2325", "11341", "16344", "23911", "15185", "18214", "2431", "21019", "17520", "11921", "13395", "6212", "6733", "648", "23764", "17673", "15549", "14630", "14705", "7178", "16929", "20217", "6703", "18337", "15231", "15193", "4445", "21693", "3333", "17052", "17551", "21021", "11185", "21826", "18620", "10679", "11266", "24911", "13781", "19148", "20588", "2310", "7243", "8663", "4112", "22613", "9296", "13047", "17301", "19687", "5646", "114", "25490", "3704", "11922", "10191", "8860", "12630", "11593", "161", "8671", "4644", "15647", "14694", "12704", "13126", "5123", "9246", "17938", "24024", "571", "9947", "1727", "23667", "19668", "23127", "14704", "22281", "11304", "20612", "10003", "2475", "12413", "21505", "15775", "12833", "21696", "4375", "4577", "6734", "13079", "12881", "19223", "16382", "3069", "926", "11734", "24789", "19186", "14894", "14226", "24866", "4580", "4280", "3005", "25122", "25939", "1350", "22285", "25973", "9716", "17129", "20063", "19546", "145", "19217", "19014", "2332", "24596", "15026", "22614", "16296", "21782", "10399", "19501", "2600", "25967", "4796", "25149", "18475", "19495", "647", "17539", "17086", "18372", "7225", "17238", "7418", "22210", "15112", "14615", "21464", "4707", "5020", "24439", "16035", "11342", "21273", "20839", "13907", "9913", "23075", "19565", "1296", "13202", "13787", "2668", "3588", "2105", "13803", "18516", "2586", "1804", "19118", "3674", "23310", "25109", "1571", "19384", "24906", "7064", "2452", "19577", "12238", "19661", "22007", "996", "20752", "970", "16416", "5418", "4162", "21573", "3673", "15054", "1292", "5665", "13926", "2724", "7599", "18603", "2687", "21201", "7293", "11384", "21995", "23458", "21934", "11882", "570", "16261", "14354", "3443", "13406", "3307", "10830", "25052", "2839", "14053", "6052", "16558", "25224", "24455", "18470", "24274", "20451", "14378", "25102", "2937", "20421", "16623", "10382", "21578", "6588", "2800", "7896", "1945", "21678", "255", "22403", "23899", "21202", "4494", "10520", "24632", "7268", "20245", "2721", "675", "3619", "11917", "4020", "229", "1007", "20314", "20318", "14626", "24239", "797", "15288", "18036", "346", "14402", "2630", "13176", "20691", "20414", "3978", "4010", "18794", "22908", "17930", "3814", "344", "4746", "11118", "19562", "5766", "1161", "24042", "17535", "21845", "19908", "25782", "17537", "11325", "11234", "3461", "19812", "19031", "14773", "22632", "3606", "2029", "16651", "13857", "18485", "4353", "6197", "4019", "18034", "10417", "4852", "6865", "14175", "19050", "23706", "9921", "15720", "23028", "14768", "7263", "12200", "24377", "17210", "639", "8104", "4222", "24393", "18845", "3842", "15182", "16016", "5091", "8931", "17723", "22483", "20322", "24011", "12593", "2303", "25685", "9500", "7951", "12572", "2802", "24631", "5612", "8052", "16091", "12218", "9700", "1432", "17565", "25757", "16691", "25862", "13247", "3432", "4194", "24411", "20382", "25649", "22481", "4107", "6073", "19663", "18764", "22054", "2069", "18653", "3212", "2769", "17415", "11953", "6648", "2547", "16316", "5038", "18509", "11881", "2292", "3646", "18199", "24900", "6356", "12707", "25675", "24328", "19808", "25094", "19617", "7294", "3185", "14811", "14924", "20672", "13641", "16112", "23917", "11581", "2", "424", "18826", "17418", "4617", "15919", "13729", "3922", "15766", "23076", "22267", "10093", "16323", "14148", "5799", "16678", "105", "10629", "8531", "10018", "7981", "8978", "2296", "1057", "7538", "2985", "23449", "7202", "15619", "12193", "23524", "3013", "8409", "21104", "22555", "19376", "15267", "17370", "17486", "15769", "25618", "25972", "1387", "14564", "21392", "8834", "19523", "2135", "14185", "19362", "9806", "15505", "11964", "9829", "12285", "20528", "17719", "7563", "5233", "1393", "22363", "9874", "24447", "21761", "25070", "15835", "11834", "23923", "790", "16393", "13089", "25609", "24555", "24759", "20943", "10856", "24581", "14253", "18576", "13817", "9513", "10968", "25022", "662", "23606", "18907", "5589", "10083", "9958", "23750", "7282", "1166", "6739", "5619", "25653", "1300", "15677", "15412", "22156", "8667", "7798", "9770", "8372", "19190", "18334", "25010", "14499", "3129", "12972", "14605", "71", "3844", "25529", "3221", "12932", "20519", "14655", "25576", "15078", "22027", "25941", "21711", "3419", "7114", "10716", "3999", "5846", "4805", "12201", "19940", "13521", "24183", "6381", "3153", "17454", "6561", "18400", "9539", "15237", "19721", "5037", "22202", "14736", "4608", "24977", "23650", "7907", "20471", "11330", "13945", "20972", "15879", "7110", "22208", "15574", "13905", "11298", "961", "3811", "20596", "8666", "14027", "3055", "11765", "7422", "5607", "23042", "22037", "1366", "11217", "24805", "10754", "25926", "3631", "10885", "24854", "21591", "22059", "7380", "10880", "3264", "5415", "11915", "3538", "10432", "15449", "17261", "15434", "24605", "1549", "8497", "23834", "5995", "17626", "24172", "4206", "17944", "16683", "23012", "2556", "7833", "17714", "13619", "9616", "19417", "16635", "4346", "12680", "726", "21366", "1618", "19028", "4922", "3225", "134", "19845", "5653", "25240", "12933", "5819", "11993", "16115", "8540", "12951", "10357", "23390", "1498", "14037", "9139", "18168", "11894", "20959", "25118", "8359", "6137", "20999", "16249", "8317", "6470", "12819", "12160", "18429", "19280", "3935", "19415", "21323", "24898", "16723", "25113", "7751", "24972", "19504", "11435", "4942", "24072", "20209", "16819", "1909", "6140", "3843", "12253", "11952", "17642", "12092", "5711", "22573", "14936", "19041", "15477", "15019", "11967", "14546", "25414", "10927", "13579", "4235", "5550", "18780", "19172", "10913", "23104", "6074", "25558", "15763", "22245", "13501", "11916", "9851", "4431", "12834", "8795", "2461", "9951", "21365", "11591", "3798", "4774", "23415", "5561", "7073", "10533", "13589", "4481", "17953", "24614", "6162", "12662", "1404", "2739", "25456", "892", "20300", "17495", "20130", "11322", "8757", "9122", "17482", "7405", "192", "14035", "14789", "12343", "17834", "16410", "13759", "5586", "4237", "23867", "12722", "15961", "17392", "21473", "4334", "12767", "16402", "21649", "6931", "8076", "17808", "24966", "24812", "4298", "874", "21020", "5993", "13679", "3520", "25877", "7561", "6374", "24796", "6107", "20945", "19171", "23318", "16968", "3056", "15570", "365", "19811", "10509", "22668", "4951", "22569", "18197", "4736", "10547", "8595", "12171", "16576", "5204", "13682", "15192", "15849", "13630", "10408", "21418", "2569", "9363", "9342", "21784", "22014", "8600", "7990", "19357", "20780", "16019", "16118", "25809", "9983", "373", "3446", "16032", "16712", "14409", "21250", "25415", "18487", "25781", "705", "14113", "22449", "25527", "24548", "6336", "16800", "10602", "11803", "17795", "1167", "2228", "13306", "15850", "14119", "22435", "18298", "3072", "16687", "14007", "25838", "11868", "23995", "2791", "21326", "15134", "514", "5002", "25248", "3923", "9060", "25293", "11313", "22468", "12795", "8475", "4489", "24899", "23640", "21013", "2690", "5059", "25078", "3267", "5429", "14646", "21946", "6515", "9953", "360", "15399", "9445", "18177", "10897", "21533", "2744", "19742", "10556", "25370", "22177", "19266", "5380", "21105", "5222", "17970", "5769", "22556", "223", "4304", "4934", "19772", "23235", "25347", "326", "16185", "18164", "300", "23472", "9731", "21787", "25950", "4828", "7859", "19696", "11664", "1023", "337", "20811", "16219", "18063", "22496", "19964", "22017", "17950", "12780", "11509", "16430", "19181", "15298", "13227", "25828", "3942", "23481", "19315", "24300", "24706", "20218", "15784", "1487", "25312", "16864", "1843", "3255", "24891", "11306", "15332", "13339", "20671", "15708", "25336", "22673", "5349", "3838", "706", "15103", "16133", "19896", "40", "7039", "5225", "14143", "18244", "23681", "10680", "21292", "20781", "6439", "16937", "5667", "12234", "16049", "11254", "26010", "730", "22964", "24160", "21383", "11866", "20644", "25103", "25430", "21022", "16177", "17045", "16014", "3784", "13992", "3380", "9003", "20111", "23846", "20560", "5326", "15609", "12101", "25002", "4385", "5731", "1682", "19487", "6293", "4773", "3026", "16239", "5768", "5065", "4611", "16904", "5637", "16792", "23509", "13434", "3532", "4910", "6903", "6264", "15285", "25105", "6796", "17942", "12091", "230", "2382", "3012", "2420", "7694", "2513", "21285", "24262", "1105", "4031", "23202", "24022", "3753", "20349", "1792", "24178", "21240", "14769", "17895", "19139", "26008", "7813", "7185", "11192", "18828", "20232", "17497", "24983", "3338", "22850", "9388", "12420", "8558", "2190", "14436", "20281", "19952", "13549", "6250", "24205", "7576", "17146", "12254", "6347", "6384", "933", "16175", "1419", "9117", "1408", "25652", "25273", "14036", "4716", "6653", "10144", "9669", "16228", "5723", "21575", "24191", "11224", "12263", "15551", "18708", "16308", "17275", "3900", "13006", "13283", "16114", "10512", "11189", "18624", "6716", "9507", "22358", "10415", "8967", "8025", "3166", "16967", "1064", "19733", "2972", "18519", "12626", "24718", "23508", "15485", "20762", "18526", "25917", "23081", "4825", "14080", "6952", "25220", "12181", "5176", "5850", "22830", "25359", "14123", "24401", "18226", "4146", "19463", "9480", "10678", "9909", "23048", "19769", "2132", "14401", "6155", "9129", "25794", "20787", "1934", "124", "21027", "15825", "14254", "22003", "11425", "16978", "23658", "10726", "7038", "17728", "6933", "21026", "22476", "2831", "18205", "25478", "3517", "635", "5861", "23250", "11162", "10220", "7804", "24255", "3570", "20394", "20028", "22745", "17677", "20107", "13480", "1325", "3663", "11985", "22109", "10877", "20447", "8033", "19774", "180", "9919", "3169", "25860", "2075", "20702", "10089", "2184", "5041", "19860", "10511", "1937", "20253", "4790", "1003", "2639", "13651", "10681", "6170", "37", "22691", "4438", "3638", "25966", "2207", "16959", "23338", "1150", "7484", "17506", "9674", "19189", "20820", "6000", "6713", "5438", "1080", "24570", "13359", "20030", "25729", "17865", "15875", "25901", "4680", "14188", "18782", "1642", "24602", "25216", "1688", "24810", "17628", "1561", "13137", "10315", "14567", "3738", "20488", "9014", "261", "658", "10008", "11612", "17225", "19119", "14651", "7260", "348", "25798", "10675", "19783", "12128", "20435", "23309", "24668", "13073", "19981", "24887", "7135", "2346", "15157", "22170", "11775", "11429", "13915", "1875", "11929", "6852", "7190", "14826", "5264", "14786", "19485", "15461", "24503", "20473", "10412", "7581", "17591", "518", "18621", "11241", "21590", "17345", "4798", "24872", "17638", "18746", "2951", "18440", "5022", "21496", "8958", "1189", "10079", "2456", "15618", "23025", "24984", "21000", "22953", "20258", "9553", "23683", "9844", "7893", "21488", "12931", "22001", "6253", "25896", "14447", "15120", "10038", "11909", "21436", "4727", "12805", "1450", "409", "14074", "17610", "4050", "15032", "16739", "9307", "17945", "4182", "1314", "23783", "18345", "951", "8190", "8898", "18870", "14750", "17042", "23474", "22155", "12231", "7037", "9581", "3076", "15624", "17338", "25195", "13756", "15741", "6468", "1588", "9974", "3680", "10445", "8763", "23270", "15576", "6632", "9106", "4214", "19629", "5465", "11963", "17088", "18369", "20045", "11230", "1560", "19498", "3530", "22936", "17593", "25227", "14592", "11372", "13802", "5901", "328", "8088", "15656", "7919", "13493", "1531", "2501", "8869", "1301", "1477", "13633", "18384", "10579", "18857", "2510", "10157", "678", "14365", "490", "17997", "13041", "18406", "19102", "18690", "20524", "83", "359", "8982", "13488", "11690", "4127", "22548", "5215", "12331", "6518", "19587", "23723", "18849", "3828", "25029", "952", "7943", "12874", "3719", "10783", "22113", "6928", "3863", "7820", "24497", "6467", "19857", "11005", "23123", "3246", "24436", "9656", "18491", "14565", "18810", "25089", "17836", "12511", "22653", "23844", "2727", "9", "7891", "1621", "24056", "9222", "11101", "12248", "15132", "13914", "12196", "11428", "20877", "4748", "3305", "1266", "1028", "10475", "6756", "5931", "12155", "24126", "23778", "17860", "25017", "20327", "9174", "22912", "19101", "21214", "12116", "263", "18083", "9057", "17567", "21869", "15476", "4819", "5008", "6628", "5344", "9463", "24577", "11823", "26015", "24953", "21864", "9087", "1341", "9439", "1854", "12267", "3749", "23908", "24774", "11532", "25348", "25657", "7068", "17891", "3495", "1864", "11044", "16571", "8787", "24923", "418", "6459", "9263", "27", "13452", "18556", "23271", "2096", "20035", "24803", "2493", "8959", "19371", "18916", "18793", "3288", "3190", "6354", "4903", "8355", "12027", "16998", "11800", "12044", "18741", "19781", "23895", "15987", "25301", "182", "4749", "24489", "7873", "8238", "7868", "6143", "3692", "9299", "20019", "20409", "16583", "15745", "4517", "4911", "17564", "11213", "21834", "22204", "22803", "15694", "10296", "25789", "12503", "6998", "19075", "7915", "14069", "22741", "23926", "24576", "979", "16398", "16384", "9923", "2794", "503", "13969", "2643", "20017", "23876", "19567", "13460", "19880", "6506", "25477", "19793", "21668", "17771", "5096", "17465", "18644", "15395", "3393", "10132", "16206", "12024", "23364", "22575", "698", "24069", "14919", "7888", "10686", "10569", "7468", "24376", "985", "4", "8265", "21083", "5737", "6473", "4670", "9085", "4281", "21827", "10090", "7326", "19370", "12436", "24716", "12872", "14232", "2361", "14872", "9996", "5229", "12531", "1170", "25436", "15999", "7470", "6451", "22139", "3164", "644", "24837", "9373", "24986", "10347", "13473", "21269", "13539", "9419", "8486", "6846", "8747", "23070", "16942", "23425", "18283", "22047", "7000", "21312", "15083", "2492", "25030", "5283", "24600", "8350", "1749", "1504", "16531", "24337", "16697", "6075", "18548", "24216", "11267", "24485", "1517", "3351", "19067", "10672", "17889", "9487", "19215", "14501", "2277", "3959", "22729", "14164", "1070", "15236", "19213", "6171", "16314", "1140", "9111", "1962", "21295", "7668", "19966", "20113", "24595", "17817", "21768", "20120", "11398", "6571", "13854", "16325", "13129", "23741", "24492", "20118", "2958", "24288", "13831", "15037", "20174", "25226", "20561", "5994", "10831", "18356", "24601", "10906", "24326", "24697", "21453", "11895", "15597", "15552", "19830", "17341", "8326", "6622", "2101", "17224", "20892", "9496", "18593", "14139", "5622", "15211", "16104", "20983", "24116", "21008", "7905", "13816", "16477", "23094", "22278", "5672", "4380", "1834", "4905", "24194", "13656", "3229", "16806", "7566", "13775", "18667", "845", "15885", "23548", "18963", "7675", "15071", "16180", "16385", "24994", "2521", "3748", "24223", "13661", "22675", "20581", "5171", "15710", "604", "1709", "21290", "24507", "16053", "5568", "21102", "6180", "15534", "6741", "22348", "4601", "14291", "16958", "15179", "14939", "8214", "12256", "16135", "20712", "4126", "2545", "24634", "17124", "8328", "3875", "5560", "6413", "8491", "9455", "8439", "10053", "737", "18115", "10793", "3353", "950", "2318", "19833", "11297", "19168", "5585", "10057", "11010", "10060", "12771", "24112", "7308", "489", "7369", "7985", "5437", "2462", "2588", "4919", "22325", "11962", "9488", "19579", "6993", "2205", "15511", "25670", "17619", "24399", "16537", "21318", "3366", "24201", "20757", "9691", "3497", "4512", "23196", "17587", "10960", "15955", "10262", "6618", "20426", "15814", "9754", "18779", "15682", "6577", "2362", "19775", "1286", "2023", "5", "12284", "13398", "10114", "13302", "3805", "18330", "17909", "539", "21762", "7230", "4283", "6983", "3275", "17108", "4811", "21957", "22099", "24771", "12660", "17078", "5527", "22659", "20655", "23690", "20832", "20298", "14435", "6687", "11786", "6991", "7386", "16310", "25993", "12206", "5733", "3903", "14805", "5830", "8073", "23761", "1142", "19749", "18287", "22516", "21769", "22290", "11455", "7474", "18595", "19196", "8526", "2416", "17038", "5327", "17046", "3117", "10465", "15664", "5115", "10834", "20203", "16598", "8344", "13562", "5875", "2957", "8713", "4818", "828", "6967", "10011", "12611", "18520", "22", "8054", "22244", "11845", "2592", "5530", "17782", "21854", "9343", "15034", "2977", "23238", "4522", "4724", "20021", "7447", "20350", "749", "6626", "24680", "18238", "19435", "22251", "25995", "22128", "3352", "24946", "471", "16505", "22986", "13777", "21886", "23962", "24093", "25307", "4979", "7158", "18785", "17455", "10137", "16474", "6058", "13014", "1711", "2848", "6296", "12715", "237", "3405", "16743", "16010", "3787", "12484", "9468", "21199", "24195", "7220", "11838", "6484", "24495", "15611", "21066", "4686", "735", "15979", "17691", "21990", "14765", "24432", "25193", "4728", "12008", "10595", "16902", "17753", "18939", "6548", "15398", "9428", "6853", "9105", "6094", "23044", "22411", "5544", "8369", "7841", "14376", "18581", "18402", "1619", "11258", "21603", "1606", "13938", "25678", "7435", "25686", "5674", "16502", "15330", "15561", "9504", "486", "1946", "4687", "22654", "8732", "23197", "1703", "2765", "6656", "7166", "25289", "11194", "16327", "8564", "20139", "19219", "20626", "16259", "8165", "5382", "6426", "8957", "18609", "20685", "669", "16268", "22663", "13466", "20025", "20320", "380", "8639", "17432", "13611", "20286", "23234", "20105", "15194", "13605", "23825", "13573", "19945", "10810", "23964", "18143", "25101", "24295", "14180", "7119", "20629", "18930", "16720", "24783", "5145", "552", "19559", "13571", "23539", "11760", "18243", "24127", "17353", "16389", "5315", "6966", "18898", "7051", "19729", "4667", "19530", "7026", "16582", "18124", "24505", "17024", "14663", "9719", "11275", "25873", "22853", "25984", "25980", "2796", "15344", "7530", "2311", "19674", "16356", "23555", "17973", "260", "972", "6003", "7829", "23642", "7671", "14497", "14367", "8210", "2138", "5638", "18145", "20163", "2783", "17735", "24294", "14863", "8032", "25712", "22779", "18057", "18522", "19998", "22213", "1024", "25188", "50", "4389", "12512", "1486", "9470", "25316", "23700", "4269", "6315", "4991", "21811", "6754", "13556", "25745", "1783", "5580", "23357", "2719", "1637", "15057", "19918", "9469", "21251", "2161", "3683", "13253", "25125", "2383", "10667", "8160", "4657", "10979", "15043", "22712", "24828", "2775", "100", "10346", "11062", "21996", "24945", "7869", "19335", "1645", "358", "18110", "13581", "21209", "14381", "10253", "5564", "11083", "219", "12911", "21237", "7975", "7234", "14102", "19591", "8255", "8287", "16846", "15013", "1430", "12808", "18405", "25623", "16153", "10872", "19898", "13990", "23789", "17954", "17048", "17555", "4662", "16217", "22565", "19630", "13279", "9809", "22236", "21396", "15011", "22840", "18684", "12860", "22258", "9430", "12495", "11432", "6456", "24352", "22622", "24676", "572", "19252", "22455", "20098", "23163", "11098", "23595", "3234", "21941", "20374", "5064", "3447", "3832", "23461", "1569", "14791", "14518", "3858", "10693", "18074", "18714", "1004", "25225", "2333", "20310", "12708", "20509", "18399", "20568", "20221", "1998", "10821", "16818", "7784", "8332", "22882", "25812", "10287", "542", "12894", "13547", "22751", "11547", "13206", "16653", "20086", "5195", "9871", "22858", "15003", "25605", "14622", "672", "14556", "5705", "12581", "23699", "11763", "22475", "9901", "3251", "10268", "22607", "5082", "6027", "5605", "16472", "6788", "24654", "20736", "12362", "6667", "680", "12308", "20875", "4367", "10503", "21233", "18459", "12386", "8351", "7624", "23109", "5952", "23510", "5554", "18237", "9846", "16009", "6160", "1121", "1172", "5699", "25261", "22697", "20102", "10886", "3620", "17232", "23454", "22802", "6992", "23135", "19337", "19924", "10051", "19518", "25161", "14098", "3498", "16313", "1248", "9207", "11799", "17172", "12469", "11058", "2140", "5640", "25391", "18628", "11637", "20343", "17235", "13098", "17695", "2960", "2870", "1837", "16263", "24423", "19996", "12432", "4527", "15465", "12445", "9609", "25349", "12068", "20148", "11501", "16757", "4780", "9645", "11886", "19992", "15794", "15651", "21926", "1612", "8187", "6682", "5168", "21622", "16462", "23977", "2861", "14051", "17447", "9042", "24802", "415", "4093", "17926", "10322", "10215", "8123", "21699", "25332", "12686", "1040", "2261", "465", "425", "5713", "24152", "18266", "11656", "1809", "13426", "23061", "16423", "21341", "25911", "12801", "22962", "7344", "868", "11736", "5486", "18196", "434", "24146", "14227", "10498", "15655", "20097", "19459", "15750", "8228", "23403", "14559", "13925", "2469", "3528", "6797", "22140", "6257", "22334", "18427", "23290", "645", "24253", "15679", "7476", "4548", "312", "19746", "25641", "6259", "5332", "17570", "519", "3004", "1290", "3990", "20686", "15621", "14843", "4078", "12986", "14677", "12608", "8077", "2139", "8199", "10745", "473", "3451", "15891", "23788", "12921", "949", "8121", "120", "1029", "8510", "884", "4256", "14099", "10587", "5338", "19827", "22844", "21911", "14591", "23316", "7095", "9228", "13269", "9191", "7433", "5751", "8539", "20074", "21289", "8590", "20680", "1885", "11625", "3776", "9959", "20854", "9929", "4851", "24937", "14342", "3982", "12380", "6429", "24920", "5421", "17234", "10611", "10887", "11801", "22488", "17890", "21245", "20709", "13392", "17007", "16805", "262", "24749", "16196", "22926", "16738", "11643", "19741", "9250", "8723", "1910", "4232", "9443", "16512", "23611", "15852", "12983", "11500", "5684", "3271", "17914", "4783", "20657", "7123", "8463", "5785", "13898", "14132", "23603", "500", "8468", "3077", "8319", "13881", "25863", "19240", "16572", "21849", "4042", "4890", "16994", "8945", "19117", "21537", "19447", "1034", "16602", "15230", "3731", "14968", "3762", "12490", "7636", "4949", "24165", "8513", "20172", "24674", "18681", "1483", "18869", "3840", "14606", "8281", "9838", "21219", "2397", "18961", "18001", "9152", "21178", "1851", "10984", "7381", "18109", "809", "632", "9818", "25210", "7699", "34", "2157", "26030", "7980", "9243", "25403", "10248", "4390", "2185", "24480", "15761", "17089", "9835", "24313", "3263", "13326", "18601", "14926", "18353", "5312", "3601", "13019", "23519", "2064", "18736", "19493", "24692", "11399", "8295", "4793", "7922", "14548", "23441", "2389", "13565", "4510", "4929", "10919", "11026", "20500", "5920", "13009", "3171", "14825", "8712", "22440", "8580", "20739", "24835", "6748", "11261", "4209", "222", "19162", "12134", "18758", "24957", "830", "2394", "10335", "17849", "10178", "16425", "1691", "6729", "14675", "5922", "20145", "16445", "9076", "19325", "391", "7623", "8562", "1507", "9353", "25895", "24639", "22710", "2448", "1770", "9911", "17526", "13465", "24587", "8596", "23071", "18623", "3362", "25413", "12770", "24862", "12108", "5407", "18888", "25640", "1954", "102", "8256", "2763", "24404", "7207", "18848", "6905", "13384", "25626", "18269", "11396", "2685", "14243", "5403", "4789", "12291", "1524", "19046", "15584", "19214", "25409", "24251", "7533", "3364", "20807", "11103", "9652", "9281", "17546", "1592", "1930", "23698", "5578", "12065", "14718", "8311", "10568", "14445", "22558", "14561", "5024", "21799", "25260", "25338", "7942", "2776", "2603", "23255", "15957", "22747", "21795", "11777", "24125", "24583", "20406", "20134", "11423", "13300", "16108", "17794", "5231", "3182", "10252", "1321", "24850", "6146", "10078", "20592", "14545", "17683", "18524", "25475", "24965", "12987", "14004", "22322", "1157", "11793", "14933", "6497", "9420", "8748", "14064", "10467", "370", "11107", "3303", "25264", "9214", "4159", "3354", "16589", "22504", "5721", "3350", "15444", "11472", "12164", "15806", "16362", "25126", "21301", "11528", "8638", "11910", "2698", "7961", "2779", "1307", "25871", "17933", "19253", "9621", "12930", "14610", "9478", "21952", "4054", "21096", "3112", "24926", "19471", "23777", "20831", "10575", "4863", "9471", "18697", "9138", "6670", "8876", "21892", "5673", "7718", "12899", "8085", "15450", "8808", "14212", "11171", "18301", "15781", "15636", "23308", "66", "24132", "25461", "7080", "3418", "668", "9744", "22942", "23317", "13711", "10123", "624", "9012", "14842", "93", "6668", "14330", "977", "15989", "21742", "4843", "1893", "23897", "13422", "3976", "9978", "17201", "6584", "4325", "16260", "25165", "17137", "25082", "19285", "6174", "16244", "14031", "11695", "16659", "5703", "11084", "20976", "14512", "9380", "4403", "15524", "22098", "10566", "2308", "9257", "17311", "13382", "18554", "6339", "20204", "24502", "24879", "21793", "18658", "22749", "23478", "16584", "812", "3506", "24910", "23216", "20855", "7131", "4395", "1182", "24936", "13020", "16548", "10425", "9591", "3428", "7850", "13054", "670", "11253", "3231", "11847", "16189", "3308", "15488", "17051", "13169", "21836", "16174", "5710", "18864", "15333", "17297", "15423", "13932", "432", "11174", "7029", "1359", "20919", "8096", "2154", "9867", "19157", "12726", "20489", "16130", "19451", "17055", "22705", "12112", "24269", "22967", "22790", "18563", "17228", "16987", "17002", "14947", "23213", "8986", "15015", "19427", "2324", "9232", "16499", "16749", "17538", "10221", "12619", "3281", "592", "7862", "6714", "25650", "9756", "14426", "9659", "14075", "2980", "3751", "738", "19243", "15965", "19239", "8200", "3035", "5446", "8100", "24979", "2155", "8308", "17391", "2127", "9453", "21872", "517", "2032", "23220", "613", "7805", "17788", "23702", "11182", "9213", "4048", "15119", "15607", "11614", "16660", "14984", "16052", "9050", "21009", "1663", "580", "5657", "25922", "15111", "24787", "1360", "2226", "24285", "7010", "24171", "6494", "11222", "10127", "5615", "17604", "14246", "15688", "19202", "13636", "5630", "6189", "15822", "4518", "14653", "6824", "22611", "22120", "11027", "7809", "12936", "22514", "19174", "24629", "19797", "4121", "25426", "20076", "9692", "10447", "26007", "8662", "19464", "7441", "17026", "16170", "5253", "25879", "12442", "3645", "1139", "20280", "4969", "3687", "17254", "6691", "204", "4546", "15353", "4598", "22012", "3269", "18579", "8367", "9354", "21476", "2159", "3503", "16359", "2107", "20565", "15653", "18468", "16885", "9069", "4151", "15820", "21411", "5139", "16478", "19160", "4684", "2884", "11768", "3848", "16357", "10639", "2770", "15358", "7630", "2967", "4492", "16017", "13056", "9297", "9073", "13769", "5999", "16963", "20264", "4710", "22772", "3775", "22959", "19901", "4971", "5652", "8799", "17992", "8694", "11974", "10591", "9311", "13663", "14887", "21751", "16152", "25308", "17577", "19484", "3581", "9739", "2819", "21147", "15498", "1850", "11269", "8404", "12959", "5132", "5384", "19528", "2231", "15275", "16144", "5481", "4857", "17218", "4877", "18042", "24832", "15162", "5547", "24117", "707", "754", "7469", "21821", "20144", "1252", "22820", "1948", "916", "696", "15500", "17451", "2009", "18549", "12258", "25710", "22888", "2422", "13222", "5435", "21695", "7181", "14441", "17433", "14295", "8017", "1497", "23633", "8142", "5276", "18444", "22000", "20520", "7045", "17617", "13078", "24784", "6935", "11030", "23014", "7866", "11907", "18481", "22219", "17069", "10652", "23179", "20077", "20743", "15911", "17065", "15558", "15256", "9447", "10441", "24193", "8573", "11401", "78", "25600", "14532", "2555", "12388", "16351", "1790", "6005", "10242", "2275", "7689", "23041", "21132", "14966", "8167", "14309", "15672", "14644", "1456", "2133", "24655", "11848", "11711", "18705", "20874", "1088", "13940", "20226", "2572", "23920", "14427", "758", "22786", "10138", "24161", "1158", "14303", "19780", "21467", "3376", "1906", "23609", "12392", "6831", "4290", "9528", "1878", "24291", "2670", "2405", "8019", "16716", "15092", "13070", "10264", "1916", "17350", "22913", "8802", "13192", "8740", "13197", "16326", "11433", "20050", "14002", "6715", "10267", "11994", "10707", "4692", "10847", "11481", "17170", "1055", "931", "16950", "9745", "7070", "279", "10657", "16269", "17931", "530", "10013", "1708", "5009", "14943", "4342", "23623", "16435", "15585", "5953", "23645", "4358", "8920", "6906", "16600", "5742", "23448", "15800", "4363", "24961", "16279", "5236", "23810", "17233", "17821", "24542", "17792", "22595", "7335", "8972", "405", "17359", "12673", "14487", "289", "19208", "8293", "12963", "25604", "5362", "16522", "9722", "23026", "5185", "13400", "22717", "941", "21114", "20925", "213", "11794", "23433", "11752", "18052", "19483", "24027", "7889", "11995", "18361", "24514", "6371", "20188", "1186", "22725", "12337", "14335", "21620", "824", "22901", "19703", "8758", "22965", "24283", "22104", "23089", "9266", "18286", "9832", "15164", "14945", "8467", "10270", "13325", "768", "17503", "8549", "6779", "557", "13805", "6737", "7681", "19222", "6061", "21014", "11779", "20220", "4519", "14346", "10243", "7619", "3539", "5016", "18500", "14194", "8117", "12584", "11675", "6567", "23615", "7150", "8064", "10082", "19198", "3778", "8026", "7317", "3043", "12124", "18318", "15675", "2729", "21671", "22998", "21885", "18489", "6195", "1381", "2964", "5272", "12852", "9994", "10894", "16839", "8641", "18958", "19686", "7048", "19278", "13849", "21057", "16464", "8413", "7851", "15715", "2692", "4429", "12733", "9381", "16613", "12599", "21241", "19881", "13318", "19911", "22884", "3696", "3283", "25725", "14229", "17847", "2737", "17299", "12458", "19748", "9268", "1853", "2944", "4321", "14510", "4482", "11156", "16955", "9696", "8861", "22726", "23477", "12848", "12030", "25614", "25822", "2571", "9064", "13935", "16508", "18692", "1932", "10261", "3728", "5977", "23340", "24773", "15225", "12648", "11935", "22227", "17320", "8148", "21045", "14422", "3984", "19983", "24260", "3094", "5077", "4550", "11989", "3893", "841", "8675", "23431", "14103", "2483", "21665", "6233", "6084", "2898", "12790", "480", "6890", "7979", "16926", "11096", "4760", "584", "3574", "7238", "10061", "23296", "15545", "16025", "19594", "24088", "4538", "4400", "22519", "24290", "10974", "12730", "24278", "25382", "4387", "3317", "21950", "13294", "8684", "17449", "1375", "21637", "12328", "20993", "8635", "2880", "3927", "3071", "12427", "21587", "4738", "14620", "6574", "13415", "25412", "19609", "21050", "19257", "24973", "13989", "2604", "13007", "25179", "7892", "19608", "1485", "7015", "19561", "8443", "25708", "18281", "3226", "18957", "7404", "3609", "306", "20699", "6050", "7440", "18634", "15118", "1083", "25519", "25913", "16304", "23600", "13042", "5218", "14183", "4000", "22834", "21499", "20659", "3374", "21809", "7551", "13504", "13544", "14909", "21378", "13870", "25375", "2031", "12803", "9550", "17434", "9080", "6215", "15036", "10041", "15418", "13994", "18230", "19425", "1435", "20020", "3057", "15069", "20542", "13320", "12513", "14514", "19632", "6076", "23663", "3322", "526", "20605", "16575", "529", "3372", "22508", "18", "11891", "12694", "7012", "6113", "20274", "4155", "16258", "4594", "24371", "12344", "21035", "10694", "9910", "9059", "20769", "3444", "8612", "15773", "14652", "8971", "17568", "25441", "2518", "15903", "14421", "6802", "14448", "4314", "10411", "4274", "21056", "22972", "4834", "21048", "21604", "5376", "8906", "10479", "6299", "3400", "8766", "2855", "24018", "11872", "859", "3702", "7684", "21759", "16524", "22704", "15697", "12902", "21967", "2104", "25139", "2878", "8484", "17508", "19895", "20448", "4559", "7987", "25247", "11183", "21288", "4488", "13770", "4673", "12325", "1175", "3365", "24643", "4531", "17553", "12677", "21710", "20784", "2373", "1238", "7586", "11017", "16983", "21912", "23494", "9555", "5477", "13118", "505", "19409", "9623", "18900", "16664", "22561", "24772", "1445", "22252", "6166", "18632", "7954", "9031", "18465", "14400", "21424", "7722", "12954", "9877", "13635", "19310", "23462", "12475", "14049", "19556", "22423", "16042", "8143", "19725", "22550", "11972", "19864", "24361", "7926", "15340", "22355", "7336", "23492", "9198", "4498", "807", "4730", "343", "21459", "22841", "2366", "407", "17882", "14657", "24344", "5817", "14456", "23062", "18990", "2093", "2044", "21072", "24556", "10944", "4853", "22070", "23455", "6408", "13991", "7283", "24848", "1786", "2653", "1402", "24075", "5323", "19232", "25820", "10317", "7286", "12928", "14914", "25124", "12917", "7006", "13425", "4871", "1858", "7756", "24422", "23203", "16621", "24572", "17967", "24591", "9847", "16516", "2036", "19260", "18501", "17144", "6340", "18454", "17636", "21356", "13646", "10222", "20397", "10300", "1877", "21755", "17830", "7654", "15755", "962", "20616", "10174", "13383", "19886", "842", "11751", "8956", "3931", "13104", "16823", "25997", "2883", "22028", "24956", "377", "25747", "5401", "18998", "9728", "8785", "12582", "9365", "4878", "21177", "16667", "5548", "18650", "4052", "14013", "1201", "22380", "12178", "10161", "2745", "11753", "14568", "24873", "12409", "19249", "12385", "19292", "25239", "8084", "3907", "23453", "4452", "5101", "19225", "5740", "23866", "19058", "6774", "13833", "22337", "16569", "4426", "13360", "13658", "24731", "25421", "8103", "4956", "15912", "25783", "24932", "20714", "25660", "4207", "19254", "13334", "12377", "13109", "22798", "20211", "651", "16300", "25469", "5690", "18293", "8304", "22409", "4477", "6418", "16164", "5650", "24486", "20858", "6642", "25738", "11225", "21327", "6091", "1195", "9416", "18271", "2344", "17470", "15684", "586", "24199", "9928", "19307", "9401", "9474", "497", "14091", "4815", "25465", "21655", "8471", "8254", "10409", "24454", "16633", "1282", "16713", "5965", "20160", "10491", "12289", "5230", "1973", "11056", "7357", "14356", "6266", "5265", "24545", "7856", "23034", "7088", "6405", "17104", "7408", "18772", "13133", "24180", "3654", "12276", "10524", "20112", "22205", "2820", "7491", "9379", "7108", "759", "1407", "25330", "22985", "990", "2227", "5151", "13486", "11729", "22661", "440", "19017", "13941", "18532", "11943", "6471", "25234", "13788", "1447", "20557", "23098", "15509", "22327", "13074", "13470", "6554", "2061", "8650", "8618", "11462", "17493", "23370", "11034", "2712", "16686", "4927", "4574", "6238", "16411", "15592", "5896", "21613", "2538", "15341", "10636", "6722", "15992", "22783", "20125", "19030", "22810", "18467", "15958", "10021", "14710", "12867", "19006", "1302", "3595", "6981", "128", "5826", "19199", "24089", "14241", "16528", "4480", "23886", "17373", "16058", "4017", "13230", "4635", "22413", "329", "7413", "21243", "18874", "4376", "567", "21515", "4759", "13430", "20110", "16057", "10030", "3050", "17832", "21903", "24029", "4564", "22759", "4382", "21469", "21311", "18947", "9173", "15454", "215", "448", "21788", "4646", "16876", "21640", "12465", "4860", "20517", "22018", "21334", "2102", "6039", "23533", "14195", "5140", "4579", "19323", "11826", "6738", "23313", "18573", "10596", "6940", "12943", "640", "16034", "20193", "4708", "25318", "5045", "5582", "3501", "4113", "3944", "23337", "10307", "9707", "20864", "20734", "3583", "16566", "4148", "6744", "17040", "15554", "19344", "20969", "6013", "879", "23733", "24136", "9862", "1654", "14122", "4361", "8194", "9406", "24000", "19443", "8509", "11407", "23442", "14341", "7345", "9675", "7174", "959", "25073", "21052", "7929", "3871", "8136", "8772", "1240", "23228", "17494", "13416", "10045", "15827", "855", "1723", "532", "10214", "20649", "11492", "1821", "23594", "11450", "19610", "24762", "21257", "24779", "17907", "16543", "12018", "8263", "6306", "16110", "10042", "4455", "8494", "9159", "23959", "5858", "2962", "21150", "23139", "9987", "7810", "22785", "14878", "12471", "24012", "9116", "15698", "3119", "25616", "11041", "3360", "1706", "17901", "21394", "17841", "12415", "203", "4567", "14301", "22809", "7309", "13515", "6557", "11531", "227", "5743", "7799", "17405", "16225", "11245", "18144", "4950", "5351", "14686", "24442", "21242", "23731", "3898", "8914", "23058", "17699", "23002", "8269", "17249", "2250", "16700", "12290", "24533", "19303", "21615", "4745", "16627", "10320", "12127", "24430", "19919", "24420", "7493", "20372", "16549", "6825", "24066", "19707", "17237", "25696", "1552", "10265", "25905", "21554", "5221", "2368", "24645", "14205", "15235", "14475", "25835", "20510", "12925", "7993", "22618", "1551", "18890", "15714", "16045", "9541", "20483", "1965", "21631", "20722", "12087", "25467", "13015", "6235", "21139", "13519", "15251", "13363", "17120", "13157", "22938", "12724", "3835", "1987", "1890", "21058", "17034", "15110", "17854", "1753", "24103", "11053", "11395", "3770", "2817", "1317", "12437", "25199", "20169", "16642", "16412", "2906", "21141", "21231", "23795", "10184", "1244", "18742", "21804", "4838", "10067", "18671", "15787", "24025", "9841", "6945", "4233", "11237", "12642", "11439", "14839", "8091", "19639", "13035", "3149", "25567", "942", "11385", "21549", "22372", "12448", "13541", "361", "18010", "3047", "2875", "17014", "21980", "14716", "8776", "8827", "13322", "12687", "13860", "22599", "12515", "376", "1899", "7955", "24434", "17448", "14249", "23711", "1721", "16426", "5537", "2918", "7622", "2929", "14473", "19912", "16859", "11506", "1648", "12692", "439", "17582", "24357", "4391", "4966", "21060", "15956", "17245", "8203", "23242", "21857", "1499", "8461", "5144", "10100", "7781", "17966", "14285", "20957", "17386", "7792", "25008", "65", "9676", "10040", "1513", "1955", "17998", "3142", "11641", "2442", "8322", "10033", "17263", "16969", "9795", "18106", "4055", "8428", "4357", "10586", "2911", "15354", "8911", "5504", "7148", "2017", "7528", "15196", "12187", "22851", "23174", "623", "20419", "7167", "4073", "10047", "4275", "4622", "15044", "13368", "6989", "5593", "13741", "23348", "9328", "19640", "14782", "22626", "4292", "14290", "19411", "2990", "3173", "10795", "5839", "23324", "22391", "6697", "10816", "21074", "7847", "21397", "16292", "19976", "2947", "6403", "10633", "23059", "2398", "16142", "18666", "8620", "5900", "21204", "20279", "740", "108", "19263", "17531", "4129", "11159", "1053", "7138", "23342", "9066", "20152", "19926", "1889", "25780", "25256", "3618", "3584", "8268", "7154", "15823", "281", "7083", "11248", "4154", "6136", "20669", "23299", "2498", "5577", "25516", "24293", "1528", "24967", "16488", "4458", "17859", "18996", "9167", "9519", "22842", "16636", "12530", "1827", "10598", "1218", "5250", "25662", "12849", "18670", "22207", "8336", "3261", "14932", "6844", "21720", "18382", "12004", "11223", "21760", "7189", "14539", "1382", "5182", "1537", "22138", "3552", "26011", "8334", "1746", "10484", "21314", "189", "5296", "5844", "22165", "6018", "11583", "22289", "8725", "8520", "2860", "10121", "18804", "5263", "3462", "18397", "12017", "7592", "25870", "5983", "5368", "8427", "13218", "16226", "12275", "9613", "11605", "13735", "24122", "9398", "6031", "24306", "21585", "5392", "12179", "3484", "207", "8242", "2260", "23644", "21713", "25268", "12985", "11218", "25214", "17955", "22666", "12900", "3847", "18950", "5325", "24798", "3154", "6606", "331", "17920", "21216", "6889", "15929", "10343", "2682", "5911", "167", "3098", "14086", "20910", "3873", "12627", "21548", "7785", "22425", "3785", "20595", "26031", "16923", "2542", "22437", "3677", "24008", "25243", "19112", "12364", "2143", "20180", "25826", "17761", "23205", "6705", "6964", "25093", "2741", "22045", "23056", "20682", "22446", "3899", "23223", "14823", "12277", "4337", "2895", "21919", "21302", "12615", "1960", "22122", "4980", "15447", "4451", "16299", "8692", "22744", "18096", "7550", "21479", "16367", "13902", "7069", "17640", "3335", "25934", "13354", "2428", "19590", "10497", "495", "13666", "6422", "17062", "3276", "12855", "21916", "7414", "2149", "2539", "3535", "9101", "24544", "11482", "2314", "20608", "24652", "20573", "524", "811", "5829", "21697", "24044", "17624", "10881", "485", "16949", "2365", "17908", "8270", "25615", "25654", "25019", "14958", "6516", "7257", "6287", "9813", "14634", "20687", "19313", "11650", "24226", "3669", "1902", "4924", "7472", "6307", "13479", "304", "6645", "23503", "16909", "4341", "7552", "1037", "21182", "20315", "12887", "17663", "16675", "16387", "24163", "24763", "4406", "24863", "13071", "19029", "5556", "8704", "335", "2445", "23883", "3958", "25497", "25562", "2453", "10452", "9873", "4352", "788", "3991", "20486", "14601", "25756", "4490", "16143", "2587", "3160", "10914", "23748", "1658", "9650", "20928", "917", "17968", "6699", "2047", "4695", "25671", "16647", "18909", "5440", "18294", "20333", "23293", "4809", "13937", "4318", "7330", "15637", "5843", "14910", "11069", "22126", "22259", "19574", "8514", "16702", "19814", "1205", "20879", "16927", "17371", "17131", "20330", "11486", "13115", "25620", "155", "80", "7565", "5187", "4515", "22636", "2993", "16444", "6014", "20546", "8575", "3233", "6337", "15321", "21428", "502", "23812", "1295", "7211", "11488", "14032", "17325", "15322", "13776", "22814", "20239", "16338", "25932", "1518", "4177", "2970", "5822", "11912", "6871", "544", "2126", "15602", "25633", "7832", "13001", "14667", "2323", "3165", "21771", "24554", "21574", "12604", "25846", "16013", "13908", "3791", "18066", "387", "15994", "12609", "9863", "6438", "18323", "6819", "3536", "6453", "7337", "15456", "13107", "8348", "15711", "10273", "20793", "20213", "23443", "14368", "19505", "5474", "15178", "14868", "19765", "19274", "25385", "19651", "25557", "13832", "10661", "18441", "15270", "13105", "7875", "11281", "22750", "21095", "24515", "9364", "23426", "11562", "25292", "15116", "1431", "13964", "10204", "10953", "10207", "13160", "12695", "20411", "25560", "1649", "7251", "12666", "18388", "1556", "22597", "7156", "17824", "17156", "10358", "4576", "15789", "1434", "13784", "11510", "9348", "16626", "2164", "14390", "6535", "25705", "13458", "20689", "24890", "3665", "23593", "25401", "11772", "23873", "7159", "17947", "18300", "11172", "7933", "15960", "367", "13686", "9924", "24792", "5546", "1133", "19667", "15279", "18664", "11754", "18569", "8476", "24853", "6712", "15258", "4140", "5992", "22672", "19502", "19273", "25343", "20601", "7397", "5918", "16650", "3329", "783", "17081", "22740", "24604", "12725", "17283", "18880", "484", "2068", "4782", "14476", "12149", "16224", "957", "18999", "6784", "13590", "17435", "8027", "8900", "12333", "6767", "8048", "10883", "18184", "5085", "13809", "16887", "13791", "17315", "1086", "1470", "20366", "25213", "4263", "2199", "20299", "14461", "17309", "9518", "16854", "16840", "11624", "21676", "4316", "19986", "19324", "23742", "19458", "9275", "15966", "9472", "5280", "4557", "21399", "1106", "24703", "14833", "15419", "13111", "9177", "17177", "12622", "2073", "15854", "2917", "14928", "22848", "24528", "10900", "20166", "13242", "15188", "5122", "8015", "6152", "21156", "16837", "25222", "22130", "15174", "15573", "14033", "23668", "10977", "9825", "757", "201", "20955", "3605", "6294", "21714", "18507", "8489", "12340", "2246", "20506", "16538", "23331", "9998", "14100", "17442", "19441", "22620", "9992", "12220", "20952", "17707", "19047", "20041", "20523", "23584", "12994", "19718", "17720", "16061", "1794", "11804", "16145", "18304", "9961", "9058", "582", "1370", "11577", "1796", "4800", "11899", "5482", "11144", "4025", "7823", "2978", "16624", "5761", "22700", "1838", "3430", "14332", "8407", "7978", "23091", "22920", "18065", "1044", "4920", "7631", "7370", "5939", "4837", "14808", "21816", "13340", "9399", "12778", "6175", "6037", "18966", "4034", "5724", "16427", "12100", "2801", "5157", "3876", "23120", "4370", "14029", "25845", "4938", "7546", "5924", "24020", "14020", "10203", "15108", "6768", "13485", "5923", "18188", "12053", "152", "13704", "13820", "975", "22062", "10730", "25770", "7904", "23362", "15572", "9290", "24535", "25355", "13613", "15503", "24043", "7494", "19076", "2782", "25552", "3803", "2825", "25718", "24855", "30", "13551", "8930", "20348", "3721", "17010", "1554", "19142", "15780", "12176", "15617", "22557", "3962", "21977", "3934", "17519", "1904", "13861", "3378", "17178", "24756", "1986", "5433", "13170", "24004", "18787", "20917", "3388", "836", "14871", "5579", "20031", "19286", "11332", "20942", "9563", "25517", "7027", "3853", "18396", "19153", "14898", "15397", "8881", "614", "7506", "9151", "16056", "14413", "11607", "22053", "15522", "24564", "14776", "19737", "11317", "151", "15244", "17646", "1938", "13559", "20802", "5587", "10446", "11567", "468", "10321", "13670", "25767", "7526", "20846", "4540", "16645", "14443", "11288", "16021", "14862", "19884", "816", "13388", "25229", "3558", "11161", "21862", "14507", "22078", "17943", "15163", "17569", "6135", "13880", "8051", "17403", "13714", "10990", "3025", "19247", "22196", "16954", "5628", "24980", "17188", "1825", "9772", "19907", "12614", "15195", "2520", "25044", "14407", "21433", "16355", "6727", "2959", "21277", "15052", "12305", "2458", "3877", "25731", "2614", "22624", "12583", "22817", "5092", "3420", "22760", "23988", "17255", "24864", "943", "20724", "307", "2896", "16770", "22587", "3755", "22386", "19084", "5520", "6760", "21888", "3698", "12905", "23894", "9300", "21271", "11947", "11128", "23287", "4079", "11818", "19203", "1630", "11602", "751", "1342", "2678", "18173", "4722", "23321", "22410", "25305", "2684", "12773", "4166", "16270", "20487", "16555", "9251", "10450", "18891", "25084", "6193", "17132", "12948", "24257", "15473", "24974", "69", "4335", "22826", "12205", "1676", "7613", "76", "20091", "5791", "11853", "2979", "25324", "19978", "6916", "3774", "24087", "4861", "24305", "13903", "313", "8505", "3764", "18463", "12637", "17512", "7934", "16255", "12550", "23982", "1069", "11745", "5794", "19648", "23214", "5287", "2696", "3693", "16116", "2626", "14370", "15085", "8307", "5570", "12228", "10749", "15863", "9317", "21776", "22940", "3379", "25341", "25134", "19625", "25379", "18273", "17655", "24190", "13451", "6353", "21452", "14141", "19115", "19356", "7078", "12076", "533", "10646", "13280", "18235", "21645", "24751", "20075", "11740", "25843", "17534", "22791", "18260", "24912", "21429", "9508", "25077", "15147", "22137", "6345", "13554", "20037", "844", "1839", "7165", "12888", "2408", "23173", "1670", "6322", "17805", "7066", "11430", "6029", "8221", "3744", "5027", "25112", "20661", "7147", "19539", "23394", "5576", "11568", "10966", "23888", "22041", "4882", "5347", "16979", "16901", "5838", "2657", "12197", "2913", "6603", "18717", "24571", "22924", "13013", "14778", "546", "23380", "2457", "1107", "19638", "9757", "9013", "3641", "17248", "2262", "21927", "21611", "8738", "4931", "11707", "20071", "3302", "4589", "5982", "6771", "23177", "15383", "10668", "23601", "4147", "17438", "4045", "22727", "558", "1905", "22568", "19026", "14880", "7966", "10414", "14011", "8469", "1985", "5209", "6348", "1171", "18362", "4191", "22781", "17427", "13796", "9934", "1337", "11502", "1824", "3060", "1467", "22709", "4569", "10688", "16392", "11210", "19899", "10638", "9668", "7994", "1484", "22489", "17689", "5331", "20092", "21370", "1942", "9484", "7712", "3912", "17400", "13093", "16860", "7761", "19758", "9043", "3247", "21991", "18131", "25416", "3038", "17633", "51", "22982", "19923", "3128", "19113", "25388", "15932", "15405", "16422", "5894", "17446", "17304", "15856", "20717", "13329", "24799", "20336", "20808", "8592", "24367", "22758", "22774", "7415", "24954", "17155", "13127", "10466", "4450", "10389", "22457", "2982", "25556", "18392", "16293", "19388", "10360", "10280", "10077", "8074", "11764", "23561", "14883", "4377", "10763", "7828", "23638", "8912", "12974", "8483", "8584", "22542", "978", "12616", "21431", "15905", "131", "11255", "8155", "1262", "20798", "15389", "13468", "22951", "17000", "3827", "19702", "3966", "14057", "25697", "13535", "13236", "21047", "8783", "6204", "10481", "22684", "18130", "15753", "4593", "15363", "14010", "9532", "17037", "3516", "9865", "23794", "11566", "18455", "13265", "4634", "18696", "10156", "19807", "4471", "2222", "6006", "2523", "6047", "5311", "13545", "9219", "10928", "101", "5226", "19027", "10430", "22480", "13436", "22923", "4145", "18807", "1667", "22315", "5249", "6619", "19436", "23987", "3450", "834", "2028", "999", "19470", "17962", "6700", "10825", "5340", "22433", "10736", "10345", "25505", "15909", "10731", "1339", "16480", "24580", "16525", "15219", "2425", "22377", "1454", "6493", "10186", "15141", "7887", "22188", "14137", "22605", "22918", "8522", "1179", "2374", "13864", "18048", "14479", "15821", "5404", "21308", "12194", "15878", "21910", "8180", "14001", "17502", "22226", "22563", "2713", "19404", "19628", "12564", "14248", "5469", "15022", "20938", "20308", "21218", "7432", "3941", "17659", "1104", "6067", "3883", "19695", "8880", "15507", "12701", "20622", "24019", "10284", "3084", "20774", "2514", "17644", "3125", "20485", "21023", "2736", "2283", "13274", "1907", "6221", "8179", "5387", "21417", "22701", "23504", "20070", "921", "17893", "456", "6575", "16222", "24135", "2173", "3082", "10292", "12335", "2840", "19537", "6888", "21832", "11677", "22381", "3623", "13863", "24971", "3608", "1200", "15427", "2871", "15869", "20392", "22376", "22657", "18088", "7557", "22473", "1397", "7137", "20767", "6623", "21935", "20567", "10840", "15372", "8642", "20554", "7816", "2938", "6968", "585", "23971", "25217", "1214", "18026", "9377", "7041", "20024", "20817", "21071", "23935", "10337", "16836", "9033", "23129", "15348", "22724", "3946", "9078", "13233", "23697", "18894", "11986", "21351", "384", "8018", "1883", "22332", "7299", "23775", "22069", "16267", "764", "22812", "9362", "19456", "5243", "1822", "16719", "7713", "10014", "25534", "23160", "23574", "12330", "14607", "26028", "16546", "12071", "15565", "12402", "22450", "12398", "13681", "22081", "23619", "20307", "15216", "16229", "4961", "224", "23551", "15345", "6732", "7745", "18587", "20639", "19777", "23695", "17685", "10390", "17826", "4555", "17393", "9255", "16076", "17566", "2476", "3080", "17529", "25799", "10288", "11471", "8883", "13273", "15335", "9372", "1833", "17651", "13836", "22123", "7241", "6112", "14223", "25869", "14151", "15339", "11397", "25178", "12552", "16759", "3866", "15050", "2419", "2315", "23172", "5472", "8853", "4258", "9742", "17381", "3575", "18578", "19997", "3676", "13585", "20791", "7807", "10119", "13369", "148", "12467", "8529", "9556", "4928", "10938", "9932", "23921", "24465", "12623", "8515", "18377", "6501", "24921", "9126", "13183", "5452", "3652", "22186", "1576", "13961", "18311", "22648", "16396", "4279", "15149", "6839", "14294", "1468", "13185", "24037", "22004", "22821", "15872", "13266", "10996", "14723", "1823", "16993", "18417", "5536", "1010", "7534", "16515", "4536", "19554", "5621", "3158", "4900", "10245", "14827", "17653", "13357", "15241", "20069", "25994", "24683", "17517", "3564", "17862", "24608", "3779", "25548", "12843", "14340", "12033", "9084", "19892", "5313", "16131", "19532", "20383", "4854", "627", "22257", "9781", "15833", "10112", "16858", "12300", "7319", "22742", "6922", "11382", "10645", "9036", "19963", "15458", "12215", "18895", "17688", "13811", "2302", "6500", "1066", "13977", "19938", "24463", "12806", "15791", "16915", "21275", "8318", "5363", "13950", "7389", "9395", "5599", "18207", "3265", "7937", "8251", "21774", "9741", "2379", "10626", "1515", "20707", "9812", "4892", "2109", "17412", "3603", "11065", "16869", "8195", "24458", "4841", "24730", "6363", "12478", "20809", "5308", "8700", "965", "18942", "22639", "22223", "16048", "17816", "12966", "15278", "24897", "23069", "18009", "15311", "19563", "14041", "14063", "15309", "24215", "21089", "17988", "10780", "7790", "3670", "3088", "2975", "1974", "22635", "9694", "1259", "20407", "6772", "25136", "25218", "8768", "16953", "3089", "16311", "5083", "20915", "6969", "5039", "23696", "3398", "21415", "20970", "6210", "26022", "21382", "5849", "26021", "19972", "4572", "19392", "15445", "4500", "3482", "24248", "3280", "24167", "8230", "17934", "10724", "1220", "22273", "20002", "15084", "8699", "16132", "13673", "10651", "23417", "24612", "3031", "8173", "21870", "15325", "2907", "22331", "15778", "25050", "11697", "9796", "16779", "7827", "7589", "11273", "14851", "15900", "8405", "22200", "9672", "20913", "588", "10473", "12785", "19475", "11071", "18889", "11639", "11410", "14323", "10776", "1414", "2276", "11087", "10785", "12824", "4605", "12242", "13162", "747", "16117", "20637", "21700", "15976", "5473", "25924", "19710", "616", "9894", "25511", "492", "14306", "10324", "2828", "23065", "2570", "23527", "2224", "14331", "16637", "11949", "4502", "16275", "21232", "15895", "1924", "8542", "4858", "16136", "1707", "1424", "19380", "23525", "24512", "17911", "5275", "9701", "17800", "57", "6247", "7960", "21134", "14658", "13723", "4970", "10755", "16649", "22173", "18021", "8153", "25328", "19974", "9245", "19291", "8166", "12891", "15290", "25472", "23978", "118", "1544", "22615", "6310", "2306", "16495", "13489", "14066", "7783", "2439", "3099", "21101", "15263", "10759", "1644", "3472", "23302", "19893", "13132", "7633", "9861", "2281", "8275", "1923", "7227", "25829", "8429", "837", "19284", "746", "24200", "11115", "15466", "481", "7162", "6512", "8394", "2865", "7532", "12754", "6165", "15287", "12732", "17079", "11681", "31", "15133", "14046", "23465", "14729", "3653", "21423", "2823", "20161", "8459", "5454", "21614", "9771", "22696", "19062", "19697", "17936", "7852", "18017", "21154", "1390", "24006", "3768", "792", "6464", "20765", "10250", "10867", "3945", "10059", "1952", "13122", "1767", "11585", "221", "2640", "4968", "16349", "14262", "2703", "5302", "2663", "8310", "4600", "24032", "4660", "6581", "10748", "4239", "5991", "12818", "16339", "6104", "46", "20516", "24504", "16487", "16334", "6706", "20870", "21708", "14662", "22233", "14513", "4696", "9602", "3715", "9880", "19928", "8361", "14817", "20788", "23024", "22479", "12056", "23516", "2606", "3592", "17702", "4995", "5633", "22215", "15767", "14110", "15159", "12978", "23371", "6904", "1897", "23151", "18312", "3390", "13812", "9370", "14403", "445", "9047", "17785", "2649", "6386", "1268", "22166", "13401", "20891", "1540", "22527", "7605", "24170", "9146", "24131", "19393", "19877", "16420", "24874", "24916", "4906", "17294", "17548", "11945", "543", "7193", "15523", "14683", "20122", "7936", "895", "19536", "14005", "21258", "25909", "15170", "8388", "24742", "11345", "355", "9699", "5191", "16671", "6402", "22678", "23530", "10151", "23641", "14530", "20932", "17274", "21702", "5416", "7256", "22580", "7786", "2506", "11657", "11310", "9598", "21443", "7516", "667", "22947", "5359", "12644", "25707", "23972", "10341", "20718", "8909", "4410", "2664", "20670", "4532", "9254", "5780", "24829", "21557", "2388", "21535", "11551", "20502", "7160", "6431", "9319", "24845", "17023", "25816", "18210", "22008", "22950", "25508", "16063", "5620", "10313", "3995", "12752", "1398", "3137", "1846", "9236", "15713", "11687", "11529", "23278", "13168", "4102", "14633", "6612", "21831", "2754", "6460", "5357", "23672", "3555", "271", "1183", "21961", "18533", "25453", "20023", "16125", "870", "5506", "10402", "5375", "9218", "15630", "1570", "18553", "12654", "891", "15892", "5972", "10812", "14321", "4978", "19372", "5927", "600", "3424", "1510", "16364", "7140", "1982", "10811", "11586", "19819", "4682", "21987", "23762", "10756", "2789", "3399", "22608", "8062", "24118", "17030", "19296", "18674", "15685", "6079", "10181", "3600", "10422", "24551", "9053", "21729", "19817", "1046", "17710", "2715", "5012", "19615", "21284", "3795", "8309", "23188", "19619", "1925", "11701", "13828", "25431", "8208", "6823", "18701", "9898", "21560", "541", "16822", "8649", "13568", "20503", "3888", "25170", "1757", "4268", "2254", "18599", "15113", "24701", "4915", "18967", "5307", "2243", "25208", "7144", "8120", "8389", "9566", "25496", "3851", "14324", "11792", "3134", "20602", "25753", "6934", "9703", "11992", "9282", "22368", "15176", "3357", "12742", "1641", "7830", "9629", "7222", "13840", "10009", "15301", "4344", "16220", "23550", "16506", "8448", "11825", "4137", "12826", "22311", "13625", "25684", "17075", "12260", "25274", "6856", "4758", "21584", "4038", "7096", "15765", "4418", "17367", "17581", "5792", "16874", "470", "12419", "24448", "18582", "13844", "20062", "16371", "16971", "12322", "9295", "7439", "13792", "18355", "13627", "19839", "18612", "8276", "23925", "21276", "8445", "22229", "1198", "23628", "21400", "2816", "25545", "4076", "20052", "14809", "17164", "21550", "24437", "5337", "5019", "21234", "14846", "606", "5478", "4701", "6254", "14721", "12596", "16120", "21887", "17016", "19866", "21856", "14140", "7388", "13409", "8574", "14096", "3254", "5348", "16036", "5295", "11371", "2666", "9997", "4988", "23676", "15180", "11879", "19732", "12578", "19091", "74", "12417", "3009", "6702", "8593", "2200", "5722", "14761", "4022", "23556", "25257", "15168", "4043", "8925", "13101", "7587", "9688", "4810", "12233", "13965", "21146", "8975", "24869", "14360", "24893", "11611", "8601", "25647", "12675", "19851", "6863", "5563", "20960", "21830", "24369", "18920", "8110", "7196", "5300", "3756", "1535", "23566", "5118", "23374", "10155", "11715", "2213", "17960", "3326", "16865", "24520", "7277", "16717", "5912", "7578", "22642", "13587", "8863", "21738", "723", "2976", "15513", "4807", "18307", "23167", "20003", "6704", "10295", "7640", "3371", "19701", "13760", "3830", "20292", "22723", "24212", "1695", "15519", "4504", "13647", "7664", "17399", "16964", "5423", "3465", "9705", "24385", "21579", "19798", "82", "2336", "23244", "25095", "2752", "10384", "2735", "7244", "24009", "16301", "11590", "7453", "15067", "397", "17453", "15756", "11003", "8023", "9995", "25852", "17230", "21929", "13026", "15959", "7204", "25595", "16609", "15271", "11850", "1473", "5150", "10794", "24050", "4424", "14803", "1785", "4133", "12036", "596", "8591", "12435", "17956", "22794", "25000", "4799", "18819", "3808", "513", "5682", "15557", "11458", "17193", "3460", "24481", "13801", "3573", "2709", "20547", "16401", "8302", "3070", "13975", "5466", "4756", "1429", "15268", "22148", "24478", "21685", "18835", "25903", "3566", "9868", "22764", "20804", "13108", "21208", "17352", "19824", "24686", "16204", "12345", "3463", "14060", "25451", "25565", "11673", "17713", "22528", "23506", "22736", "11979", "22902", "16619", "19932", "10036", "1222", "24510", "11613", "6892", "16961", "22486", "7649", "10502", "23245", "3845", "6318", "13494", "11856", "16482", "4039", "23560", "8380", "7306", "14535", "3437", "25111", "2774", "20064", "13874", "3391", "10461", "21167", "21320", "562", "17609", "8174", "9814", "11588", "23247", "12508", "6763", "5632", "21015", "5388", "23307", "6641", "3925", "21734", "12188", "17061", "21348", "12683", "715", "14454", "22427", "2859", "13362", "2879", "3315", "9870", "35", "9253", "3569", "18521", "11244", "18436", "25143", "14689", "24538", "12055", "19888", "9037", "22903", "5759", "21175", "24235", "498", "16067", "23017", "4219", "23168", "3783", "17287", "6562", "9286", "5431", "12846", "13428", "3542", "17402", "12840", "10663", "17963", "11513", "6270", "9386", "11190", "15109", "13082", "11344", "16796", "3589", "8924", "15927", "13345", "23161", "14855", "22224", "3614", "21221", "14668", "19400", "9109", "13526", "11536", "14820", "10015", "16484", "17397", "13897", "6262", "4291", "19072", "22767", "13980", "25483", "958", "7482", "24425", "14844", "15648", "3224", "23039", "17407", "11440", "20455", "7836", "24424", "12379", "25489", "7212", "8393", "6532", "6241", "1281", "6996", "20985", "3123", "938", "7146", "576", "6127", "12537", "13928", "17739", "9641", "12012", "23569", "23363", "15603", "13459", "15813", "2756", "2623", "22360", "2019", "14767", "11634", "24438", "14344", "15914", "11227", "6877", "13386", "832", "23802", "5890", "18309", "17906", "24107", "17271", "19052", "21079", "6822", "17169", "14155", "4196", "10012", "24451", "24532", "22541", "21602", "20797", "22324", "13826", "858", "13402", "25492", "16237", "25635", "25405", "20841", "12793", "16102", "24740", "14489", "4340", "12592", "10017", "1734", "4886", "6987", "9526", "5500", "23183", "2694", "7321", "9336", "26019", "21212", "23341", "12366", "5720", "20956", "21726", "15605", "10179", "24881", "5079", "7099", "25446", "13514", "18141", "6105", "16005", "18982", "15470", "488", "19659", "4453", "9272", "646", "8284", "17983", "4712", "8414", "16976", "2288", "23018", "12526", "18517", "25486", "7143", "19055", "21107", "5683", "19262", "18680", "18464", "13153", "7525", "4119", "10489", "24738", "25906", "12965", "7540", "5512", "8774", "4449", "22837", "13979", "18822", "15665", "21406", "10744", "4946", "14177", "11991", "22080", "18004", "4729", "8815", "8133", "8563", "13669", "104", "1495", "25081", "15414", "22447", "15837", "7958", "3042", "17431", "18575", "23068", "8219", "20701", "6272", "15666", "7229", "7818", "22823", "21580", "2426", "1416", "16563", "4648", "8831", "17426", "20444", "4709", "21054", "16672", "19180", "7574", "14922", "15941", "20518", "23871", "12147", "5524", "16725", "16250", "2168", "8891", "14158", "23410", "17964", "14480", "10571", "3167", "5246", "5614", "16802", "19803", "14308", "15799", "25223", "127", "3852", "7821", "9327", "12488", "9444", "14228", "20303", "5664", "10553", "25137", "12408", "4587", "23536", "2700", "4958", "18912", "18618", "20341", "3544", "6770", "7250", "24272", "22096", "22958", "19070", "13588", "17990", "1522", "12698", "14428", "12156", "6480", "7208", "19586", "7498", "18014", "25667", "13492", "15629", "7362", "13998", "10169", "9473", "2885", "19424", "21135", "22026", "4820", "8948", "1678", "20437", "14142", "23140", "25353", "13464", "21153", "2516", "17114", "13433", "17257", "7266", "13423", "724", "23581", "8456", "22525", "25824", "1539", "9238", "8767", "17814", "25750", "21291", "7111", "23133", "10985", "7759", "22272", "6230", "12182", "234", "13680", "3207", "18041", "2218", "21779", "10377", "7527", "4733", "15612", "20664", "17361", "15530", "14774", "19326", "20231", "12222", "16312", "17724", "24905", "13592", "5286", "1713", "22152", "354", "17331", "18527", "9912", "20833", "8129", "103", "19300", "19837", "19021", "18097", "6664", "8933", "15492", "23329", "6062", "17316", "18704", "22115", "2488", "12668", "4765", "19085", "5643", "5373", "5574", "15559", "2084", "7499", "11157", "14283", "9316", "18557", "17726", "14537", "1356", "17319", "601", "12937", "21485", "7940", "4084", "18606", "23816", "18095", "3217", "20267", "25258", "23099", "20202", "15442", "23280", "22523", "2936", "16527", "6920", "22886", "2055", "18672", "21523", "23934", "7390", "23893", "8832", "6731", "4473", "23171", "25680", "19188", "19902", "15847", "13375", "1462", "11270", "278", "12384", "4092", "14446", "8185", "10594", "8377", "17596", "10851", "10792", "22934", "17456", "3897", "20422", "18827", "11520", "22688", "17229", "12434", "1344", "1464", "2119", "2597", "23368", "21984", "13746", "21223", "13081", "14494", "19003", "12553", "6321", "23373", "25881", "22230", "817", "1006", "15615", "15441", "7757", "2081", "25561", "16062", "4651", "23936", "23841", "95", "22269", "24316", "22838", "1221", "743", "22442", "13887", "3580", "7046", "21197", "22889", "18566", "22551", "20391", "16156", "3456", "12923", "1523", "1110", "18499", "13764", "21931", "21922", "20663", "2930", "8996", "25867", "15098", "3", "14162", "6159", "4786", "8470", "18605", "11950", "25337", "15779", "386", "15297", "16748", "5543", "15018", "13616", "16905", "3658", "9208", "20395", "4688", "5131", "25034", "17215", "710", "9103", "8630", "23999", "22190", "22987", "17387", "16545", "9991", "20233", "9636", "3960", "1933", "11081", "21220", "9201", "8962", "11121", "25564", "21171", "20692", "7982", "18896", "13191", "19844", "11839", "12472", "18777", "17032", "21583", "19643", "3690", "21361", "22330", "20190", "6373", "5170", "18274", "17396", "19557", "20860", "23423", "1665", "8546", "23875", "12523", "829", "10260", "21564", "12114", "20465", "14526", "6887", "15041", "13420", "5336", "9592", "16330", "2528", "21988", "18555", "12657", "23885", "19616", "6950", "25016", "2753", "14216", "10942", "25189", "22432", "13650", "1365", "3253", "963", "183", "11102", "19684", "13442", "10029", "9189", "8567", "15208", "8457", "16682", "25180", "9679", "24776", "3010", "11820", "21274", "8643", "9133", "1131", "17660", "8001", "20507", "16199", "17602", "23189", "17612", "23126", "19760", "5457", "17573", "16595", "3560", "9746", "18227", "14971", "7728", "22063", "621", "20881", "12950", "9599", "19461", "16409", "22702", "20893", "5294", "6276", "15295", "6605", "18838", "8020", "2742", "169", "22971", "15661", "6794", "12131", "13149", "206", "20848", "20683", "6971", "19682", "9435", "10796", "9210", "2832", "10309", "9530", "25720", "24236", "887", "4787", "22874", "23253", "9729", "9193", "20065", "24487", "18053", "19201", "3029", "23218", "6912", "22407", "6789", "248", "21471", "17197", "19691", "9155", "1236", "23877", "23931", "8357", "24351", "11854", "3411", "17332", "4511", "11348", "9954", "13862", "22665", "6450", "756", "16951", "5773", "24597", "3016", "24466", "23990", "14272", "5866", "12901", "23269", "13703", "11699", "6855", "4272", "1547", "18332", "12602", "5584", "7965", "11582", "24188", "442", "7488", "9653", "5689", "3336", "25700", "4223", "8752", "6651", "16746", "17150", "19521", "20352", "12798", "3133", "18146", "16379", "14521", "22149", "15700", "15640", "20454", "11147", "5925", "12026", "24270", "847", "20186", "8314", "2236", "9433", "20380", "10955", "16406", "2474", "16827", "426", "21266", "9643", "10634", "7301", "11029", "16931", "6607", "5680", "663", "5671", "6208", "9402", "10574", "5072", "18200", "17420", "19327", "7262", "16242", "8135", "3157", "6650", "15025", "19683", "25841", "19395", "1836", "7109", "9858", "24523", "15408", "7602", "6097", "5562", "22239", "25611", "3919", "9712", "16677", "23153", "15310", "20759", "11187", "193", "16429", "1562", "17260", "17654", "9062", "8011", "13557", "18910", "10861", "16815", "10233", "10714", "13615", "5183", "23693", "19955", "22989", "16441", "23852", "18577", "3950", "21030", "10272", "20434", "23125", "9654", "24222", "8206", "5508"]]} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn.config b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn.config new file mode 100644 index 0000000..b681784 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/configs/nas-benchmark/svhn.config @@ -0,0 +1,13 @@ +{ + "scheduler": ["str", "cos"], + "eta_min" : ["float", "0.0"], + "epochs" : ["int", "200"], + "warmup" : ["int", "0"], + "optim" : ["str", "SGD"], + "LR" : ["float", "0.1"], + "decay" : ["float", "0.0005"], + "momentum" : ["float", "0.9"], + "nesterov" : ["bool", "1"], + "criterion": ["str", "Softmax"], + "batch_size": ["int", "256"] +} diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/functions.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/functions.py new file mode 100644 index 0000000..0eb47ef --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/functions.py @@ -0,0 +1,140 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +##################################################### +import time, torch +from procedures import prepare_seed, get_optim_scheduler +from nasbench_utils import get_model_infos, obtain_accuracy +from config_utils import dict2config +from log_utils import AverageMeter, time_string, convert_secs2time +from nas_bench_201_models import get_cell_based_tiny_net + + +__all__ = ['evaluate_for_seed', 'pure_evaluate'] + + +def pure_evaluate(xloader, network, criterion=torch.nn.CrossEntropyLoss()): + data_time, batch_time, batch = AverageMeter(), AverageMeter(), None + losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter() + latencies = [] + network.eval() + with torch.no_grad(): + end = time.time() + for i, (inputs, targets) in enumerate(xloader): + targets = targets.cuda(non_blocking=True) + inputs = inputs.cuda(non_blocking=True) + data_time.update(time.time() - end) + # forward + features, logits = network(inputs) + loss = criterion(logits, targets) + batch_time.update(time.time() - end) + if batch is None or batch == inputs.size(0): + batch = inputs.size(0) + latencies.append( batch_time.val - data_time.val ) + # record loss and accuracy + prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5)) + losses.update(loss.item(), inputs.size(0)) + top1.update (prec1.item(), inputs.size(0)) + top5.update (prec5.item(), inputs.size(0)) + end = time.time() + if len(latencies) > 2: latencies = latencies[1:] + return losses.avg, top1.avg, top5.avg, latencies + + + +def procedure(xloader, network, criterion, scheduler, optimizer, mode): + losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter() + if mode == 'train' : network.train() + elif mode == 'valid': network.eval() + else: raise ValueError("The mode is not right : {:}".format(mode)) + + data_time, batch_time, end = AverageMeter(), AverageMeter(), time.time() + for i, (inputs, targets) in enumerate(xloader): + if mode == 'train': scheduler.update(None, 1.0 * i / len(xloader)) + + targets = targets.cuda(non_blocking=True) + if mode == 'train': optimizer.zero_grad() + # forward + features, logits = network(inputs) + loss = criterion(logits, targets) + # backward + if mode == 'train': + loss.backward() + optimizer.step() + # record loss and accuracy + prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5)) + losses.update(loss.item(), inputs.size(0)) + top1.update (prec1.item(), inputs.size(0)) + top5.update (prec5.item(), inputs.size(0)) + # count time + batch_time.update(time.time() - end) + end = time.time() + return losses.avg, top1.avg, top5.avg, batch_time.sum + + + +def evaluate_for_seed(arch_config, config, arch, train_loader, valid_loaders, seed, logger): + prepare_seed(seed) # random seed + net = get_cell_based_tiny_net(dict2config({'name': 'infer.tiny', + 'C': arch_config['channel'], 'N': arch_config['num_cells'], + 'genotype': arch, 'num_classes': config.class_num} + , None) + ) + #net = TinyNetwork(arch_config['channel'], arch_config['num_cells'], arch, config.class_num) + if 'ckpt_path' in arch_config.keys(): + ckpt = torch.load(arch_config['ckpt_path']) + ckpt['classifier.weight'] = net.state_dict()['classifier.weight'] + ckpt['classifier.bias'] = net.state_dict()['classifier.bias'] + net.load_state_dict(ckpt) + + flop, param = get_model_infos(net, config.xshape) + logger.log('Network : {:}'.format(net.get_message()), False) + logger.log('{:} Seed-------------------------- {:} --------------------------'.format(time_string(), seed)) + logger.log('FLOP = {:} MB, Param = {:} MB'.format(flop, param)) + # train and valid + optimizer, scheduler, criterion = get_optim_scheduler(net.parameters(), config) + network, criterion = torch.nn.DataParallel(net).cuda(), criterion.cuda() + # network, criterion = torch.nn.DataParallel(net).to(torch.device(f"cuda:{device}")), criterion.to(torch.device(f"cuda:{device}")) + # start training + start_time, epoch_time, total_epoch = time.time(), AverageMeter(), config.epochs + config.warmup + train_losses, train_acc1es, train_acc5es, valid_losses, valid_acc1es, valid_acc5es = {}, {}, {}, {}, {}, {} + train_times , valid_times = {}, {} + for epoch in range(total_epoch): + scheduler.update(epoch, 0.0) + + train_loss, train_acc1, train_acc5, train_tm = procedure(train_loader, network, criterion, scheduler, optimizer, 'train') + train_losses[epoch] = train_loss + train_acc1es[epoch] = train_acc1 + train_acc5es[epoch] = train_acc5 + train_times [epoch] = train_tm + with torch.no_grad(): + for key, xloder in valid_loaders.items(): + valid_loss, valid_acc1, valid_acc5, valid_tm = procedure(xloder , network, criterion, None, None, 'valid') + valid_losses['{:}@{:}'.format(key,epoch)] = valid_loss + valid_acc1es['{:}@{:}'.format(key,epoch)] = valid_acc1 + valid_acc5es['{:}@{:}'.format(key,epoch)] = valid_acc5 + valid_times ['{:}@{:}'.format(key,epoch)] = valid_tm + + # measure elapsed time + epoch_time.update(time.time() - start_time) + start_time = time.time() + need_time = 'Time Left: {:}'.format( convert_secs2time(epoch_time.avg * (total_epoch-epoch-1), True) ) + logger.log('{:} {:} epoch={:03d}/{:03d} :: Train [loss={:.5f}, acc@1={:.2f}%, acc@5={:.2f}%] Valid [loss={:.5f}, acc@1={:.2f}%, acc@5={:.2f}%]'.format(time_string(), need_time, epoch, total_epoch, train_loss, train_acc1, train_acc5, valid_loss, valid_acc1, valid_acc5)) + info_seed = {'flop' : flop, + 'param': param, + 'channel' : arch_config['channel'], + 'num_cells' : arch_config['num_cells'], + 'config' : config._asdict(), + 'total_epoch' : total_epoch , + 'train_losses': train_losses, + 'train_acc1es': train_acc1es, + 'train_acc5es': train_acc5es, + 'train_times' : train_times, + 'valid_losses': valid_losses, + 'valid_acc1es': valid_acc1es, + 'valid_acc5es': valid_acc5es, + 'valid_times' : valid_times, + 'net_state_dict': net.state_dict(), + 'net_string' : '{:}'.format(net), + 'finish-train': True + } + return info_seed diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/__init__.py new file mode 100644 index 0000000..6175653 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/__init__.py @@ -0,0 +1,9 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +# every package does not rely on pytorch or tensorflow +# I tried to list all dependency here: os, sys, time, numpy, (possibly) matplotlib +from .logger import Logger#, PrintLogger +from .meter import AverageMeter +from .time_utils import time_for_file, time_string, time_string_short, time_print, convert_secs2time +from .time_utils import time_string, convert_secs2time diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/logger.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/logger.py new file mode 100644 index 0000000..e60c78f --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/logger.py @@ -0,0 +1,150 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from pathlib import Path +import importlib, warnings +import os, sys, time, numpy as np +if sys.version_info.major == 2: # Python 2.x + from StringIO import StringIO as BIO +else: # Python 3.x + from io import BytesIO as BIO + +if importlib.util.find_spec('tensorflow'): + import tensorflow as tf + + +class PrintLogger(object): + + def __init__(self): + """Create a summary writer logging to log_dir.""" + self.name = 'PrintLogger' + + def log(self, string): + print (string) + + def close(self): + print ('-'*30 + ' close printer ' + '-'*30) + + +class Logger(object): + + def __init__(self, log_dir, seed, create_model_dir=True, use_tf=False): + """Create a summary writer logging to log_dir.""" + self.seed = int(seed) + self.log_dir = Path(log_dir) + self.model_dir = Path(log_dir) / 'checkpoint' + self.log_dir.mkdir (parents=True, exist_ok=True) + if create_model_dir: + self.model_dir.mkdir(parents=True, exist_ok=True) + #self.meta_dir.mkdir(mode=0o775, parents=True, exist_ok=True) + + self.use_tf = bool(use_tf) + self.tensorboard_dir = self.log_dir / ('tensorboard-{:}'.format(time.strftime( '%d-%h', time.gmtime(time.time()) ))) + #self.tensorboard_dir = self.log_dir / ('tensorboard-{:}'.format(time.strftime( '%d-%h-at-%H:%M:%S', time.gmtime(time.time()) ))) + self.logger_path = self.log_dir / 'seed-{:}-T-{:}.log'.format(self.seed, time.strftime('%d-%h-at-%H-%M-%S', time.gmtime(time.time()))) + self.logger_file = open(self.logger_path, 'w') + + if self.use_tf: + self.tensorboard_dir.mkdir(mode=0o775, parents=True, exist_ok=True) + self.writer = tf.summary.FileWriter(str(self.tensorboard_dir)) + else: + self.writer = None + + def __repr__(self): + return ('{name}(dir={log_dir}, use-tf={use_tf}, writer={writer})'.format(name=self.__class__.__name__, **self.__dict__)) + + def path(self, mode): + valids = ('model', 'best', 'info', 'log') + if mode == 'model': return self.model_dir / 'seed-{:}-basic.pth'.format(self.seed) + elif mode == 'best' : return self.model_dir / 'seed-{:}-best.pth'.format(self.seed) + elif mode == 'info' : return self.log_dir / 'seed-{:}-last-info.pth'.format(self.seed) + elif mode == 'log' : return self.log_dir + else: raise TypeError('Unknow mode = {:}, valid modes = {:}'.format(mode, valids)) + + def extract_log(self): + return self.logger_file + + def close(self): + self.logger_file.close() + if self.writer is not None: + self.writer.close() + + def log(self, string, save=True, stdout=False): + if stdout: + sys.stdout.write(string); sys.stdout.flush() + else: + print (string) + if save: + self.logger_file.write('{:}\n'.format(string)) + self.logger_file.flush() + + def scalar_summary(self, tags, values, step): + """Log a scalar variable.""" + if not self.use_tf: + warnings.warn('Do set use-tensorflow installed but call scalar_summary') + else: + assert isinstance(tags, list) == isinstance(values, list), 'Type : {:} vs {:}'.format(type(tags), type(values)) + if not isinstance(tags, list): + tags, values = [tags], [values] + for tag, value in zip(tags, values): + summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)]) + self.writer.add_summary(summary, step) + self.writer.flush() + + def image_summary(self, tag, images, step): + """Log a list of images.""" + import scipy + if not self.use_tf: + warnings.warn('Do set use-tensorflow installed but call scalar_summary') + return + + img_summaries = [] + for i, img in enumerate(images): + # Write the image to a string + try: + s = StringIO() + except: + s = BytesIO() + scipy.misc.toimage(img).save(s, format="png") + + # Create an Image object + img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(), + height=img.shape[0], + width=img.shape[1]) + # Create a Summary value + img_summaries.append(tf.Summary.Value(tag='{}/{}'.format(tag, i), image=img_sum)) + + # Create and write Summary + summary = tf.Summary(value=img_summaries) + self.writer.add_summary(summary, step) + self.writer.flush() + + def histo_summary(self, tag, values, step, bins=1000): + """Log a histogram of the tensor of values.""" + if not self.use_tf: raise ValueError('Do not have tensorflow') + import tensorflow as tf + + # Create a histogram using numpy + counts, bin_edges = np.histogram(values, bins=bins) + + # Fill the fields of the histogram proto + hist = tf.HistogramProto() + hist.min = float(np.min(values)) + hist.max = float(np.max(values)) + hist.num = int(np.prod(values.shape)) + hist.sum = float(np.sum(values)) + hist.sum_squares = float(np.sum(values**2)) + + # Drop the start of the first bin + bin_edges = bin_edges[1:] + + # Add bin edges and counts + for edge in bin_edges: + hist.bucket_limit.append(edge) + for c in counts: + hist.bucket.append(c) + + # Create and write Summary + summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)]) + self.writer.add_summary(summary, step) + self.writer.flush() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/meter.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/meter.py new file mode 100644 index 0000000..cbb9dd1 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/meter.py @@ -0,0 +1,98 @@ +import numpy as np + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self): + self.reset() + + def reset(self): + self.val = 0.0 + self.avg = 0.0 + self.sum = 0.0 + self.count = 0.0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __repr__(self): + return ('{name}(val={val}, avg={avg}, count={count})'.format(name=self.__class__.__name__, **self.__dict__)) + + +class RecorderMeter(object): + """Computes and stores the minimum loss value and its epoch index""" + def __init__(self, total_epoch): + self.reset(total_epoch) + + def reset(self, total_epoch): + assert total_epoch > 0, 'total_epoch should be greater than 0 vs {:}'.format(total_epoch) + self.total_epoch = total_epoch + self.current_epoch = 0 + self.epoch_losses = np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val] + self.epoch_losses = self.epoch_losses - 1 + self.epoch_accuracy= np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val] + self.epoch_accuracy= self.epoch_accuracy + + def update(self, idx, train_loss, train_acc, val_loss, val_acc): + assert idx >= 0 and idx < self.total_epoch, 'total_epoch : {} , but update with the {} index'.format(self.total_epoch, idx) + self.epoch_losses [idx, 0] = train_loss + self.epoch_losses [idx, 1] = val_loss + self.epoch_accuracy[idx, 0] = train_acc + self.epoch_accuracy[idx, 1] = val_acc + self.current_epoch = idx + 1 + return self.max_accuracy(False) == self.epoch_accuracy[idx, 1] + + def max_accuracy(self, istrain): + if self.current_epoch <= 0: return 0 + if istrain: return self.epoch_accuracy[:self.current_epoch, 0].max() + else: return self.epoch_accuracy[:self.current_epoch, 1].max() + + def plot_curve(self, save_path): + import matplotlib + matplotlib.use('agg') + import matplotlib.pyplot as plt + title = 'the accuracy/loss curve of train/val' + dpi = 100 + width, height = 1600, 1000 + legend_fontsize = 10 + figsize = width / float(dpi), height / float(dpi) + + fig = plt.figure(figsize=figsize) + x_axis = np.array([i for i in range(self.total_epoch)]) # epochs + y_axis = np.zeros(self.total_epoch) + + plt.xlim(0, self.total_epoch) + plt.ylim(0, 100) + interval_y = 5 + interval_x = 5 + plt.xticks(np.arange(0, self.total_epoch + interval_x, interval_x)) + plt.yticks(np.arange(0, 100 + interval_y, interval_y)) + plt.grid() + plt.title(title, fontsize=20) + plt.xlabel('the training epoch', fontsize=16) + plt.ylabel('accuracy', fontsize=16) + + y_axis[:] = self.epoch_accuracy[:, 0] + plt.plot(x_axis, y_axis, color='g', linestyle='-', label='train-accuracy', lw=2) + plt.legend(loc=4, fontsize=legend_fontsize) + + y_axis[:] = self.epoch_accuracy[:, 1] + plt.plot(x_axis, y_axis, color='y', linestyle='-', label='valid-accuracy', lw=2) + plt.legend(loc=4, fontsize=legend_fontsize) + + + y_axis[:] = self.epoch_losses[:, 0] + plt.plot(x_axis, y_axis*50, color='g', linestyle=':', label='train-loss-x50', lw=2) + plt.legend(loc=4, fontsize=legend_fontsize) + + y_axis[:] = self.epoch_losses[:, 1] + plt.plot(x_axis, y_axis*50, color='y', linestyle=':', label='valid-loss-x50', lw=2) + plt.legend(loc=4, fontsize=legend_fontsize) + + if save_path is not None: + fig.savefig(save_path, dpi=dpi, bbox_inches='tight') + print ('---- save figure {} into {}'.format(title, save_path)) + plt.close(fig) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/time_utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/time_utils.py new file mode 100644 index 0000000..4a0f78e --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/log_utils/time_utils.py @@ -0,0 +1,42 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import time, sys +import numpy as np + +def time_for_file(): + ISOTIMEFORMAT='%d-%h-at-%H-%M-%S' + return '{:}'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) + +def time_string(): + ISOTIMEFORMAT='%Y-%m-%d %X' + string = '[{:}]'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) + return string + +def time_string_short(): + ISOTIMEFORMAT='%Y%m%d' + string = '{:}'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) + return string + +def time_print(string, is_print=True): + if (is_print): + print('{} : {}'.format(time_string(), string)) + +def convert_secs2time(epoch_time, return_str=False): + need_hour = int(epoch_time / 3600) + need_mins = int((epoch_time - 3600*need_hour) / 60) + need_secs = int(epoch_time - 3600*need_hour - 60*need_mins) + if return_str: + str = '[{:02d}:{:02d}:{:02d}]'.format(need_hour, need_mins, need_secs) + return str + else: + return need_hour, need_mins, need_secs + +def print_log(print_string, log): + #if isinstance(log, Logger): log.log('{:}'.format(print_string)) + if hasattr(log, 'log'): log.log('{:}'.format(print_string)) + else: + print("{:}".format(print_string)) + if log is not None: + log.write('{:}\n'.format(print_string)) + log.flush() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/__init__.py new file mode 100644 index 0000000..1f31583 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/__init__.py @@ -0,0 +1,4 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from .get_dataset_with_transform import get_datasets diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/aircraft.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/aircraft.py new file mode 100644 index 0000000..e578eb1 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/aircraft.py @@ -0,0 +1,179 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import torch.utils.data as data +from torchvision.datasets.folder import pil_loader, accimage_loader, default_loader +from PIL import Image +import os +import numpy as np + + +def make_dataset(dir, image_ids, targets): + assert (len(image_ids) == len(targets)) + images = [] + dir = os.path.expanduser(dir) + for i in range(len(image_ids)): + item = (os.path.join(dir, 'data', 'images', + '%s.jpg' % image_ids[i]), targets[i]) + images.append(item) + return images + + +def find_classes(classes_file): + # read classes file, separating out image IDs and class names + image_ids = [] + targets = [] + f = open(classes_file, 'r') + for line in f: + split_line = line.split(' ') + image_ids.append(split_line[0]) + targets.append(' '.join(split_line[1:])) + f.close() + + # index class names + classes = np.unique(targets) + class_to_idx = {classes[i]: i for i in range(len(classes))} + targets = [class_to_idx[c] for c in targets] + + return (image_ids, targets, classes, class_to_idx) + + +class FGVCAircraft(data.Dataset): + """`FGVC-Aircraft `_ Dataset. + Args: + root (string): Root directory path to dataset. + class_type (string, optional): The level of FGVC-Aircraft fine-grain classification + to label data with (i.e., ``variant``, ``family``, or ``manufacturer``). + transform (callable, optional): A function/transform that takes in a PIL image + and returns a transformed version. E.g. ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + loader (callable, optional): A function to load an image given its path. + download (bool, optional): If true, downloads the dataset from the internet and + puts it in the root directory. If dataset is already downloaded, it is not + downloaded again. + """ + url = 'http://www.robots.ox.ac.uk/~vgg/data/fgvc-aircraft/archives/fgvc-aircraft-2013b.tar.gz' + class_types = ('variant', 'family', 'manufacturer') + splits = ('train', 'val', 'trainval', 'test') + + def __init__(self, root, class_type='variant', split='train', transform=None, + target_transform=None, loader=default_loader, download=False): + if split not in self.splits: + raise ValueError('Split "{}" not found. Valid splits are: {}'.format( + split, ', '.join(self.splits), + )) + if class_type not in self.class_types: + raise ValueError('Class type "{}" not found. Valid class types are: {}'.format( + class_type, ', '.join(self.class_types), + )) + self.root = os.path.expanduser(root) + self.root = os.path.join(self.root, 'fgvc-aircraft-2013b') + self.class_type = class_type + self.split = split + self.classes_file = os.path.join(self.root, 'data', + 'images_%s_%s.txt' % (self.class_type, self.split)) + + if download: + self.download() + + (image_ids, targets, classes, class_to_idx) = find_classes(self.classes_file) + samples = make_dataset(self.root, image_ids, targets) + + self.transform = transform + self.target_transform = target_transform + self.loader = loader + + self.samples = samples + self.classes = classes + self.class_to_idx = class_to_idx + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + + path, target = self.samples[index] + sample = self.loader(path) + if self.transform is not None: + sample = self.transform(sample) + if self.target_transform is not None: + target = self.target_transform(target) + + return sample, target + + def __len__(self): + return len(self.samples) + + def __repr__(self): + fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Root Location: {}\n'.format(self.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + def _check_exists(self): + return os.path.exists(os.path.join(self.root, 'data', 'images')) and \ + os.path.exists(self.classes_file) + + def download(self): + """Download the FGVC-Aircraft data if it doesn't exist already.""" + from six.moves import urllib + import tarfile + + if self._check_exists(): + return + + # prepare to download data to PARENT_DIR/fgvc-aircraft-2013.tar.gz + print('Downloading %s ... (may take a few minutes)' % self.url) + parent_dir = os.path.abspath(os.path.join(self.root, os.pardir)) + tar_name = self.url.rpartition('/')[-1] + tar_path = os.path.join(parent_dir, tar_name) + data = urllib.request.urlopen(self.url) + + # download .tar.gz file + with open(tar_path, 'wb') as f: + f.write(data.read()) + + # extract .tar.gz to PARENT_DIR/fgvc-aircraft-2013b + data_folder = tar_path.strip('.tar.gz') + print('Extracting %s to %s ... (may take a few minutes)' % (tar_path, data_folder)) + tar = tarfile.open(tar_path) + tar.extractall(parent_dir) + + # if necessary, rename data folder to self.root + if not os.path.samefile(data_folder, self.root): + print('Renaming %s to %s ...' % (data_folder, self.root)) + os.rename(data_folder, self.root) + + # delete .tar.gz file + print('Deleting %s ...' % tar_path) + os.remove(tar_path) + + print('Done!') + + +if __name__ == '__main__': + air = FGVCAircraft('/w14/dataset/fgvc-aircraft-2013b', class_type='manufacturer', split='train', transform=None, + target_transform=None, loader=default_loader, download=False) + print(len(air)) + print(len(air)) + air = FGVCAircraft('/w14/dataset/fgvc-aircraft-2013b', class_type='manufacturer', split='val', transform=None, + target_transform=None, loader=default_loader, download=False) + air = FGVCAircraft('/w14/dataset/fgvc-aircraft-2013b', class_type='manufacturer', split='trainval', transform=None, + target_transform=None, loader=default_loader, download=False) + print(len(air)) + air = FGVCAircraft('/w14/dataset/fgvc-aircraft-2013b/', class_type='manufacturer', split='test', transform=None, + target_transform=None, loader=default_loader, download=False) + print(len(air)) + import pdb; + pdb.set_trace() + print(len(air)) \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/get_dataset_with_transform.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/get_dataset_with_transform.py new file mode 100644 index 0000000..249f403 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/get_dataset_with_transform.py @@ -0,0 +1,304 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +# Modified by Hayeon Lee, Eunyoung Hyung 2021. 03. +################################################## +import os +import sys +import torch +import os.path as osp +import numpy as np +import torchvision.datasets as dset +import torchvision.transforms as transforms +from copy import deepcopy +# from PIL import Image +import random +import pdb +from .aircraft import FGVCAircraft +from .pets import PetDataset +from config_utils import load_config + +Dataset2Class = {'cifar10': 10, + 'cifar100': 100, + 'mnist': 10, + 'svhn': 10, + 'aircraft': 30, + 'pets': 37} + + +class CUTOUT(object): + + def __init__(self, length): + self.length = length + + def __repr__(self): + return ('{name}(length={length})'.format(name=self.__class__.__name__, **self.__dict__)) + + def __call__(self, img): + h, w = img.size(1), img.size(2) + mask = np.ones((h, w), np.float32) + y = np.random.randint(h) + x = np.random.randint(w) + + y1 = np.clip(y - self.length // 2, 0, h) + y2 = np.clip(y + self.length // 2, 0, h) + x1 = np.clip(x - self.length // 2, 0, w) + x2 = np.clip(x + self.length // 2, 0, w) + + mask[y1: y2, x1: x2] = 0. + mask = torch.from_numpy(mask) + mask = mask.expand_as(img) + img *= mask + return img + + +imagenet_pca = { + 'eigval': np.asarray([0.2175, 0.0188, 0.0045]), + 'eigvec': np.asarray([ + [-0.5675, 0.7192, 0.4009], + [-0.5808, -0.0045, -0.8140], + [-0.5836, -0.6948, 0.4203], + ]) +} + + +class Lighting(object): + def __init__(self, alphastd, + eigval=imagenet_pca['eigval'], + eigvec=imagenet_pca['eigvec']): + self.alphastd = alphastd + assert eigval.shape == (3,) + assert eigvec.shape == (3, 3) + self.eigval = eigval + self.eigvec = eigvec + + def __call__(self, img): + if self.alphastd == 0.: + return img + rnd = np.random.randn(3) * self.alphastd + rnd = rnd.astype('float32') + v = rnd + old_dtype = np.asarray(img).dtype + v = v * self.eigval + v = v.reshape((3, 1)) + inc = np.dot(self.eigvec, v).reshape((3,)) + img = np.add(img, inc) + if old_dtype == np.uint8: + img = np.clip(img, 0, 255) + img = Image.fromarray(img.astype(old_dtype), 'RGB') + return img + + def __repr__(self): + return self.__class__.__name__ + '()' + + +def get_datasets(name, root, cutout, use_num_cls=None): + if name == 'cifar10': + mean = [x / 255 for x in [125.3, 123.0, 113.9]] + std = [x / 255 for x in [63.0, 62.1, 66.7]] + elif name == 'cifar100': + mean = [x / 255 for x in [129.3, 124.1, 112.4]] + std = [x / 255 for x in [68.2, 65.4, 70.4]] + elif name.startswith('mnist'): + mean, std = [0.1307, 0.1307, 0.1307], [0.3081, 0.3081, 0.3081] + elif name.startswith('svhn'): + mean, std = [0.4376821, 0.4437697, 0.47280442], [ + 0.19803012, 0.20101562, 0.19703614] + elif name.startswith('aircraft'): + mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] + elif name.startswith('pets'): + mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] + else: + raise TypeError("Unknow dataset : {:}".format(name)) + + # Data Argumentation + if name == 'cifar10' or name == 'cifar100': + lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(), + transforms.Normalize(mean, std)] + if cutout > 0: + lists += [CUTOUT(cutout)] + train_transform = transforms.Compose(lists) + test_transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize(mean, std)]) + xshape = (1, 3, 32, 32) + elif name.startswith('cub200'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + xshape = (1, 3, 32, 32) + elif name.startswith('mnist'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Lambda(lambda x: x.repeat(3, 1, 1)), + transforms.Normalize(mean, std), + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Lambda(lambda x: x.repeat(3, 1, 1)), + transforms.Normalize(mean, std) + ]) + xshape = (1, 3, 32, 32) + elif name.startswith('svhn'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + xshape = (1, 3, 32, 32) + elif name.startswith('aircraft'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std), + ]) + xshape = (1, 3, 32, 32) + elif name.startswith('pets'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std), + ]) + xshape = (1, 3, 32, 32) + else: + raise TypeError("Unknow dataset : {:}".format(name)) + + if name == 'cifar10': + train_data = dset.CIFAR10( + root, train=True, transform=train_transform, download=True) + test_data = dset.CIFAR10( + root, train=False, transform=test_transform, download=True) + assert len(train_data) == 50000 and len(test_data) == 10000 + elif name == 'cifar100': + train_data = dset.CIFAR100( + root, train=True, transform=train_transform, download=True) + test_data = dset.CIFAR100( + root, train=False, transform=test_transform, download=True) + assert len(train_data) == 50000 and len(test_data) == 10000 + elif name == 'mnist': + train_data = dset.MNIST( + root, train=True, transform=train_transform, download=True) + test_data = dset.MNIST( + root, train=False, transform=test_transform, download=True) + assert len(train_data) == 60000 and len(test_data) == 10000 + elif name == 'svhn': + train_data = dset.SVHN(root, split='train', + transform=train_transform, download=True) + test_data = dset.SVHN(root, split='test', + transform=test_transform, download=True) + assert len(train_data) == 73257 and len(test_data) == 26032 + elif name == 'aircraft': + train_data = FGVCAircraft(root, class_type='manufacturer', split='trainval', + transform=train_transform, download=False) + test_data = FGVCAircraft(root, class_type='manufacturer', split='test', + transform=test_transform, download=False) + assert len(train_data) == 6667 and len(test_data) == 3333 + elif name == 'pets': + train_data = PetDataset(root, train=True, num_cl=37, + val_split=0.15, transforms=train_transform) + test_data = PetDataset(root, train=False, num_cl=37, + val_split=0.15, transforms=test_transform) + else: + raise TypeError("Unknow dataset : {:}".format(name)) + + class_num = Dataset2Class[name] if use_num_cls is None else len( + use_num_cls) + return train_data, test_data, xshape, class_num + + +def get_nas_search_loaders(train_data, valid_data, dataset, config_root, batch_size, workers, num_cls=None): + if isinstance(batch_size, (list, tuple)): + batch, test_batch = batch_size + else: + batch, test_batch = batch_size, batch_size + if dataset == 'cifar10': + # split_Fpath = 'configs/nas-benchmark/cifar-split.txt' + cifar_split = load_config( + '{:}/cifar-split.txt'.format(config_root), None, None) + # search over the proposed training and validation set + train_split, valid_split = cifar_split.train, cifar_split.valid + # logger.log('Load split file from {:}'.format(split_Fpath)) # they are two disjoint groups in the original CIFAR-10 training set + # To split data + xvalid_data = deepcopy(train_data) + if hasattr(xvalid_data, 'transforms'): # to avoid a print issue + xvalid_data.transforms = valid_data.transform + xvalid_data.transform = deepcopy(valid_data.transform) + search_data = SearchDataset( + dataset, train_data, train_split, valid_split) + # data loader + search_loader = torch.utils.data.DataLoader(search_data, batch_size=batch, shuffle=True, num_workers=workers, + pin_memory=True) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + train_split), + num_workers=workers, pin_memory=True) + valid_loader = torch.utils.data.DataLoader(xvalid_data, batch_size=test_batch, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + valid_split), + num_workers=workers, pin_memory=True) + elif dataset == 'cifar100': + cifar100_test_split = load_config( + '{:}/cifar100-test-split.txt'.format(config_root), None, None) + search_train_data = train_data + search_valid_data = deepcopy(valid_data) + search_valid_data.transform = train_data.transform + search_data = SearchDataset(dataset, [search_train_data, search_valid_data], + list(range(len(search_train_data))), + cifar100_test_split.xvalid) + search_loader = torch.utils.data.DataLoader(search_data, batch_size=batch, shuffle=True, num_workers=workers, + pin_memory=True) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch, shuffle=True, num_workers=workers, + pin_memory=True) + valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=test_batch, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + cifar100_test_split.xvalid), num_workers=workers, pin_memory=True) + elif dataset in ['mnist', 'svhn', 'aircraft', 'pets']: + if not os.path.exists('{:}/{}-test-split.txt'.format(config_root, dataset)): + import json + label_list = list(range(len(valid_data))) + random.shuffle(label_list) + strlist = [str(label_list[i]) for i in range(len(label_list))] + split = {'xvalid': ["int", strlist[:len(valid_data) // 2]], + 'xtest': ["int", strlist[len(valid_data) // 2:]]} + with open('{:}/{}-test-split.txt'.format(config_root, dataset), 'w') as f: + f.write(json.dumps(split)) + test_split = load_config( + '{:}/{}-test-split.txt'.format(config_root, dataset), None, None) + + search_train_data = train_data + search_valid_data = deepcopy(valid_data) + search_valid_data.transform = train_data.transform + search_data = SearchDataset(dataset, [search_train_data, search_valid_data], + list(range(len(search_train_data))), test_split.xvalid) + search_loader = torch.utils.data.DataLoader(search_data, batch_size=batch, shuffle=True, + num_workers=workers, pin_memory=True) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch, shuffle=True, + num_workers=workers, pin_memory=True) + valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=test_batch, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + test_split.xvalid), num_workers=workers, pin_memory=True) + else: + raise ValueError('invalid dataset : {:}'.format(dataset)) + return search_loader, train_loader, valid_loader diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/pets.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/pets.py new file mode 100644 index 0000000..899c793 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_datasets/pets.py @@ -0,0 +1,45 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import torch +from glob import glob +from torch.utils.data.dataset import Dataset +import os +from PIL import Image + + +def load_image(filename): + img = Image.open(filename) + img = img.convert('RGB') + return img + +class PetDataset(Dataset): + def __init__(self, root, train=True, num_cl=37, val_split=0.2, transforms=None): + self.data = torch.load(os.path.join(root,'{}{}.pth'.format('train' if train else 'test', + int(100*(1-val_split)) if train else int(100*val_split)))) + self.len = len(self.data) + self.transform = transforms + def __getitem__(self, index): + img, label = self.data[index] + if self.transform: + img = self.transform(img) + return img, label + def __len__(self): + return self.len + +if __name__ == '__main__': + # Added + import torchvision.transforms as transforms + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + train_transform = transforms.Compose( + [transforms.Resize(256), transforms.RandomRotation(45), transforms.CenterCrop(224), + transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize]) + test_transform = transforms.Compose( + [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize]) + root = '/w14/dataset/MetaGen/pets' + train_data, test_data = get_pets(root, num_cl=37, val_split=0.2, + tr_transform=train_transform, + te_transform=test_transform) + import pdb; + pdb.set_trace() diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/SharedUtils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/SharedUtils.py new file mode 100644 index 0000000..8938752 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/SharedUtils.py @@ -0,0 +1,34 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch +import torch.nn as nn + + +def additive_func(A, B): + assert A.dim() == B.dim() and A.size(0) == B.size(0), '{:} vs {:}'.format(A.size(), B.size()) + C = min(A.size(1), B.size(1)) + if A.size(1) == B.size(1): + return A + B + elif A.size(1) < B.size(1): + out = B.clone() + out[:,:C] += A + return out + else: + out = A.clone() + out[:,:C] += B + return out + + +def change_key(key, value): + def func(m): + if hasattr(m, key): + setattr(m, key, value) + return func + + +def parse_channel_info(xstring): + blocks = xstring.split(' ') + blocks = [x.split('-') for x in blocks] + blocks = [[int(_) for _ in x] for x in blocks] + return blocks diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/__init__.py new file mode 100644 index 0000000..de56bc6 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/__init__.py @@ -0,0 +1,45 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from os import path as osp +from typing import List, Text +import torch + +__all__ = ['get_cell_based_tiny_net', 'get_search_spaces', \ + 'CellStructure', 'CellArchitectures' + ] + +# useful modules +from config_utils import dict2config +from .SharedUtils import change_key +from .cell_searchs import CellStructure, CellArchitectures + + +# Cell-based NAS Models +def get_cell_based_tiny_net(config): + if config.name == 'infer.tiny': + from .cell_infers import TinyNetwork + if hasattr(config, 'genotype'): + genotype = config.genotype + elif hasattr(config, 'arch_str'): + genotype = CellStructure.str2structure(config.arch_str) + else: raise ValueError('Can not find genotype from this config : {:}'.format(config)) + return TinyNetwork(config.C, config.N, genotype, config.num_classes) + else: + raise ValueError('invalid network name : {:}'.format(config.name)) + + +# obtain the search space, i.e., a dict mapping the operation name into a python-function for this op +def get_search_spaces(xtype, name) -> List[Text]: + if xtype == 'cell' or xtype == 'tss': # The topology search space. + from .cell_operations import SearchSpaceNames + assert name in SearchSpaceNames, 'invalid name [{:}] in {:}'.format(name, SearchSpaceNames.keys()) + return SearchSpaceNames[name] + elif xtype == 'sss': # The size search space. + if name == 'nas-bench-301': + return {'candidates': [8, 16, 24, 32, 40, 48, 56, 64], + 'numbers': 5} + else: + raise ValueError('Invalid name : {:}'.format(name)) + else: + raise ValueError('invalid search-space type is {:}'.format(xtype)) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/__init__.py new file mode 100644 index 0000000..052b477 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/__init__.py @@ -0,0 +1,4 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +from .tiny_network import TinyNetwork diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/cells.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/cells.py new file mode 100644 index 0000000..7a279e9 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/cells.py @@ -0,0 +1,122 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### + +import torch +import torch.nn as nn +from copy import deepcopy +from ..cell_operations import OPS + + +# Cell for NAS-Bench-201 +class InferCell(nn.Module): + + def __init__(self, genotype, C_in, C_out, stride): + super(InferCell, self).__init__() + + self.layers = nn.ModuleList() + self.node_IN = [] + self.node_IX = [] + self.genotype = deepcopy(genotype) + for i in range(1, len(genotype)): + node_info = genotype[i-1] + cur_index = [] + cur_innod = [] + for (op_name, op_in) in node_info: + if op_in == 0: + layer = OPS[op_name](C_in , C_out, stride, True, True) + else: + layer = OPS[op_name](C_out, C_out, 1, True, True) + # import pdb; pdb.set_trace() + + cur_index.append( len(self.layers) ) + cur_innod.append( op_in ) + self.layers.append( layer ) + self.node_IX.append( cur_index ) + self.node_IN.append( cur_innod ) + self.nodes = len(genotype) + self.in_dim = C_in + self.out_dim = C_out + + def extra_repr(self): + string = 'info :: nodes={nodes}, inC={in_dim}, outC={out_dim}'.format(**self.__dict__) + laystr = [] + for i, (node_layers, node_innods) in enumerate(zip(self.node_IX,self.node_IN)): + y = ['I{:}-L{:}'.format(_ii, _il) for _il, _ii in zip(node_layers, node_innods)] + x = '{:}<-({:})'.format(i+1, ','.join(y)) + laystr.append( x ) + return string + ', [{:}]'.format( ' | '.join(laystr) ) + ', {:}'.format(self.genotype.tostr()) + + def forward(self, inputs): + nodes = [inputs] + for i, (node_layers, node_innods) in enumerate(zip(self.node_IX,self.node_IN)): + node_feature = sum( self.layers[_il](nodes[_ii]) for _il, _ii in zip(node_layers, node_innods) ) + nodes.append( node_feature ) + return nodes[-1] + + + +# Learning Transferable Architectures for Scalable Image Recognition, CVPR 2018 +class NASNetInferCell(nn.Module): + + def __init__(self, genotype, C_prev_prev, C_prev, C, reduction, reduction_prev, affine, track_running_stats): + super(NASNetInferCell, self).__init__() + self.reduction = reduction + if reduction_prev: self.preprocess0 = OPS['skip_connect'](C_prev_prev, C, 2, affine, track_running_stats) + else : self.preprocess0 = OPS['nor_conv_1x1'](C_prev_prev, C, 1, affine, track_running_stats) + self.preprocess1 = OPS['nor_conv_1x1'](C_prev, C, 1, affine, track_running_stats) + + if not reduction: + nodes, concats = genotype['normal'], genotype['normal_concat'] + else: + nodes, concats = genotype['reduce'], genotype['reduce_concat'] + self._multiplier = len(concats) + self._concats = concats + self._steps = len(nodes) + self._nodes = nodes + self.edges = nn.ModuleDict() + for i, node in enumerate(nodes): + for in_node in node: + name, j = in_node[0], in_node[1] + stride = 2 if reduction and j < 2 else 1 + node_str = '{:}<-{:}'.format(i+2, j) + self.edges[node_str] = OPS[name](C, C, stride, affine, track_running_stats) + + # [TODO] to support drop_prob in this function.. + def forward(self, s0, s1, unused_drop_prob): + s0 = self.preprocess0(s0) + s1 = self.preprocess1(s1) + + states = [s0, s1] + for i, node in enumerate(self._nodes): + clist = [] + for in_node in node: + name, j = in_node[0], in_node[1] + node_str = '{:}<-{:}'.format(i+2, j) + op = self.edges[ node_str ] + clist.append( op(states[j]) ) + states.append( sum(clist) ) + return torch.cat([states[x] for x in self._concats], dim=1) + + +class AuxiliaryHeadCIFAR(nn.Module): + + def __init__(self, C, num_classes): + """assuming input size 8x8""" + super(AuxiliaryHeadCIFAR, self).__init__() + self.features = nn.Sequential( + nn.ReLU(inplace=True), + nn.AvgPool2d(5, stride=3, padding=0, count_include_pad=False), # image size = 2 x 2 + nn.Conv2d(C, 128, 1, bias=False), + nn.BatchNorm2d(128), + nn.ReLU(inplace=True), + nn.Conv2d(128, 768, 2, bias=False), + nn.BatchNorm2d(768), + nn.ReLU(inplace=True) + ) + self.classifier = nn.Linear(768, num_classes) + + def forward(self, x): + x = self.features(x) + x = self.classifier(x.view(x.size(0),-1)) + return x diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/tiny_network.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/tiny_network.py new file mode 100644 index 0000000..d3c71db --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_infers/tiny_network.py @@ -0,0 +1,66 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +from ..cell_operations import ResNetBasicblock +from .cells import InferCell + + +# The macro structure for architectures in NAS-Bench-201 +class TinyNetwork(nn.Module): + + def __init__(self, C, N, genotype, num_classes): + super(TinyNetwork, self).__init__() + self._C = C + self._layerN = N + + self.stem = nn.Sequential( + nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(C)) + + layer_channels = [C ] * N + [C*2 ] + [C*2 ] * N + [C*4 ] + [C*4 ] * N + layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N + + C_prev = C + self.cells = nn.ModuleList() + for index, (C_curr, reduction) in enumerate(zip(layer_channels, layer_reductions)): + if reduction: + cell = ResNetBasicblock(C_prev, C_curr, 2, True) + else: + cell = InferCell(genotype, C_prev, C_curr, 1) + self.cells.append( cell ) + C_prev = cell.out_dim + self._Layer= len(self.cells) + + self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True)) + self.global_pooling = nn.AdaptiveAvgPool2d(1) + self.classifier = nn.Linear(C_prev, num_classes) + + def get_message(self): + string = self.extra_repr() + for i, cell in enumerate(self.cells): + string += '\n {:02d}/{:02d} :: {:}'.format(i, len(self.cells), cell.extra_repr()) + return string + + def extra_repr(self): + return ('{name}(C={_C}, N={_layerN}, L={_Layer})'.format(name=self.__class__.__name__, **self.__dict__)) + + def forward(self, inputs): + feature = self.stem(inputs) + for i, cell in enumerate(self.cells): + feature = cell(feature) + ''' + out2 = self.lastact(feature) + out = self.global_pooling( out2 ) + out = out.view(out.size(0), -1) + out2 = out2.view(out2.size(0), -1) + logits = self.classifier(out) + return out2, logits + + ''' + out = self.lastact(feature) + out = self.global_pooling( out ) + out = out.view(out.size(0), -1) + logits = self.classifier(out) + + return out, logits diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_operations.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_operations.py new file mode 100644 index 0000000..c7528c1 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_operations.py @@ -0,0 +1,308 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +import torch +import torch.nn as nn + +__all__ = ['OPS', 'ResNetBasicblock', 'SearchSpaceNames'] + +OPS = { + 'none' : lambda C_in, C_out, stride, affine, track_running_stats: Zero(C_in, C_out, stride), + 'avg_pool_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: POOLING(C_in, C_out, stride, 'avg', affine, track_running_stats), + 'max_pool_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: POOLING(C_in, C_out, stride, 'max', affine, track_running_stats), + 'nor_conv_7x7' : lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(C_in, C_out, (7,7), (stride,stride), (3,3), (1,1), affine, track_running_stats), + 'nor_conv_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(C_in, C_out, (3,3), (stride,stride), (1,1), (1,1), affine, track_running_stats), + 'nor_conv_1x1' : lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(C_in, C_out, (1,1), (stride,stride), (0,0), (1,1), affine, track_running_stats), + 'dua_sepc_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: DualSepConv(C_in, C_out, (3,3), (stride,stride), (1,1), (1,1), affine, track_running_stats), + 'dua_sepc_5x5' : lambda C_in, C_out, stride, affine, track_running_stats: DualSepConv(C_in, C_out, (5,5), (stride,stride), (2,2), (1,1), affine, track_running_stats), + 'dil_sepc_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: SepConv(C_in, C_out, (3,3), (stride,stride), (2,2), (2,2), affine, track_running_stats), + 'dil_sepc_5x5' : lambda C_in, C_out, stride, affine, track_running_stats: SepConv(C_in, C_out, (5,5), (stride,stride), (4,4), (2,2), affine, track_running_stats), + 'skip_connect' : lambda C_in, C_out, stride, affine, track_running_stats: Identity() if stride == 1 and C_in == C_out else FactorizedReduce(C_in, C_out, stride, affine, track_running_stats), +} + +CONNECT_NAS_BENCHMARK = ['none', 'skip_connect', 'nor_conv_3x3'] +NAS_BENCH_201 = ['none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3'] +DARTS_SPACE = ['none', 'skip_connect', 'dua_sepc_3x3', 'dua_sepc_5x5', 'dil_sepc_3x3', 'dil_sepc_5x5', 'avg_pool_3x3', 'max_pool_3x3'] + +SearchSpaceNames = {'connect-nas' : CONNECT_NAS_BENCHMARK, + 'nas-bench-201': NAS_BENCH_201, + 'nas-bench-301': NAS_BENCH_201, + 'darts' : DARTS_SPACE} + + +class ReLUConvBN(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine, track_running_stats=True): + super(ReLUConvBN, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_out, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=not affine), + nn.BatchNorm2d(C_out, affine=affine, track_running_stats=track_running_stats) + ) + + def forward(self, x): + return self.op(x) + + +class SepConv(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine, track_running_stats=True): + super(SepConv, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=C_in, bias=False), + nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=not affine), + nn.BatchNorm2d(C_out, affine=affine, track_running_stats=track_running_stats), + ) + + def forward(self, x): + return self.op(x) + + +class DualSepConv(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine, track_running_stats=True): + super(DualSepConv, self).__init__() + self.op_a = SepConv(C_in, C_in , kernel_size, stride, padding, dilation, affine, track_running_stats) + self.op_b = SepConv(C_in, C_out, kernel_size, 1, padding, dilation, affine, track_running_stats) + + def forward(self, x): + x = self.op_a(x) + x = self.op_b(x) + return x + + +class ResNetBasicblock(nn.Module): + + def __init__(self, inplanes, planes, stride, affine=True): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + self.conv_a = ReLUConvBN(inplanes, planes, 3, stride, 1, 1, affine) + self.conv_b = ReLUConvBN( planes, planes, 3, 1, 1, 1, affine) + if stride == 2: + self.downsample = nn.Sequential( + nn.AvgPool2d(kernel_size=2, stride=2, padding=0), + nn.Conv2d(inplanes, planes, kernel_size=1, stride=1, padding=0, bias=False)) + elif inplanes != planes: + self.downsample = ReLUConvBN(inplanes, planes, 1, 1, 0, 1, affine) + else: + self.downsample = None + self.in_dim = inplanes + self.out_dim = planes + self.stride = stride + self.num_conv = 2 + + def extra_repr(self): + string = '{name}(inC={in_dim}, outC={out_dim}, stride={stride})'.format(name=self.__class__.__name__, **self.__dict__) + return string + + def forward(self, inputs): + + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + return residual + basicblock + + +class POOLING(nn.Module): + + def __init__(self, C_in, C_out, stride, mode, affine=True, track_running_stats=True): + super(POOLING, self).__init__() + if C_in == C_out: + self.preprocess = None + else: + self.preprocess = ReLUConvBN(C_in, C_out, 1, 1, 0, 1, affine, track_running_stats) + if mode == 'avg' : self.op = nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False) + elif mode == 'max': self.op = nn.MaxPool2d(3, stride=stride, padding=1) + else : raise ValueError('Invalid mode={:} in POOLING'.format(mode)) + + def forward(self, inputs): + if self.preprocess: x = self.preprocess(inputs) + else : x = inputs + return self.op(x) + + +class Identity(nn.Module): + + def __init__(self): + super(Identity, self).__init__() + + def forward(self, x): + return x + + +class Zero(nn.Module): + + def __init__(self, C_in, C_out, stride): + super(Zero, self).__init__() + self.C_in = C_in + self.C_out = C_out + self.stride = stride + self.is_zero = True + + def forward(self, x): + if self.C_in == self.C_out: + if self.stride == 1: return x.mul(0.) + else : return x[:,:,::self.stride,::self.stride].mul(0.) + else: + shape = list(x.shape) + shape[1] = self.C_out + zeros = x.new_zeros(shape, dtype=x.dtype, device=x.device) + return zeros + + def extra_repr(self): + return 'C_in={C_in}, C_out={C_out}, stride={stride}'.format(**self.__dict__) + + +class FactorizedReduce(nn.Module): + + def __init__(self, C_in, C_out, stride, affine, track_running_stats): + super(FactorizedReduce, self).__init__() + self.stride = stride + self.C_in = C_in + self.C_out = C_out + self.relu = nn.ReLU(inplace=False) + if stride == 2: + #assert C_out % 2 == 0, 'C_out : {:}'.format(C_out) + C_outs = [C_out // 2, C_out - C_out // 2] + self.convs = nn.ModuleList() + for i in range(2): + self.convs.append(nn.Conv2d(C_in, C_outs[i], 1, stride=stride, padding=0, bias=not affine)) + self.pad = nn.ConstantPad2d((0, 1, 0, 1), 0) + elif stride == 1: + self.conv = nn.Conv2d(C_in, C_out, 1, stride=stride, padding=0, bias=False) + else: + raise ValueError('Invalid stride : {:}'.format(stride)) + self.bn = nn.BatchNorm2d(C_out, affine=affine, track_running_stats=track_running_stats) + + def forward(self, x): + if self.stride == 2: + x = self.relu(x) + y = self.pad(x) + out = torch.cat([self.convs[0](x), self.convs[1](y[:,:,1:,1:])], dim=1) + else: + out = self.conv(x) + out = self.bn(out) + return out + + def extra_repr(self): + return 'C_in={C_in}, C_out={C_out}, stride={stride}'.format(**self.__dict__) + + +# Auto-ReID: Searching for a Part-Aware ConvNet for Person Re-Identification, ICCV 2019 +class PartAwareOp(nn.Module): + + def __init__(self, C_in, C_out, stride, part=4): + super().__init__() + self.part = 4 + self.hidden = C_in // 3 + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.local_conv_list = nn.ModuleList() + for i in range(self.part): + self.local_conv_list.append( + nn.Sequential(nn.ReLU(), nn.Conv2d(C_in, self.hidden, 1), nn.BatchNorm2d(self.hidden, affine=True)) + ) + self.W_K = nn.Linear(self.hidden, self.hidden) + self.W_Q = nn.Linear(self.hidden, self.hidden) + + if stride == 2 : self.last = FactorizedReduce(C_in + self.hidden, C_out, 2) + elif stride == 1: self.last = FactorizedReduce(C_in + self.hidden, C_out, 1) + else: raise ValueError('Invalid Stride : {:}'.format(stride)) + + def forward(self, x): + batch, C, H, W = x.size() + assert H >= self.part, 'input size too small : {:} vs {:}'.format(x.shape, self.part) + IHs = [0] + for i in range(self.part): IHs.append( min(H, int((i+1)*(float(H)/self.part))) ) + local_feat_list = [] + for i in range(self.part): + feature = x[:, :, IHs[i]:IHs[i+1], :] + xfeax = self.avg_pool(feature) + xfea = self.local_conv_list[i]( xfeax ) + local_feat_list.append( xfea ) + part_feature = torch.cat(local_feat_list, dim=2).view(batch, -1, self.part) + part_feature = part_feature.transpose(1,2).contiguous() + part_K = self.W_K(part_feature) + part_Q = self.W_Q(part_feature).transpose(1,2).contiguous() + weight_att = torch.bmm(part_K, part_Q) + attention = torch.softmax(weight_att, dim=2) + aggreateF = torch.bmm(attention, part_feature).transpose(1,2).contiguous() + features = [] + for i in range(self.part): + feature = aggreateF[:, :, i:i+1].expand(batch, self.hidden, IHs[i+1]-IHs[i]) + feature = feature.view(batch, self.hidden, IHs[i+1]-IHs[i], 1) + features.append( feature ) + features = torch.cat(features, dim=2).expand(batch, self.hidden, H, W) + final_fea = torch.cat((x,features), dim=1) + outputs = self.last( final_fea ) + return outputs + + +def drop_path(x, drop_prob): + if drop_prob > 0.: + keep_prob = 1. - drop_prob + mask = x.new_zeros(x.size(0), 1, 1, 1) + mask = mask.bernoulli_(keep_prob) + x = torch.div(x, keep_prob) + x.mul_(mask) + return x + + +# Searching for A Robust Neural Architecture in Four GPU Hours +class GDAS_Reduction_Cell(nn.Module): + + def __init__(self, C_prev_prev, C_prev, C, reduction_prev, multiplier, affine, track_running_stats): + super(GDAS_Reduction_Cell, self).__init__() + if reduction_prev: + self.preprocess0 = FactorizedReduce(C_prev_prev, C, 2, affine, track_running_stats) + else: + self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0, 1, affine, track_running_stats) + self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0, 1, affine, track_running_stats) + self.multiplier = multiplier + + self.reduction = True + self.ops1 = nn.ModuleList( + [nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C, C, (1, 3), stride=(1, 2), padding=(0, 1), groups=8, bias=False), + nn.Conv2d(C, C, (3, 1), stride=(2, 1), padding=(1, 0), groups=8, bias=False), + nn.BatchNorm2d(C, affine=True), + nn.ReLU(inplace=False), + nn.Conv2d(C, C, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(C, affine=True)), + nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C, C, (1, 3), stride=(1, 2), padding=(0, 1), groups=8, bias=False), + nn.Conv2d(C, C, (3, 1), stride=(2, 1), padding=(1, 0), groups=8, bias=False), + nn.BatchNorm2d(C, affine=True), + nn.ReLU(inplace=False), + nn.Conv2d(C, C, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(C, affine=True))]) + + self.ops2 = nn.ModuleList( + [nn.Sequential( + nn.MaxPool2d(3, stride=1, padding=1), + nn.BatchNorm2d(C, affine=True)), + nn.Sequential( + nn.MaxPool2d(3, stride=2, padding=1), + nn.BatchNorm2d(C, affine=True))]) + + def forward(self, s0, s1, drop_prob = -1): + s0 = self.preprocess0(s0) + s1 = self.preprocess1(s1) + + X0 = self.ops1[0] (s0) + X1 = self.ops1[1] (s1) + if self.training and drop_prob > 0.: + X0, X1 = drop_path(X0, drop_prob), drop_path(X1, drop_prob) + + #X2 = self.ops2[0] (X0+X1) + X2 = self.ops2[0] (s0) + X3 = self.ops2[1] (s1) + if self.training and drop_prob > 0.: + X2, X3 = drop_path(X2, drop_prob), drop_path(X3, drop_prob) + return torch.cat([X0, X1, X2, X3], dim=1) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_searchs/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_searchs/__init__.py new file mode 100644 index 0000000..df26f92 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_searchs/__init__.py @@ -0,0 +1,26 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +# The macro structure is defined in NAS-Bench-201 +# from .search_model_darts import TinyNetworkDarts +# from .search_model_gdas import TinyNetworkGDAS +# from .search_model_setn import TinyNetworkSETN +# from .search_model_enas import TinyNetworkENAS +# from .search_model_random import TinyNetworkRANDOM +# from .generic_model import GenericNAS201Model +from .genotypes import Structure as CellStructure, architectures as CellArchitectures +# NASNet-based macro structure +# from .search_model_gdas_nasnet import NASNetworkGDAS +# from .search_model_darts_nasnet import NASNetworkDARTS + + +# nas201_super_nets = {'DARTS-V1': TinyNetworkDarts, +# "DARTS-V2": TinyNetworkDarts, +# "GDAS": TinyNetworkGDAS, +# "SETN": TinyNetworkSETN, +# "ENAS": TinyNetworkENAS, +# "RANDOM": TinyNetworkRANDOM, +# "generic": GenericNAS201Model} + +# nasnet_super_nets = {"GDAS": NASNetworkGDAS, +# "DARTS": NASNetworkDARTS} diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_searchs/genotypes.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_searchs/genotypes.py new file mode 100644 index 0000000..b2b4091 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/cell_searchs/genotypes.py @@ -0,0 +1,198 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from copy import deepcopy + + +def get_combination(space, num): + combs = [] + for i in range(num): + if i == 0: + for func in space: + combs.append( [(func, i)] ) + else: + new_combs = [] + for string in combs: + for func in space: + xstring = string + [(func, i)] + new_combs.append( xstring ) + combs = new_combs + return combs + + +class Structure: + + def __init__(self, genotype): + assert isinstance(genotype, list) or isinstance(genotype, tuple), 'invalid class of genotype : {:}'.format(type(genotype)) + self.node_num = len(genotype) + 1 + self.nodes = [] + self.node_N = [] + for idx, node_info in enumerate(genotype): + assert isinstance(node_info, list) or isinstance(node_info, tuple), 'invalid class of node_info : {:}'.format(type(node_info)) + assert len(node_info) >= 1, 'invalid length : {:}'.format(len(node_info)) + for node_in in node_info: + assert isinstance(node_in, list) or isinstance(node_in, tuple), 'invalid class of in-node : {:}'.format(type(node_in)) + assert len(node_in) == 2 and node_in[1] <= idx, 'invalid in-node : {:}'.format(node_in) + self.node_N.append( len(node_info) ) + self.nodes.append( tuple(deepcopy(node_info)) ) + + def tolist(self, remove_str): + # convert this class to the list, if remove_str is 'none', then remove the 'none' operation. + # note that we re-order the input node in this function + # return the-genotype-list and success [if unsuccess, it is not a connectivity] + genotypes = [] + for node_info in self.nodes: + node_info = list( node_info ) + node_info = sorted(node_info, key=lambda x: (x[1], x[0])) + node_info = tuple(filter(lambda x: x[0] != remove_str, node_info)) + if len(node_info) == 0: return None, False + genotypes.append( node_info ) + return genotypes, True + + def node(self, index): + assert index > 0 and index <= len(self), 'invalid index={:} < {:}'.format(index, len(self)) + return self.nodes[index] + + def tostr(self): + strings = [] + for node_info in self.nodes: + string = '|'.join([x[0]+'~{:}'.format(x[1]) for x in node_info]) + string = '|{:}|'.format(string) + strings.append( string ) + return '+'.join(strings) + + def check_valid(self): + nodes = {0: True} + for i, node_info in enumerate(self.nodes): + sums = [] + for op, xin in node_info: + if op == 'none' or nodes[xin] is False: x = False + else: x = True + sums.append( x ) + nodes[i+1] = sum(sums) > 0 + return nodes[len(self.nodes)] + + def to_unique_str(self, consider_zero=False): + # this is used to identify the isomorphic cell, which rerquires the prior knowledge of operation + # two operations are special, i.e., none and skip_connect + nodes = {0: '0'} + for i_node, node_info in enumerate(self.nodes): + cur_node = [] + for op, xin in node_info: + if consider_zero is None: + x = '('+nodes[xin]+')' + '@{:}'.format(op) + elif consider_zero: + if op == 'none' or nodes[xin] == '#': x = '#' # zero + elif op == 'skip_connect': x = nodes[xin] + else: x = '('+nodes[xin]+')' + '@{:}'.format(op) + else: + if op == 'skip_connect': x = nodes[xin] + else: x = '('+nodes[xin]+')' + '@{:}'.format(op) + cur_node.append(x) + nodes[i_node+1] = '+'.join( sorted(cur_node) ) + return nodes[ len(self.nodes) ] + + def check_valid_op(self, op_names): + for node_info in self.nodes: + for inode_edge in node_info: + #assert inode_edge[0] in op_names, 'invalid op-name : {:}'.format(inode_edge[0]) + if inode_edge[0] not in op_names: return False + return True + + def __repr__(self): + return ('{name}({node_num} nodes with {node_info})'.format(name=self.__class__.__name__, node_info=self.tostr(), **self.__dict__)) + + def __len__(self): + return len(self.nodes) + 1 + + def __getitem__(self, index): + return self.nodes[index] + + @staticmethod + def str2structure(xstr): + if isinstance(xstr, Structure): return xstr + assert isinstance(xstr, str), 'must take string (not {:}) as input'.format(type(xstr)) + nodestrs = xstr.split('+') + genotypes = [] + for i, node_str in enumerate(nodestrs): + inputs = list(filter(lambda x: x != '', node_str.split('|'))) + for xinput in inputs: assert len(xinput.split('~')) == 2, 'invalid input length : {:}'.format(xinput) + inputs = ( xi.split('~') for xi in inputs ) + input_infos = tuple( (op, int(IDX)) for (op, IDX) in inputs) + genotypes.append( input_infos ) + return Structure( genotypes ) + + @staticmethod + def str2fullstructure(xstr, default_name='none'): + assert isinstance(xstr, str), 'must take string (not {:}) as input'.format(type(xstr)) + nodestrs = xstr.split('+') + genotypes = [] + for i, node_str in enumerate(nodestrs): + inputs = list(filter(lambda x: x != '', node_str.split('|'))) + for xinput in inputs: assert len(xinput.split('~')) == 2, 'invalid input length : {:}'.format(xinput) + inputs = ( xi.split('~') for xi in inputs ) + input_infos = list( (op, int(IDX)) for (op, IDX) in inputs) + all_in_nodes= list(x[1] for x in input_infos) + for j in range(i): + if j not in all_in_nodes: input_infos.append((default_name, j)) + node_info = sorted(input_infos, key=lambda x: (x[1], x[0])) + genotypes.append( tuple(node_info) ) + return Structure( genotypes ) + + @staticmethod + def gen_all(search_space, num, return_ori): + assert isinstance(search_space, list) or isinstance(search_space, tuple), 'invalid class of search-space : {:}'.format(type(search_space)) + assert num >= 2, 'There should be at least two nodes in a neural cell instead of {:}'.format(num) + all_archs = get_combination(search_space, 1) + for i, arch in enumerate(all_archs): + all_archs[i] = [ tuple(arch) ] + + for inode in range(2, num): + cur_nodes = get_combination(search_space, inode) + new_all_archs = [] + for previous_arch in all_archs: + for cur_node in cur_nodes: + new_all_archs.append( previous_arch + [tuple(cur_node)] ) + all_archs = new_all_archs + if return_ori: + return all_archs + else: + return [Structure(x) for x in all_archs] + + + +ResNet_CODE = Structure( + [(('nor_conv_3x3', 0), ), # node-1 + (('nor_conv_3x3', 1), ), # node-2 + (('skip_connect', 0), ('skip_connect', 2))] # node-3 + ) + +AllConv3x3_CODE = Structure( + [(('nor_conv_3x3', 0), ), # node-1 + (('nor_conv_3x3', 0), ('nor_conv_3x3', 1)), # node-2 + (('nor_conv_3x3', 0), ('nor_conv_3x3', 1), ('nor_conv_3x3', 2))] # node-3 + ) + +AllFull_CODE = Structure( + [(('skip_connect', 0), ('nor_conv_1x1', 0), ('nor_conv_3x3', 0), ('avg_pool_3x3', 0)), # node-1 + (('skip_connect', 0), ('nor_conv_1x1', 0), ('nor_conv_3x3', 0), ('avg_pool_3x3', 0), ('skip_connect', 1), ('nor_conv_1x1', 1), ('nor_conv_3x3', 1), ('avg_pool_3x3', 1)), # node-2 + (('skip_connect', 0), ('nor_conv_1x1', 0), ('nor_conv_3x3', 0), ('avg_pool_3x3', 0), ('skip_connect', 1), ('nor_conv_1x1', 1), ('nor_conv_3x3', 1), ('avg_pool_3x3', 1), ('skip_connect', 2), ('nor_conv_1x1', 2), ('nor_conv_3x3', 2), ('avg_pool_3x3', 2))] # node-3 + ) + +AllConv1x1_CODE = Structure( + [(('nor_conv_1x1', 0), ), # node-1 + (('nor_conv_1x1', 0), ('nor_conv_1x1', 1)), # node-2 + (('nor_conv_1x1', 0), ('nor_conv_1x1', 1), ('nor_conv_1x1', 2))] # node-3 + ) + +AllIdentity_CODE = Structure( + [(('skip_connect', 0), ), # node-1 + (('skip_connect', 0), ('skip_connect', 1)), # node-2 + (('skip_connect', 0), ('skip_connect', 1), ('skip_connect', 2))] # node-3 + ) + +architectures = {'resnet' : ResNet_CODE, + 'all_c3x3': AllConv3x3_CODE, + 'all_c1x1': AllConv1x1_CODE, + 'all_idnt': AllIdentity_CODE, + 'all_full': AllFull_CODE} diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet.py new file mode 100644 index 0000000..a6524d6 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet.py @@ -0,0 +1,167 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +import torch.nn.functional as F +from ..initialization import initialize_resnet + + +class ConvBNReLU(nn.Module): + + def __init__(self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu): + super(ConvBNReLU, self).__init__() + if has_avg : self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) + else : self.avg = None + self.conv = nn.Conv2d(nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, dilation=1, groups=1, bias=bias) + if has_bn : self.bn = nn.BatchNorm2d(nOut) + else : self.bn = None + if has_relu: self.relu = nn.ReLU(inplace=True) + else : self.relu = None + + def forward(self, inputs): + if self.avg : out = self.avg( inputs ) + else : out = inputs + conv = self.conv( out ) + if self.bn : out = self.bn( conv ) + else : out = conv + if self.relu: out = self.relu( out ) + else : out = out + + return out + + +class ResNetBasicblock(nn.Module): + num_conv = 2 + expansion = 1 + def __init__(self, iCs, stride): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 3,'invalid lengths of iCs : {:}'.format(iCs) + + self.conv_a = ConvBNReLU(iCs[0], iCs[1], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_b = ConvBNReLU(iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=True, has_bn=False, has_relu=False) + residual_in = iCs[2] + elif iCs[0] != iCs[2]: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=False,has_bn=True , has_relu=False) + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[2]) + self.out_dim = iCs[2] + + def forward(self, inputs): + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + basicblock + return F.relu(out, inplace=True) + + + +class ResNetBottleneck(nn.Module): + expansion = 4 + num_conv = 3 + def __init__(self, iCs, stride): + super(ResNetBottleneck, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 4,'invalid lengths of iCs : {:}'.format(iCs) + self.conv_1x1 = ConvBNReLU(iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_3x3 = ConvBNReLU(iCs[1], iCs[2], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_1x4 = ConvBNReLU(iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=True , has_bn=False, has_relu=False) + residual_in = iCs[3] + elif iCs[0] != iCs[3]: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=False, has_relu=False) + residual_in = iCs[3] + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[3]) + self.out_dim = iCs[3] + + def forward(self, inputs): + + bottleneck = self.conv_1x1(inputs) + bottleneck = self.conv_3x3(bottleneck) + bottleneck = self.conv_1x4(bottleneck) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + bottleneck + return F.relu(out, inplace=True) + + + +class InferCifarResNet(nn.Module): + + def __init__(self, block_name, depth, xblocks, xchannels, num_classes, zero_init_residual): + super(InferCifarResNet, self).__init__() + + #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + if block_name == 'ResNetBasicblock': + block = ResNetBasicblock + assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' + layer_blocks = (depth - 2) // 6 + elif block_name == 'ResNetBottleneck': + block = ResNetBottleneck + assert (depth - 2) % 9 == 0, 'depth should be one of 164' + layer_blocks = (depth - 2) // 9 + else: + raise ValueError('invalid block : {:}'.format(block_name)) + assert len(xblocks) == 3, 'invalid xblocks : {:}'.format(xblocks) + + self.message = 'InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}'.format(depth, layer_blocks) + self.num_classes = num_classes + self.xchannels = xchannels + self.layers = nn.ModuleList( [ ConvBNReLU(xchannels[0], xchannels[1], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True) ] ) + last_channel_idx = 1 + for stage in range(3): + for iL in range(layer_blocks): + num_conv = block.num_conv + iCs = self.xchannels[last_channel_idx:last_channel_idx+num_conv+1] + stride = 2 if stage > 0 and iL == 0 else 1 + module = block(iCs, stride) + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + self.layers.append ( module ) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(stage, iL, layer_blocks, len(self.layers)-1, iCs, module.out_dim, stride) + if iL + 1 == xblocks[stage]: # reach the maximum depth + out_channel = module.out_dim + for iiL in range(iL+1, layer_blocks): + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + break + + self.avgpool = nn.AvgPool2d(8) + self.classifier = nn.Linear(self.xchannels[-1], num_classes) + + self.apply(initialize_resnet) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, ResNetBasicblock): + nn.init.constant_(m.conv_b.bn.weight, 0) + elif isinstance(m, ResNetBottleneck): + nn.init.constant_(m.conv_1x4.bn.weight, 0) + + def get_message(self): + return self.message + + def forward(self, inputs): + x = inputs + for i, layer in enumerate(self.layers): + x = layer( x ) + features = self.avgpool(x) + features = features.view(features.size(0), -1) + logits = self.classifier(features) + return features, logits diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_depth.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_depth.py new file mode 100644 index 0000000..d773fc5 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_depth.py @@ -0,0 +1,150 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +import torch.nn.functional as F +from ..initialization import initialize_resnet + + +class ConvBNReLU(nn.Module): + + def __init__(self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu): + super(ConvBNReLU, self).__init__() + if has_avg : self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) + else : self.avg = None + self.conv = nn.Conv2d(nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, dilation=1, groups=1, bias=bias) + if has_bn : self.bn = nn.BatchNorm2d(nOut) + else : self.bn = None + if has_relu: self.relu = nn.ReLU(inplace=True) + else : self.relu = None + + def forward(self, inputs): + if self.avg : out = self.avg( inputs ) + else : out = inputs + conv = self.conv( out ) + if self.bn : out = self.bn( conv ) + else : out = conv + if self.relu: out = self.relu( out ) + else : out = out + + return out + + +class ResNetBasicblock(nn.Module): + num_conv = 2 + expansion = 1 + def __init__(self, inplanes, planes, stride): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + + self.conv_a = ConvBNReLU(inplanes, planes, 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_b = ConvBNReLU( planes, planes, 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False) + if stride == 2: + self.downsample = ConvBNReLU(inplanes, planes, 1, 1, 0, False, has_avg=True, has_bn=False, has_relu=False) + elif inplanes != planes: + self.downsample = ConvBNReLU(inplanes, planes, 1, 1, 0, False, has_avg=False,has_bn=True , has_relu=False) + else: + self.downsample = None + self.out_dim = planes + + def forward(self, inputs): + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + basicblock + return F.relu(out, inplace=True) + + + +class ResNetBottleneck(nn.Module): + expansion = 4 + num_conv = 3 + def __init__(self, inplanes, planes, stride): + super(ResNetBottleneck, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + self.conv_1x1 = ConvBNReLU(inplanes, planes, 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_3x3 = ConvBNReLU( planes, planes, 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_1x4 = ConvBNReLU(planes, planes*self.expansion, 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + if stride == 2: + self.downsample = ConvBNReLU(inplanes, planes*self.expansion, 1, 1, 0, False, has_avg=True , has_bn=False, has_relu=False) + elif inplanes != planes*self.expansion: + self.downsample = ConvBNReLU(inplanes, planes*self.expansion, 1, 1, 0, False, has_avg=False, has_bn=False, has_relu=False) + else: + self.downsample = None + self.out_dim = planes*self.expansion + + def forward(self, inputs): + + bottleneck = self.conv_1x1(inputs) + bottleneck = self.conv_3x3(bottleneck) + bottleneck = self.conv_1x4(bottleneck) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + bottleneck + return F.relu(out, inplace=True) + + + +class InferDepthCifarResNet(nn.Module): + + def __init__(self, block_name, depth, xblocks, num_classes, zero_init_residual): + super(InferDepthCifarResNet, self).__init__() + + #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + if block_name == 'ResNetBasicblock': + block = ResNetBasicblock + assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' + layer_blocks = (depth - 2) // 6 + elif block_name == 'ResNetBottleneck': + block = ResNetBottleneck + assert (depth - 2) % 9 == 0, 'depth should be one of 164' + layer_blocks = (depth - 2) // 9 + else: + raise ValueError('invalid block : {:}'.format(block_name)) + assert len(xblocks) == 3, 'invalid xblocks : {:}'.format(xblocks) + + self.message = 'InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}'.format(depth, layer_blocks) + self.num_classes = num_classes + self.layers = nn.ModuleList( [ ConvBNReLU(3, 16, 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True) ] ) + self.channels = [16] + for stage in range(3): + for iL in range(layer_blocks): + iC = self.channels[-1] + planes = 16 * (2**stage) + stride = 2 if stage > 0 and iL == 0 else 1 + module = block(iC, planes, stride) + self.channels.append( module.out_dim ) + self.layers.append ( module ) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iC={:}, oC={:3d}, stride={:}".format(stage, iL, layer_blocks, len(self.layers)-1, planes, module.out_dim, stride) + if iL + 1 == xblocks[stage]: # reach the maximum depth + break + + self.avgpool = nn.AvgPool2d(8) + self.classifier = nn.Linear(self.channels[-1], num_classes) + + self.apply(initialize_resnet) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, ResNetBasicblock): + nn.init.constant_(m.conv_b.bn.weight, 0) + elif isinstance(m, ResNetBottleneck): + nn.init.constant_(m.conv_1x4.bn.weight, 0) + + def get_message(self): + return self.message + + def forward(self, inputs): + x = inputs + for i, layer in enumerate(self.layers): + x = layer( x ) + features = self.avgpool(x) + features = features.view(features.size(0), -1) + logits = self.classifier(features) + return features, logits diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_width.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_width.py new file mode 100644 index 0000000..7183875 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_width.py @@ -0,0 +1,160 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +import torch.nn.functional as F +from ..initialization import initialize_resnet + + +class ConvBNReLU(nn.Module): + + def __init__(self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu): + super(ConvBNReLU, self).__init__() + if has_avg : self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) + else : self.avg = None + self.conv = nn.Conv2d(nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, dilation=1, groups=1, bias=bias) + if has_bn : self.bn = nn.BatchNorm2d(nOut) + else : self.bn = None + if has_relu: self.relu = nn.ReLU(inplace=True) + else : self.relu = None + + def forward(self, inputs): + if self.avg : out = self.avg( inputs ) + else : out = inputs + conv = self.conv( out ) + if self.bn : out = self.bn( conv ) + else : out = conv + if self.relu: out = self.relu( out ) + else : out = out + + return out + + +class ResNetBasicblock(nn.Module): + num_conv = 2 + expansion = 1 + def __init__(self, iCs, stride): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 3,'invalid lengths of iCs : {:}'.format(iCs) + + self.conv_a = ConvBNReLU(iCs[0], iCs[1], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_b = ConvBNReLU(iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=True, has_bn=False, has_relu=False) + residual_in = iCs[2] + elif iCs[0] != iCs[2]: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=False,has_bn=True , has_relu=False) + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[2]) + self.out_dim = iCs[2] + + def forward(self, inputs): + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + basicblock + return F.relu(out, inplace=True) + + + +class ResNetBottleneck(nn.Module): + expansion = 4 + num_conv = 3 + def __init__(self, iCs, stride): + super(ResNetBottleneck, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 4,'invalid lengths of iCs : {:}'.format(iCs) + self.conv_1x1 = ConvBNReLU(iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_3x3 = ConvBNReLU(iCs[1], iCs[2], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_1x4 = ConvBNReLU(iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=True , has_bn=False, has_relu=False) + residual_in = iCs[3] + elif iCs[0] != iCs[3]: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=False, has_relu=False) + residual_in = iCs[3] + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[3]) + self.out_dim = iCs[3] + + def forward(self, inputs): + + bottleneck = self.conv_1x1(inputs) + bottleneck = self.conv_3x3(bottleneck) + bottleneck = self.conv_1x4(bottleneck) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + bottleneck + return F.relu(out, inplace=True) + + + +class InferWidthCifarResNet(nn.Module): + + def __init__(self, block_name, depth, xchannels, num_classes, zero_init_residual): + super(InferWidthCifarResNet, self).__init__() + + #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + if block_name == 'ResNetBasicblock': + block = ResNetBasicblock + assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' + layer_blocks = (depth - 2) // 6 + elif block_name == 'ResNetBottleneck': + block = ResNetBottleneck + assert (depth - 2) % 9 == 0, 'depth should be one of 164' + layer_blocks = (depth - 2) // 9 + else: + raise ValueError('invalid block : {:}'.format(block_name)) + + self.message = 'InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}'.format(depth, layer_blocks) + self.num_classes = num_classes + self.xchannels = xchannels + self.layers = nn.ModuleList( [ ConvBNReLU(xchannels[0], xchannels[1], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True) ] ) + last_channel_idx = 1 + for stage in range(3): + for iL in range(layer_blocks): + num_conv = block.num_conv + iCs = self.xchannels[last_channel_idx:last_channel_idx+num_conv+1] + stride = 2 if stage > 0 and iL == 0 else 1 + module = block(iCs, stride) + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + self.layers.append ( module ) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(stage, iL, layer_blocks, len(self.layers)-1, iCs, module.out_dim, stride) + + self.avgpool = nn.AvgPool2d(8) + self.classifier = nn.Linear(self.xchannels[-1], num_classes) + + self.apply(initialize_resnet) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, ResNetBasicblock): + nn.init.constant_(m.conv_b.bn.weight, 0) + elif isinstance(m, ResNetBottleneck): + nn.init.constant_(m.conv_1x4.bn.weight, 0) + + def get_message(self): + return self.message + + def forward(self, inputs): + x = inputs + for i, layer in enumerate(self.layers): + x = layer( x ) + features = self.avgpool(x) + features = features.view(features.size(0), -1) + logits = self.classifier(features) + return features, logits diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferImagenetResNet.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferImagenetResNet.py new file mode 100644 index 0000000..8f06db7 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferImagenetResNet.py @@ -0,0 +1,170 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +import torch.nn.functional as F +from ..initialization import initialize_resnet + + +class ConvBNReLU(nn.Module): + + num_conv = 1 + def __init__(self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu): + super(ConvBNReLU, self).__init__() + if has_avg : self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) + else : self.avg = None + self.conv = nn.Conv2d(nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, dilation=1, groups=1, bias=bias) + if has_bn : self.bn = nn.BatchNorm2d(nOut) + else : self.bn = None + if has_relu: self.relu = nn.ReLU(inplace=True) + else : self.relu = None + + def forward(self, inputs): + if self.avg : out = self.avg( inputs ) + else : out = inputs + conv = self.conv( out ) + if self.bn : out = self.bn( conv ) + else : out = conv + if self.relu: out = self.relu( out ) + else : out = out + + return out + + +class ResNetBasicblock(nn.Module): + num_conv = 2 + expansion = 1 + def __init__(self, iCs, stride): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 3,'invalid lengths of iCs : {:}'.format(iCs) + + self.conv_a = ConvBNReLU(iCs[0], iCs[1], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_b = ConvBNReLU(iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=True, has_bn=True, has_relu=False) + residual_in = iCs[2] + elif iCs[0] != iCs[2]: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=False,has_bn=True , has_relu=False) + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[2]) + self.out_dim = iCs[2] + + def forward(self, inputs): + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + basicblock + return F.relu(out, inplace=True) + + + +class ResNetBottleneck(nn.Module): + expansion = 4 + num_conv = 3 + def __init__(self, iCs, stride): + super(ResNetBottleneck, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 4,'invalid lengths of iCs : {:}'.format(iCs) + self.conv_1x1 = ConvBNReLU(iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_3x3 = ConvBNReLU(iCs[1], iCs[2], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_1x4 = ConvBNReLU(iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=True , has_bn=True, has_relu=False) + residual_in = iCs[3] + elif iCs[0] != iCs[3]: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[3] + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[3]) + self.out_dim = iCs[3] + + def forward(self, inputs): + + bottleneck = self.conv_1x1(inputs) + bottleneck = self.conv_3x3(bottleneck) + bottleneck = self.conv_1x4(bottleneck) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + bottleneck + return F.relu(out, inplace=True) + + + +class InferImagenetResNet(nn.Module): + + def __init__(self, block_name, layers, xblocks, xchannels, deep_stem, num_classes, zero_init_residual): + super(InferImagenetResNet, self).__init__() + + #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + if block_name == 'BasicBlock': + block = ResNetBasicblock + elif block_name == 'Bottleneck': + block = ResNetBottleneck + else: + raise ValueError('invalid block : {:}'.format(block_name)) + assert len(xblocks) == len(layers), 'invalid layers : {:} vs xblocks : {:}'.format(layers, xblocks) + + self.message = 'InferImagenetResNet : Depth : {:} -> {:}, Layers for each block : {:}'.format(sum(layers)*block.num_conv, sum(xblocks)*block.num_conv, xblocks) + self.num_classes = num_classes + self.xchannels = xchannels + if not deep_stem: + self.layers = nn.ModuleList( [ ConvBNReLU(xchannels[0], xchannels[1], 7, 2, 3, False, has_avg=False, has_bn=True, has_relu=True) ] ) + last_channel_idx = 1 + else: + self.layers = nn.ModuleList( [ ConvBNReLU(xchannels[0], xchannels[1], 3, 2, 1, False, has_avg=False, has_bn=True, has_relu=True) + ,ConvBNReLU(xchannels[1], xchannels[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True) ] ) + last_channel_idx = 2 + self.layers.append( nn.MaxPool2d(kernel_size=3, stride=2, padding=1) ) + for stage, layer_blocks in enumerate(layers): + for iL in range(layer_blocks): + num_conv = block.num_conv + iCs = self.xchannels[last_channel_idx:last_channel_idx+num_conv+1] + stride = 2 if stage > 0 and iL == 0 else 1 + module = block(iCs, stride) + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + self.layers.append ( module ) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(stage, iL, layer_blocks, len(self.layers)-1, iCs, module.out_dim, stride) + if iL + 1 == xblocks[stage]: # reach the maximum depth + out_channel = module.out_dim + for iiL in range(iL+1, layer_blocks): + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + break + assert last_channel_idx + 1 == len(self.xchannels), '{:} vs {:}'.format(last_channel_idx, len(self.xchannels)) + self.avgpool = nn.AdaptiveAvgPool2d((1,1)) + self.classifier = nn.Linear(self.xchannels[-1], num_classes) + + self.apply(initialize_resnet) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, ResNetBasicblock): + nn.init.constant_(m.conv_b.bn.weight, 0) + elif isinstance(m, ResNetBottleneck): + nn.init.constant_(m.conv_1x4.bn.weight, 0) + + def get_message(self): + return self.message + + def forward(self, inputs): + x = inputs + for i, layer in enumerate(self.layers): + x = layer( x ) + features = self.avgpool(x) + features = features.view(features.size(0), -1) + logits = self.classifier(features) + return features, logits diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferMobileNetV2.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferMobileNetV2.py new file mode 100644 index 0000000..d072b99 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferMobileNetV2.py @@ -0,0 +1,122 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +# MobileNetV2: Inverted Residuals and Linear Bottlenecks, CVPR 2018 +from torch import nn +from ..initialization import initialize_resnet +from ..SharedUtils import parse_channel_info + + +class ConvBNReLU(nn.Module): + def __init__(self, in_planes, out_planes, kernel_size, stride, groups, has_bn=True, has_relu=True): + super(ConvBNReLU, self).__init__() + padding = (kernel_size - 1) // 2 + self.conv = nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False) + if has_bn: self.bn = nn.BatchNorm2d(out_planes) + else : self.bn = None + if has_relu: self.relu = nn.ReLU6(inplace=True) + else : self.relu = None + + def forward(self, x): + out = self.conv( x ) + if self.bn: out = self.bn ( out ) + if self.relu: out = self.relu( out ) + return out + + +class InvertedResidual(nn.Module): + def __init__(self, channels, stride, expand_ratio, additive): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2], 'invalid stride : {:}'.format(stride) + assert len(channels) in [2, 3], 'invalid channels : {:}'.format(channels) + + if len(channels) == 2: + layers = [] + else: + layers = [ConvBNReLU(channels[0], channels[1], 1, 1, 1)] + layers.extend([ + # dw + ConvBNReLU(channels[-2], channels[-2], 3, stride, channels[-2]), + # pw-linear + ConvBNReLU(channels[-2], channels[-1], 1, 1, 1, True, False), + ]) + self.conv = nn.Sequential(*layers) + self.additive = additive + if self.additive and channels[0] != channels[-1]: + self.shortcut = ConvBNReLU(channels[0], channels[-1], 1, 1, 1, True, False) + else: + self.shortcut = None + self.out_dim = channels[-1] + + def forward(self, x): + out = self.conv(x) + # if self.additive: return additive_func(out, x) + if self.shortcut: return out + self.shortcut(x) + else : return out + + +class InferMobileNetV2(nn.Module): + def __init__(self, num_classes, xchannels, xblocks, dropout): + super(InferMobileNetV2, self).__init__() + block = InvertedResidual + inverted_residual_setting = [ + # t, c, n, s + [1, 16 , 1, 1], + [6, 24 , 2, 2], + [6, 32 , 3, 2], + [6, 64 , 4, 2], + [6, 96 , 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + assert len(inverted_residual_setting) == len(xblocks), 'invalid number of layers : {:} vs {:}'.format(len(inverted_residual_setting), len(xblocks)) + for block_num, ir_setting in zip(xblocks, inverted_residual_setting): + assert block_num <= ir_setting[2], '{:} vs {:}'.format(block_num, ir_setting) + xchannels = parse_channel_info(xchannels) + #for i, chs in enumerate(xchannels): + # if i > 0: assert chs[0] == xchannels[i-1][-1], 'Layer[{:}] is invalid {:} vs {:}'.format(i, xchannels[i-1], chs) + self.xchannels = xchannels + self.message = 'InferMobileNetV2 : xblocks={:}'.format(xblocks) + # building first layer + features = [ConvBNReLU(xchannels[0][0], xchannels[0][1], 3, 2, 1)] + last_channel_idx = 1 + + # building inverted residual blocks + for stage, (t, c, n, s) in enumerate(inverted_residual_setting): + for i in range(n): + stride = s if i == 0 else 1 + additv = True if i > 0 else False + module = block(self.xchannels[last_channel_idx], stride, t, additv) + features.append(module) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, Cs={:}, stride={:}, expand={:}, original-C={:}".format(stage, i, n, len(features), self.xchannels[last_channel_idx], stride, t, c) + last_channel_idx += 1 + if i + 1 == xblocks[stage]: + out_channel = module.out_dim + for iiL in range(i+1, n): + last_channel_idx += 1 + self.xchannels[last_channel_idx][0] = module.out_dim + break + # building last several layers + features.append(ConvBNReLU(self.xchannels[last_channel_idx][0], self.xchannels[last_channel_idx][1], 1, 1, 1)) + assert last_channel_idx + 2 == len(self.xchannels), '{:} vs {:}'.format(last_channel_idx, len(self.xchannels)) + # make it nn.Sequential + self.features = nn.Sequential(*features) + + # building classifier + self.classifier = nn.Sequential( + nn.Dropout(dropout), + nn.Linear(self.xchannels[last_channel_idx][1], num_classes), + ) + + # weight initialization + self.apply( initialize_resnet ) + + def get_message(self): + return self.message + + def forward(self, inputs): + features = self.features(inputs) + vectors = features.mean([2, 3]) + predicts = self.classifier(vectors) + return features, predicts diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferTinyCellNet.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferTinyCellNet.py new file mode 100644 index 0000000..d92c222 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/InferTinyCellNet.py @@ -0,0 +1,58 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +from typing import List, Text, Any +import torch.nn as nn +from models.cell_operations import ResNetBasicblock +from models.cell_infers.cells import InferCell + + +class DynamicShapeTinyNet(nn.Module): + + def __init__(self, channels: List[int], genotype: Any, num_classes: int): + super(DynamicShapeTinyNet, self).__init__() + self._channels = channels + if len(channels) % 3 != 2: + raise ValueError('invalid number of layers : {:}'.format(len(channels))) + self._num_stage = N = len(channels) // 3 + + self.stem = nn.Sequential( + nn.Conv2d(3, channels[0], kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(channels[0])) + + # layer_channels = [C ] * N + [C*2 ] + [C*2 ] * N + [C*4 ] + [C*4 ] * N + layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N + + c_prev = channels[0] + self.cells = nn.ModuleList() + for index, (c_curr, reduction) in enumerate(zip(channels, layer_reductions)): + if reduction : cell = ResNetBasicblock(c_prev, c_curr, 2, True) + else : cell = InferCell(genotype, c_prev, c_curr, 1) + self.cells.append( cell ) + c_prev = cell.out_dim + self._num_layer = len(self.cells) + + self.lastact = nn.Sequential(nn.BatchNorm2d(c_prev), nn.ReLU(inplace=True)) + self.global_pooling = nn.AdaptiveAvgPool2d(1) + self.classifier = nn.Linear(c_prev, num_classes) + + def get_message(self) -> Text: + string = self.extra_repr() + for i, cell in enumerate(self.cells): + string += '\n {:02d}/{:02d} :: {:}'.format(i, len(self.cells), cell.extra_repr()) + return string + + def extra_repr(self): + return ('{name}(C={_channels}, N={_num_stage}, L={_num_layer})'.format(name=self.__class__.__name__, **self.__dict__)) + + def forward(self, inputs): + feature = self.stem(inputs) + for i, cell in enumerate(self.cells): + feature = cell(feature) + + out = self.lastact(feature) + out = self.global_pooling( out ) + out = out.view(out.size(0), -1) + logits = self.classifier(out) + + return out, logits diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/__init__.py new file mode 100644 index 0000000..0f6cf36 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/__init__.py @@ -0,0 +1,9 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +from .InferCifarResNet_width import InferWidthCifarResNet +from .InferImagenetResNet import InferImagenetResNet +from .InferCifarResNet_depth import InferDepthCifarResNet +from .InferCifarResNet import InferCifarResNet +from .InferMobileNetV2 import InferMobileNetV2 +from .InferTinyCellNet import DynamicShapeTinyNet \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/shared_utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/shared_utils.py new file mode 100644 index 0000000..c29620c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nas_bench_201_models/shape_infers/shared_utils.py @@ -0,0 +1,5 @@ +def parse_channel_info(xstring): + blocks = xstring.split(' ') + blocks = [x.split('-') for x in blocks] + blocks = [[int(_) for _ in x] for x in blocks] + return blocks diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/__init__.py new file mode 100644 index 0000000..61cc68f --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/__init__.py @@ -0,0 +1,2 @@ +from .evaluation_utils import obtain_accuracy +from .flop_benchmark import get_model_infos diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/evaluation_utils.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/evaluation_utils.py new file mode 100644 index 0000000..78733d9 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/evaluation_utils.py @@ -0,0 +1,17 @@ +import torch + +def obtain_accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + # correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/flop_benchmark.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/flop_benchmark.py new file mode 100644 index 0000000..133cf2c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/nasbench_utils/flop_benchmark.py @@ -0,0 +1,181 @@ +import torch +import torch.nn as nn +import numpy as np + + +def count_parameters_in_MB(model): + if isinstance(model, nn.Module): + return np.sum(np.prod(v.size()) for v in model.parameters())/1e6 + else: + return np.sum(np.prod(v.size()) for v in model)/1e6 + + +def get_model_infos(model, shape): + #model = copy.deepcopy( model ) + + model = add_flops_counting_methods(model) + #model = model.cuda() + model.eval() + + #cache_inputs = torch.zeros(*shape).cuda() + #cache_inputs = torch.zeros(*shape) + cache_inputs = torch.rand(*shape) + if next(model.parameters()).is_cuda: cache_inputs = cache_inputs.cuda() + #print_log('In the calculating function : cache input size : {:}'.format(cache_inputs.size()), log) + with torch.no_grad(): + _____ = model(cache_inputs) + FLOPs = compute_average_flops_cost( model ) / 1e6 + Param = count_parameters_in_MB(model) + + if hasattr(model, 'auxiliary_param'): + aux_params = count_parameters_in_MB(model.auxiliary_param()) + print ('The auxiliary params of this model is : {:}'.format(aux_params)) + print ('We remove the auxiliary params from the total params ({:}) when counting'.format(Param)) + Param = Param - aux_params + + #print_log('FLOPs : {:} MB'.format(FLOPs), log) + torch.cuda.empty_cache() + model.apply( remove_hook_function ) + return FLOPs, Param + + +# ---- Public functions +def add_flops_counting_methods( model ): + model.__batch_counter__ = 0 + add_batch_counter_hook_function( model ) + model.apply( add_flops_counter_variable_or_reset ) + model.apply( add_flops_counter_hook_function ) + return model + + + +def compute_average_flops_cost(model): + """ + A method that will be available after add_flops_counting_methods() is called on a desired net object. + Returns current mean flops consumption per image. + """ + batches_count = model.__batch_counter__ + flops_sum = 0 + #or isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d) \ + for module in model.modules(): + if isinstance(module, torch.nn.Conv2d) or isinstance(module, torch.nn.Linear) \ + or isinstance(module, torch.nn.Conv1d) \ + or hasattr(module, 'calculate_flop_self'): + flops_sum += module.__flops__ + return flops_sum / batches_count + + +# ---- Internal functions +def pool_flops_counter_hook(pool_module, inputs, output): + batch_size = inputs[0].size(0) + kernel_size = pool_module.kernel_size + out_C, output_height, output_width = output.shape[1:] + assert out_C == inputs[0].size(1), '{:} vs. {:}'.format(out_C, inputs[0].size()) + + overall_flops = batch_size * out_C * output_height * output_width * kernel_size * kernel_size + pool_module.__flops__ += overall_flops + + +def self_calculate_flops_counter_hook(self_module, inputs, output): + overall_flops = self_module.calculate_flop_self(inputs[0].shape, output.shape) + self_module.__flops__ += overall_flops + + +def fc_flops_counter_hook(fc_module, inputs, output): + batch_size = inputs[0].size(0) + xin, xout = fc_module.in_features, fc_module.out_features + assert xin == inputs[0].size(1) and xout == output.size(1), 'IO=({:}, {:})'.format(xin, xout) + overall_flops = batch_size * xin * xout + if fc_module.bias is not None: + overall_flops += batch_size * xout + fc_module.__flops__ += overall_flops + + +def conv1d_flops_counter_hook(conv_module, inputs, outputs): + batch_size = inputs[0].size(0) + outL = outputs.shape[-1] + [kernel] = conv_module.kernel_size + in_channels = conv_module.in_channels + out_channels = conv_module.out_channels + groups = conv_module.groups + conv_per_position_flops = kernel * in_channels * out_channels / groups + + active_elements_count = batch_size * outL + overall_flops = conv_per_position_flops * active_elements_count + + if conv_module.bias is not None: + overall_flops += out_channels * active_elements_count + conv_module.__flops__ += overall_flops + + +def conv2d_flops_counter_hook(conv_module, inputs, output): + batch_size = inputs[0].size(0) + output_height, output_width = output.shape[2:] + + kernel_height, kernel_width = conv_module.kernel_size + in_channels = conv_module.in_channels + out_channels = conv_module.out_channels + groups = conv_module.groups + conv_per_position_flops = kernel_height * kernel_width * in_channels * out_channels / groups + + active_elements_count = batch_size * output_height * output_width + overall_flops = conv_per_position_flops * active_elements_count + + if conv_module.bias is not None: + overall_flops += out_channels * active_elements_count + conv_module.__flops__ += overall_flops + + +def batch_counter_hook(module, inputs, output): + # Can have multiple inputs, getting the first one + inputs = inputs[0] + batch_size = inputs.shape[0] + module.__batch_counter__ += batch_size + + +def add_batch_counter_hook_function(module): + if not hasattr(module, '__batch_counter_handle__'): + handle = module.register_forward_hook(batch_counter_hook) + module.__batch_counter_handle__ = handle + + +def add_flops_counter_variable_or_reset(module): + if isinstance(module, torch.nn.Conv2d) or isinstance(module, torch.nn.Linear) \ + or isinstance(module, torch.nn.Conv1d) \ + or isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d) \ + or hasattr(module, 'calculate_flop_self'): + module.__flops__ = 0 + + +def add_flops_counter_hook_function(module): + if isinstance(module, torch.nn.Conv2d): + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(conv2d_flops_counter_hook) + module.__flops_handle__ = handle + elif isinstance(module, torch.nn.Conv1d): + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(conv1d_flops_counter_hook) + module.__flops_handle__ = handle + elif isinstance(module, torch.nn.Linear): + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(fc_flops_counter_hook) + module.__flops_handle__ = handle + elif isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d): + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(pool_flops_counter_hook) + module.__flops_handle__ = handle + elif hasattr(module, 'calculate_flop_self'): # self-defined module + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(self_calculate_flops_counter_hook) + module.__flops_handle__ = handle + + +def remove_hook_function(module): + hookers = ['__batch_counter_handle__', '__flops_handle__'] + for hooker in hookers: + if hasattr(module, hooker): + handle = getattr(module, hooker) + handle.remove() + keys = ['__flops__', '__batch_counter__', '__flops__'] + hookers + for ckey in keys: + if hasattr(module, ckey): delattr(module, ckey) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/__init__.py new file mode 100644 index 0000000..df1c298 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/__init__.py @@ -0,0 +1,28 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from .starts import get_machine_info, save_checkpoint, copy_checkpoint +from .optimizers import get_optim_scheduler +from .starts import prepare_seed #, prepare_logger, get_machine_info, save_checkpoint, copy_checkpoint +''' +from .funcs_nasbench import evaluate_for_seed as bench_evaluate_for_seed +from .funcs_nasbench import pure_evaluate as bench_pure_evaluate +from .funcs_nasbench import get_nas_bench_loaders + +def get_procedures(procedure): + from .basic_main import basic_train, basic_valid + from .search_main import search_train, search_valid + from .search_main_v2 import search_train_v2 + from .simple_KD_main import simple_KD_train, simple_KD_valid + + train_funcs = {'basic' : basic_train, \ + 'search': search_train,'Simple-KD': simple_KD_train, \ + 'search-v2': search_train_v2} + valid_funcs = {'basic' : basic_valid, \ + 'search': search_valid,'Simple-KD': simple_KD_valid, \ + 'search-v2': search_valid} + + train_func = train_funcs[procedure] + valid_func = valid_funcs[procedure] + return train_func, valid_func +''' diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/optimizers.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/optimizers.py new file mode 100644 index 0000000..7fe086d --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/optimizers.py @@ -0,0 +1,204 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import math, torch +import torch.nn as nn +from bisect import bisect_right +from torch.optim import Optimizer + + +class _LRScheduler(object): + + def __init__(self, optimizer, warmup_epochs, epochs): + if not isinstance(optimizer, Optimizer): + raise TypeError('{:} is not an Optimizer'.format(type(optimizer).__name__)) + self.optimizer = optimizer + for group in optimizer.param_groups: + group.setdefault('initial_lr', group['lr']) + self.base_lrs = list(map(lambda group: group['initial_lr'], optimizer.param_groups)) + self.max_epochs = epochs + self.warmup_epochs = warmup_epochs + self.current_epoch = 0 + self.current_iter = 0 + + def extra_repr(self): + return '' + + def __repr__(self): + return ('{name}(warmup={warmup_epochs}, max-epoch={max_epochs}, current::epoch={current_epoch}, iter={current_iter:.2f}'.format(name=self.__class__.__name__, **self.__dict__) + + ', {:})'.format(self.extra_repr())) + + def state_dict(self): + return {key: value for key, value in self.__dict__.items() if key != 'optimizer'} + + def load_state_dict(self, state_dict): + self.__dict__.update(state_dict) + + def get_lr(self): + raise NotImplementedError + + def get_min_info(self): + lrs = self.get_lr() + return '#LR=[{:.6f}~{:.6f}] epoch={:03d}, iter={:4.2f}#'.format(min(lrs), max(lrs), self.current_epoch, self.current_iter) + + def get_min_lr(self): + return min( self.get_lr() ) + + def update(self, cur_epoch, cur_iter): + if cur_epoch is not None: + assert isinstance(cur_epoch, int) and cur_epoch>=0, 'invalid cur-epoch : {:}'.format(cur_epoch) + self.current_epoch = cur_epoch + if cur_iter is not None: + assert isinstance(cur_iter, float) and cur_iter>=0, 'invalid cur-iter : {:}'.format(cur_iter) + self.current_iter = cur_iter + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr + + + +class CosineAnnealingLR(_LRScheduler): + + def __init__(self, optimizer, warmup_epochs, epochs, T_max, eta_min): + self.T_max = T_max + self.eta_min = eta_min + super(CosineAnnealingLR, self).__init__(optimizer, warmup_epochs, epochs) + + def extra_repr(self): + return 'type={:}, T-max={:}, eta-min={:}'.format('cosine', self.T_max, self.eta_min) + + def get_lr(self): + lrs = [] + for base_lr in self.base_lrs: + if self.current_epoch >= self.warmup_epochs and self.current_epoch < self.max_epochs: + last_epoch = self.current_epoch - self.warmup_epochs + #if last_epoch < self.T_max: + #if last_epoch < self.max_epochs: + lr = self.eta_min + (base_lr - self.eta_min) * (1 + math.cos(math.pi * last_epoch / self.T_max)) / 2 + #else: + # lr = self.eta_min + (base_lr - self.eta_min) * (1 + math.cos(math.pi * (self.T_max-1.0) / self.T_max)) / 2 + elif self.current_epoch >= self.max_epochs: + lr = self.eta_min + else: + lr = (self.current_epoch / self.warmup_epochs + self.current_iter / self.warmup_epochs) * base_lr + lrs.append( lr ) + return lrs + + + +class MultiStepLR(_LRScheduler): + + def __init__(self, optimizer, warmup_epochs, epochs, milestones, gammas): + assert len(milestones) == len(gammas), 'invalid {:} vs {:}'.format(len(milestones), len(gammas)) + self.milestones = milestones + self.gammas = gammas + super(MultiStepLR, self).__init__(optimizer, warmup_epochs, epochs) + + def extra_repr(self): + return 'type={:}, milestones={:}, gammas={:}, base-lrs={:}'.format('multistep', self.milestones, self.gammas, self.base_lrs) + + def get_lr(self): + lrs = [] + for base_lr in self.base_lrs: + if self.current_epoch >= self.warmup_epochs: + last_epoch = self.current_epoch - self.warmup_epochs + idx = bisect_right(self.milestones, last_epoch) + lr = base_lr + for x in self.gammas[:idx]: lr *= x + else: + lr = (self.current_epoch / self.warmup_epochs + self.current_iter / self.warmup_epochs) * base_lr + lrs.append( lr ) + return lrs + + +class ExponentialLR(_LRScheduler): + + def __init__(self, optimizer, warmup_epochs, epochs, gamma): + self.gamma = gamma + super(ExponentialLR, self).__init__(optimizer, warmup_epochs, epochs) + + def extra_repr(self): + return 'type={:}, gamma={:}, base-lrs={:}'.format('exponential', self.gamma, self.base_lrs) + + def get_lr(self): + lrs = [] + for base_lr in self.base_lrs: + if self.current_epoch >= self.warmup_epochs: + last_epoch = self.current_epoch - self.warmup_epochs + assert last_epoch >= 0, 'invalid last_epoch : {:}'.format(last_epoch) + lr = base_lr * (self.gamma ** last_epoch) + else: + lr = (self.current_epoch / self.warmup_epochs + self.current_iter / self.warmup_epochs) * base_lr + lrs.append( lr ) + return lrs + + +class LinearLR(_LRScheduler): + + def __init__(self, optimizer, warmup_epochs, epochs, max_LR, min_LR): + self.max_LR = max_LR + self.min_LR = min_LR + super(LinearLR, self).__init__(optimizer, warmup_epochs, epochs) + + def extra_repr(self): + return 'type={:}, max_LR={:}, min_LR={:}, base-lrs={:}'.format('LinearLR', self.max_LR, self.min_LR, self.base_lrs) + + def get_lr(self): + lrs = [] + for base_lr in self.base_lrs: + if self.current_epoch >= self.warmup_epochs: + last_epoch = self.current_epoch - self.warmup_epochs + assert last_epoch >= 0, 'invalid last_epoch : {:}'.format(last_epoch) + ratio = (self.max_LR - self.min_LR) * last_epoch / self.max_epochs / self.max_LR + lr = base_lr * (1-ratio) + else: + lr = (self.current_epoch / self.warmup_epochs + self.current_iter / self.warmup_epochs) * base_lr + lrs.append( lr ) + return lrs + + + +class CrossEntropyLabelSmooth(nn.Module): + + def __init__(self, num_classes, epsilon): + super(CrossEntropyLabelSmooth, self).__init__() + self.num_classes = num_classes + self.epsilon = epsilon + self.logsoftmax = nn.LogSoftmax(dim=1) + + def forward(self, inputs, targets): + log_probs = self.logsoftmax(inputs) + targets = torch.zeros_like(log_probs).scatter_(1, targets.unsqueeze(1), 1) + targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes + loss = (-targets * log_probs).mean(0).sum() + return loss + + + +def get_optim_scheduler(parameters, config): + assert hasattr(config, 'optim') and hasattr(config, 'scheduler') and hasattr(config, 'criterion'), 'config must have optim / scheduler / criterion keys instead of {:}'.format(config) + if config.optim == 'SGD': + optim = torch.optim.SGD(parameters, config.LR, momentum=config.momentum, weight_decay=config.decay, nesterov=config.nesterov) + elif config.optim == 'RMSprop': + optim = torch.optim.RMSprop(parameters, config.LR, momentum=config.momentum, weight_decay=config.decay) + else: + raise ValueError('invalid optim : {:}'.format(config.optim)) + + if config.scheduler == 'cos': + T_max = getattr(config, 'T_max', config.epochs) + scheduler = CosineAnnealingLR(optim, config.warmup, config.epochs, T_max, config.eta_min) + elif config.scheduler == 'multistep': + scheduler = MultiStepLR(optim, config.warmup, config.epochs, config.milestones, config.gammas) + elif config.scheduler == 'exponential': + scheduler = ExponentialLR(optim, config.warmup, config.epochs, config.gamma) + elif config.scheduler == 'linear': + scheduler = LinearLR(optim, config.warmup, config.epochs, config.LR, config.LR_min) + else: + raise ValueError('invalid scheduler : {:}'.format(config.scheduler)) + + if config.criterion == 'Softmax': + criterion = torch.nn.CrossEntropyLoss() + elif config.criterion == 'SmoothSoftmax': + criterion = CrossEntropyLabelSmooth(config.class_num, config.label_smooth) + else: + raise ValueError('invalid criterion : {:}'.format(config.criterion)) + return optim, scheduler, criterion diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/starts.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/starts.py new file mode 100644 index 0000000..b1b19d3 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/nas_bench_201/procedures/starts.py @@ -0,0 +1,64 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +import os, sys, torch, random, PIL, copy, numpy as np +from os import path as osp +from shutil import copyfile + + +def prepare_seed(rand_seed): + random.seed(rand_seed) + np.random.seed(rand_seed) + torch.manual_seed(rand_seed) + torch.cuda.manual_seed(rand_seed) + torch.cuda.manual_seed_all(rand_seed) + + +def prepare_logger(xargs): + args = copy.deepcopy( xargs ) + from log_utils import Logger + logger = Logger(args.save_dir, args.rand_seed) + logger.log('Main Function with logger : {:}'.format(logger)) + logger.log('Arguments : -------------------------------') + for name, value in args._get_kwargs(): + logger.log('{:16} : {:}'.format(name, value)) + logger.log("Python Version : {:}".format(sys.version.replace('\n', ' '))) + logger.log("Pillow Version : {:}".format(PIL.__version__)) + logger.log("PyTorch Version : {:}".format(torch.__version__)) + logger.log("cuDNN Version : {:}".format(torch.backends.cudnn.version())) + logger.log("CUDA available : {:}".format(torch.cuda.is_available())) + logger.log("CUDA GPU numbers : {:}".format(torch.cuda.device_count())) + logger.log("CUDA_VISIBLE_DEVICES : {:}".format(os.environ['CUDA_VISIBLE_DEVICES'] if 'CUDA_VISIBLE_DEVICES' in os.environ else 'None')) + return logger + + +def get_machine_info(): + info = "Python Version : {:}".format(sys.version.replace('\n', ' ')) + info+= "\nPillow Version : {:}".format(PIL.__version__) + info+= "\nPyTorch Version : {:}".format(torch.__version__) + info+= "\ncuDNN Version : {:}".format(torch.backends.cudnn.version()) + info+= "\nCUDA available : {:}".format(torch.cuda.is_available()) + info+= "\nCUDA GPU numbers : {:}".format(torch.cuda.device_count()) + if 'CUDA_VISIBLE_DEVICES' in os.environ: + info+= "\nCUDA_VISIBLE_DEVICES={:}".format(os.environ['CUDA_VISIBLE_DEVICES']) + else: + info+= "\nDoes not set CUDA_VISIBLE_DEVICES" + return info + + +def save_checkpoint(state, filename, logger): + if osp.isfile(filename): + if hasattr(logger, 'log'): logger.log('Find {:} exist, delete is at first before saving'.format(filename)) + os.remove(filename) + torch.save(state, filename) + assert osp.isfile(filename), 'save filename : {:} failed, which is not found.'.format(filename) + if hasattr(logger, 'log'): logger.log('save checkpoint into {:}'.format(filename)) + return filename + + +def copy_checkpoint(src, dst, logger): + if osp.isfile(dst): + if hasattr(logger, 'log'): logger.log('Find {:} exist, delete is at first before saving'.format(dst)) + os.remove(dst) + copyfile(src, dst) + if hasattr(logger, 'log'): logger.log('copy the file from {:} into {:}'.format(src, dst)) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/parser.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/parser.py new file mode 100644 index 0000000..8556a20 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/parser.py @@ -0,0 +1,39 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import argparse + +def str2bool(v): + return v.lower() in ['t', 'true', True] + + +def get_parser(): + parser = argparse.ArgumentParser() + # general settings + parser.add_argument('--seed', type=int, default=333) + parser.add_argument('--gpu', type=str, default='0', help='set visible gpus') + parser.add_argument('--model_name', type=str, default=None, help='select model [generator|predictor]') + parser.add_argument('--save-path', type=str, default='results', help='the path of save directory') + parser.add_argument('--data-path', type=str, default='data', help='the path of save directory') + parser.add_argument('--save-epoch', type=int, default=20, help='how many epochs to wait each time to save model states') + parser.add_argument('--max-epoch', type=int, default=400, help='number of epochs to train') + parser.add_argument('--batch_size', type=int, default=32, help='batch size for generator') + parser.add_argument('--graph-data-name', default='nasbench201', help='graph dataset name') + parser.add_argument('--nvt', type=int, default=7, help='number of different node types, 7: NAS-Bench-201 including in/out node') + # set encoder + parser.add_argument('--num-sample', type=int, default=20, help='the number of images as input for set encoder') + # graph encoder + parser.add_argument('--hs', type=int, default=56, help='hidden size of GRUs') + parser.add_argument('--nz', type=int, default=56, help='the number of dimensions of latent vectors z') + # test + parser.add_argument('--test', action='store_true', default=False, help='turn on test mode') + parser.add_argument('--load-epoch', type=int, default=20, help='checkpoint epoch loaded for meta-test') + parser.add_argument('--data-name', type=str, default=None, help='meta-test dataset name') + parser.add_argument('--num-class', type=int, default=None, help='the number of class of dataset') + parser.add_argument('--num-gen-arch', type=int, default=800, help='the number of candidate architectures generated by the generator') + parser.add_argument('--train-arch', type=str2bool, default=True, help='whether to train the searched architecture') + + args = parser.parse_args() + + return args diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/__init__.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/__init__.py new file mode 100644 index 0000000..7ea5b7c --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/__init__.py @@ -0,0 +1,5 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from .predictor import Predictor diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/predictor.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/predictor.py new file mode 100644 index 0000000..ddcc147 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/predictor.py @@ -0,0 +1,321 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import torch +import os +import random +from tqdm import tqdm +import numpy as np +import time +import os +import shutil +import sys + +from torch import nn, optim +from torch.optim.lr_scheduler import ReduceLROnPlateau +from scipy.stats import pearsonr + +from transfer_nag_lib.MetaD2A_nas_bench_201.metad2a_utils import load_graph_config, decode_igraph_to_NAS_BENCH_201_string +from transfer_nag_lib.MetaD2A_nas_bench_201.metad2a_utils import Log, get_log +from transfer_nag_lib.MetaD2A_nas_bench_201.metad2a_utils import load_model, save_model, mean_confidence_interval +from transfer_nag_lib.MetaD2A_nas_bench_201.loader import get_meta_train_loader, get_meta_test_loader, MetaTestDataset +from .predictor_model import PredictorModel +from transfer_nag_lib.MetaD2A_nas_bench_201.nas_bench_201 import train_single_model +sys.path.append(os.path.join('.')) +from all_path import * + + +class Predictor: + def __init__(self, args): + # temp (MetaD2A) + args.nz = 56 + + self.args = args + self.batch_size = args.batch_size + self.data_path = args.data_path + self.num_sample = args.num_sample + self.max_epoch = args.max_epoch + self.save_epoch = args.save_epoch + # self.model_load_path = args.model_load_path + # self.model_path = os.path.join(os.getcwd(), 'meta_nas', 'TNAS-DCS', 'MetaD2A_nas_bench_201', + # 'results', 'predictor', 'model') + self.model_path = MODEL_METAD2A_PATH + self.save_path = args.save_path + self.model_name = 'predictor' + self.test = args.test + self.device = torch.device("cuda:0") + self.max_corr_dict = {'corr': -1, 'epoch': -1} + self.train_arch = args.train_arch + self.nasbench201 = torch.load(NASBENCH201) + + graph_config = load_graph_config( + args.graph_data_name, args.nvt, args.data_path) + + self.model = PredictorModel(args, graph_config) + self.model.to(self.device) + + if self.test: + self.data_name = args.data_name + self.num_class = args.num_class + self.load_epoch = args.load_epoch + load_model(self.model, self.model_path, + load_max_pt='ckpt_max_corr.pt') + else: + self.optimizer = optim.Adam(self.model.parameters(), lr=1e-4) + self.scheduler = ReduceLROnPlateau(self.optimizer, 'min', + factor=0.1, patience=10, verbose=True) + self.mtrloader = get_meta_train_loader( + self.batch_size, self.data_path, self.num_sample, is_pred=True) + + self.acc_mean = self.mtrloader.dataset.mean + self.acc_std = self.mtrloader.dataset.std + + self.mtrlog = Log(self.args, open(os.path.join( + self.save_path, self.model_name, 'meta_train_predictor.log'), 'w')) + self.mtrlog.print_args() + + def forward(self, x, arch): + D_mu = self.model.set_encode(x.to(self.device)) + G_mu = self.model.graph_encode(arch) + y_pred = self.model.predict(D_mu, G_mu) + return y_pred + + def meta_train(self): + sttime = time.time() + for epoch in range(1, self.max_epoch + 1): + self.mtrlog.ep_sttime = time.time() + loss, corr = self.meta_train_epoch(epoch) + self.scheduler.step(loss) + self.mtrlog.print_pred_log(loss, corr, 'train', epoch) + valoss, vacorr = self.meta_validation(epoch) + if self.max_corr_dict['corr'] < vacorr: + self.max_corr_dict['corr'] = vacorr + self.max_corr_dict['epoch'] = epoch + self.max_corr_dict['loss'] = valoss + save_model(epoch, self.model, self.model_path, max_corr=True) + + self.mtrlog.print_pred_log( + valoss, vacorr, 'valid', max_corr_dict=self.max_corr_dict) + + if epoch % self.save_epoch == 0: + save_model(epoch, self.model, self.model_path) + + self.mtrlog.save_time_log() + self.mtrlog.max_corr_log(self.max_corr_dict) + + def meta_train_epoch(self, epoch): + self.model.to(self.device) + self.model.train() + + self.mtrloader.dataset.set_mode('train') + + dlen = len(self.mtrloader.dataset) + trloss = 0 + y_all, y_pred_all = [], [] + pbar = tqdm(self.mtrloader) + + for x, g, acc in pbar: + self.optimizer.zero_grad() + y_pred = self.forward(x, g) + y = acc.to(self.device) + loss = self.model.mseloss(y_pred, y.unsqueeze(-1)) + loss.backward() + self.optimizer.step() + + y = y.tolist() + y_pred = y_pred.squeeze().tolist() + y_all += y + y_pred_all += y_pred + pbar.set_description(get_log( + epoch, loss, y_pred, y, self.acc_std, self.acc_mean)) + trloss += float(loss) + + return trloss/dlen, pearsonr(np.array(y_all), + np.array(y_pred_all))[0] + + def meta_validation(self, epoch): + self.model.to(self.device) + self.model.eval() + + valoss = 0 + self.mtrloader.dataset.set_mode('valid') + dlen = len(self.mtrloader.dataset) + y_all, y_pred_all = [], [] + pbar = tqdm(self.mtrloader) + + with torch.no_grad(): + for x, g, acc in pbar: + y_pred = self.forward(x, g) + y = acc.to(self.device) + loss = self.model.mseloss(y_pred, y.unsqueeze(-1)) + + y = y.tolist() + y_pred = y_pred.squeeze().tolist() + y_all += y + y_pred_all += y_pred + pbar.set_description(get_log( + epoch, loss, y_pred, y, self.acc_std, self.acc_mean, tag='val')) + valoss += float(loss) + + return valoss/dlen, pearsonr(np.array(y_all), + np.array(y_pred_all))[0] + + def meta_test(self): + if self.data_name == 'all': + for data_name in ['cifar10', 'cifar100', 'mnist', 'svhn', 'aircraft', 'pets']: + self.meta_test_per_dataset(data_name) + else: + self.meta_test_per_dataset(self.data_name) + + def meta_test_per_dataset(self, data_name): + # self.nasbench201 = torch.load( + # os.path.join(self.data_path, 'nasbench201.pt')) + + self.test_dataset = MetaTestDataset( + self.data_path, data_name, self.num_sample, self.num_class) + + meta_test_path = os.path.join( + self.save_path, 'meta_test', data_name, 'best_arch') + if not os.path.exists(meta_test_path): + os.makedirs(meta_test_path) + f_arch_str = open( + os.path.join(meta_test_path, 'architecture.txt'), 'w') + save_path = os.path.join(meta_test_path, 'accuracy.txt') + f = open(save_path, 'w') + arch_runs = [] + elasped_time = [] + + if 'cifar' in data_name: + N = 30 + runs = 10 + acc_runs = [] + else: + N = 1 + runs = 1 + + print( + f'==> select top architectures for {data_name} by meta-predictor...') + for run in range(1, runs + 1): + print(f'==> run #{run}') + gen_arch_str = self.load_generated_archs(data_name, run) + gen_arch_igraph = self.get_items( + full_target=self.nasbench201['arch']['igraph'], + full_source=self.nasbench201['arch']['str'], + source=gen_arch_str) + y_pred_all = [] + self.model.eval() + self.model.to(self.device) + + sttime = time.time() + with torch.no_grad(): + for i, arch_igraph in enumerate(gen_arch_igraph): + x, g = self.collect_data(arch_igraph) + y_pred = self.forward(x, g) + y_pred = torch.mean(y_pred) + y_pred_all.append(y_pred.cpu().detach().item()) + + top_arch_lst = self.select_top_arch( + data_name, torch.tensor(y_pred_all), gen_arch_str, N) + arch_runs.append(top_arch_lst[0]) + elasped = time.time() - sttime + elasped_time.append(elasped) + + if 'cifar' in data_name: + acc = self.select_top_acc(data_name, top_arch_lst) + acc_runs.append(acc) + + for run, arch_str in enumerate(arch_runs): + f_arch_str.write(f'{arch_str}\n') + print(f'{arch_str}') + + time_path = os.path.join( + self.save_path, 'meta_test', data_name, 'time.txt') + with open(time_path, 'a') as f_time: + msg = f'predictor average elasped time {np.mean(elasped_time):.2f}s' + print(f'==> save time in {time_path}') + f_time.write(msg+'\n') + print(msg) + + if self.train_arch: + if not 'cifar' in data_name: + acc_runs = self.train_single_arch( + data_name, arch_runs[0], meta_test_path) + print(f'==> save results in {save_path}') + for r, acc in enumerate(acc_runs): + msg = f'run {r+1} {acc:.2f} (%)' + f.write(msg+'\n') + print(msg) + + m, h = mean_confidence_interval(acc_runs) + msg = f'Avg {m:.2f}+-{h.item():.2f} (%)' + f.write(msg+'\n') + print(msg) + + def train_single_arch(self, data_name, arch_str, meta_test_path): + seeds = (777, 888, 999) + train_single_model(save_dir=meta_test_path, + workers=8, + datasets=[data_name], + xpaths=[f'{self.data_path}/raw-data/{data_name}'], + splits=[0], + use_less=False, + seeds=seeds, + model_str=arch_str, + arch_config={'channel': 16, 'num_cells': 5}) + epoch = 49 if data_name == 'mnist' else 199 + test_acc_lst = [] + for seed in seeds: + result = torch.load(os.path.join( + meta_test_path, f'seed-0{seed}.pth')) + test_acc_lst.append( + result[data_name]['valid_acc1es'][f'x-test@{epoch}']) + return test_acc_lst + + def select_top_arch_acc( + self, data_name, y_pred_all, gen_arch_str, N): + _, sorted_idx = torch.sort(y_pred_all, descending=True) + gen_test_acc = self.get_items( + full_target=self.nasbench201['test-acc'][data_name], + full_source=self.nasbench201['arch']['str'], + source=gen_arch_str) + sorted_gen_test_acc = torch.tensor(gen_test_acc)[sorted_idx] + sotred_gen_arch_str = [gen_arch_str[_] for _ in sorted_idx] + + max_idx = torch.argmax(sorted_gen_test_acc[:N]).item() + final_acc = sorted_gen_test_acc[:N][max_idx] + final_str = sotred_gen_arch_str[:N][max_idx] + return final_acc, final_str + + def select_top_arch( + self, data_name, y_pred_all, gen_arch_str, N): + _, sorted_idx = torch.sort(y_pred_all, descending=True) + sotred_gen_arch_str = [gen_arch_str[_] for _ in sorted_idx] + final_str = sotred_gen_arch_str[:N] + return final_str + + def select_top_acc(self, data_name, final_str): + final_test_acc = self.get_items( + full_target=self.nasbench201['test-acc'][data_name], + full_source=self.nasbench201['arch']['str'], + source=final_str) + max_test_acc = max(final_test_acc) + return max_test_acc + + def collect_data(self, arch_igraph): + x_batch, g_batch = [], [] + for _ in range(10): + x_batch.append(self.test_dataset[0]) + g_batch.append(arch_igraph) + return torch.stack(x_batch).to(self.device), g_batch + + def get_items(self, full_target, full_source, source): + return [full_target[full_source.index(_)] for _ in source] + + def load_generated_archs(self, data_name, run): + mtest_path = os.path.join( + self.save_path, 'meta_test', data_name, 'generated_arch') + with open(os.path.join(mtest_path, f'run_{run}.txt'), 'r') as f: + gen_arch_str = [_.split()[0] for _ in f.readlines()[1:]] + return gen_arch_str diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/predictor_model.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/predictor_model.py new file mode 100644 index 0000000..11b7dee --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/predictor/predictor_model.py @@ -0,0 +1,246 @@ +###################################################################################### +# Copyright (c) muhanzhang, D-VAE, NeurIPS 2019 [GitHub D-VAE] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import math +import random +import torch +from torch import nn +from torch.nn import functional as F +import numpy as np +import igraph + +from transfer_nag_lib.MetaD2A_nas_bench_201.set_encoder.setenc_models import SetPool + + +class PredictorModel(nn.Module): + def __init__(self, args, graph_config): + super(PredictorModel, self).__init__() + self.max_n = graph_config['max_n'] # maximum number of vertices + self.nvt = args.nvt # number of vertex types + self.START_TYPE = graph_config['START_TYPE'] + self.END_TYPE = graph_config['END_TYPE'] + self.hs = args.hs # hidden state size of each vertex + self.nz = args.nz # size of latent representation z + self.gs = args.hs # size of graph state + self.bidir = True # whether to use bidirectional encoding + self.vid = True + self.device = None + self.input_type = 'DG' + self.num_sample = args.num_sample + + if self.vid: + self.vs = self.hs + self.max_n # vertex state size = hidden state + vid + else: + self.vs = self.hs + + # 0. encoding-related + self.grue_forward = nn.GRUCell(self.nvt, self.hs) # encoder GRU + self.grue_backward = nn.GRUCell(self.nvt, self.hs) # backward encoder GRU + self.fc1 = nn.Linear(self.gs, self.nz) # latent mean + self.fc2 = nn.Linear(self.gs, self.nz) # latent logvar + + # 2. gate-related + self.gate_forward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.gate_backward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.mapper_forward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) # disable bias to ensure padded zeros also mapped to zeros + self.mapper_backward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) + + # 3. bidir-related, to unify sizes + if self.bidir: + self.hv_unify = nn.Sequential( + nn.Linear(self.hs * 2, self.hs), + ) + self.hg_unify = nn.Sequential( + nn.Linear(self.gs * 2, self.gs), + ) + + # 4. other + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + self.tanh = nn.Tanh() + self.logsoftmax1 = nn.LogSoftmax(1) + + # 6. predictor + np = self.gs + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + input_dim = 0 + if 'D' in self.input_type: + input_dim += self.nz + if 'G' in self.input_type: + input_dim += self.nz + + self.pred_fc = nn.Sequential( + nn.Linear(input_dim, self.hs), + nn.Tanh(), + nn.Linear(self.hs, 1) + ) + self.mseloss = nn.MSELoss(reduction='sum') + + + def predict(self, D_mu, G_mu): + input_vec = [] + if 'D' in self.input_type: + input_vec.append(D_mu) + if 'G' in self.input_type: + input_vec.append(G_mu) + input_vec = torch.cat(input_vec, dim=1) + return self.pred_fc(input_vec) + + def get_device(self): + if self.device is None: + self.device = next(self.parameters()).device + return self.device + + def _get_zeros(self, n, length): + return torch.zeros(n, length).to(self.get_device()) # get a zero hidden state + + def _get_zero_hidden(self, n=1): + return self._get_zeros(n, self.hs) # get a zero hidden state + + def _one_hot(self, idx, length): + if type(idx) in [list, range]: + if idx == []: + return None + idx = torch.LongTensor(idx).unsqueeze(0).t() + x = torch.zeros((len(idx), length)).scatter_(1, idx, 1).to(self.get_device()) + else: + idx = torch.LongTensor([idx]).unsqueeze(0) + x = torch.zeros((1, length)).scatter_(1, idx, 1).to(self.get_device()) + return x + + def _gated(self, h, gate, mapper): + return gate(h) * mapper(h) + + def _collate_fn(self, G): + return [g.copy() for g in G] + + def _propagate_to(self, G, v, propagator, H=None, reverse=False, gate=None, mapper=None): + # propagate messages to vertex index v for all graphs in G + # return the new messages (states) at v + G = [g for g in G if g.vcount() > v] + if len(G) == 0: + return + if H is not None: + idx = [i for i, g in enumerate(G) if g.vcount() > v] + H = H[idx] + v_types = [g.vs[v]['type'] for g in G] + X = self._one_hot(v_types, self.nvt) + if reverse: + H_name = 'H_backward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.successors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.successors(v), self.max_n) for g in G] + gate, mapper = self.gate_backward, self.mapper_backward + else: + H_name = 'H_forward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.predecessors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.predecessors(v), self.max_n) for g in G] + if gate is None: + gate, mapper = self.gate_forward, self.mapper_forward + if self.vid: + H_pred = [[torch.cat([x[i], y[i:i + 1]], 1) for i in range(len(x))] for x, y in zip(H_pred, vids)] + # if h is not provided, use gated sum of v's predecessors' states as the input hidden state + if H is None: + max_n_pred = max([len(x) for x in H_pred]) # maximum number of predecessors + if max_n_pred == 0: + H = self._get_zero_hidden(len(G)) + else: + H_pred = [torch.cat(h_pred + + [self._get_zeros(max_n_pred - len(h_pred), self.vs)], 0).unsqueeze(0) + for h_pred in H_pred] # pad all to same length + H_pred = torch.cat(H_pred, 0) # batch * max_n_pred * vs + H = self._gated(H_pred, gate, mapper).sum(1) # batch * hs + Hv = propagator(X, H) + for i, g in enumerate(G): + g.vs[v][H_name] = Hv[i:i + 1] + return Hv + + def _propagate_from(self, G, v, propagator, H0=None, reverse=False): + # perform a series of propagation_to steps starting from v following a topo order + # assume the original vertex indices are in a topological order + if reverse: + prop_order = range(v, -1, -1) + else: + prop_order = range(v, self.max_n) + Hv = self._propagate_to(G, v, propagator, H0, reverse=reverse) # the initial vertex + for v_ in prop_order[1:]: + self._propagate_to(G, v_, propagator, reverse=reverse) + return Hv + + def _get_graph_state(self, G, decode=False): + # get the graph states + # when decoding, use the last generated vertex's state as the graph state + # when encoding, use the ending vertex state or unify the starting and ending vertex states + Hg = [] + for g in G: + hg = g.vs[g.vcount() - 1]['H_forward'] + if self.bidir and not decode: # decoding never uses backward propagation + hg_b = g.vs[0]['H_backward'] + hg = torch.cat([hg, hg_b], 1) + Hg.append(hg) + Hg = torch.cat(Hg, 0) + if self.bidir and not decode: + Hg = self.hg_unify(Hg) + return Hg + + + def set_encode(self, X): + proto_batch = [] + for x in X: + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + return v + + + def graph_encode(self, G): + # encode graphs G into latent vectors + if type(G) != list: + G = [G] + self._propagate_from(G, 0, self.grue_forward, H0=self._get_zero_hidden(len(G)), + reverse=False) + if self.bidir: + self._propagate_from(G, self.max_n - 1, self.grue_backward, + H0=self._get_zero_hidden(len(G)), reverse=True) + Hg = self._get_graph_state(G) + mu = self.fc1(Hg) + #logvar = self.fc2(Hg) + return mu #, logvar + + + def reparameterize(self, mu, logvar, eps_scale=0.01): + # return z ~ N(mu, std) + if self.training: + std = logvar.mul(0.5).exp_() + eps = torch.randn_like(std) * eps_scale + return eps.mul(std).add_(mu) + else: + return mu + \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/process_dataset.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/process_dataset.py new file mode 100644 index 0000000..e5d49b6 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/process_dataset.py @@ -0,0 +1,158 @@ +import numpy as np +import torchvision.models as models +import torchvision.datasets as dset +import os +import torch +import argparse +import random +import torchvision.transforms as transforms +import os, sys +if sys.version_info[0] == 2: + import cPickle as pickle +else: + import pickle +from PIL import Image + +parser = argparse.ArgumentParser("sota") +parser.add_argument('--gpu', type=str, default='0', help='set visible gpus') +parser.add_argument('--data-path', type=str, default='data', help='the path of save directory') +parser.add_argument('--dataset', type=str, default='cifar10', help='choose dataset') +parser.add_argument('--seed', type=int, default=-1, help='random seed') +args = parser.parse_args() + +if args.seed is None or args.seed < 0: args.seed = random.randint(1, 100000) + +os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu +np.random.seed(args.seed) +random.seed(args.seed) + +# remove last fully-connected layer +model = models.resnet18(pretrained=True).eval() +feature_extractor = torch.nn.Sequential(*list(model.children())[:-1]) + + +def get_transform(dataset): + if args.dataset == 'mnist': + mean, std = [0.1307, 0.1307, 0.1307], [0.3081, 0.3081, 0.3081] + elif args.dataset == 'svhn': + mean, std = [0.4376821, 0.4437697, 0.47280442], [0.19803012, 0.20101562, 0.19703614] + elif args.dataset == 'cifar10': + mean = [x / 255 for x in [125.3, 123.0, 113.9]] + std = [x / 255 for x in [63.0, 62.1, 66.7]] + elif args.dataset == 'cifar100': + mean = [x / 255 for x in [129.3, 124.1, 112.4]] + std = [x / 255 for x in [68.2, 65.4, 70.4]] + elif args.dataset == 'imagenet32': + mean = [x / 255 for x in [122.68, 116.66, 104.01]] + std = [x / 255 for x in [66.22, 64.20, 67.86]] + + transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + if dataset == 'mnist': + transform.transforms.append(transforms.Lambda(lambda x: x.repeat(3, 1, 1))) + return transform + + +def process(dataset, n_classes): + data_label = {i: [] for i in range(n_classes)} + for x, y in dataset: + data_label[y].append(x) + for i in range(n_classes): + data_label[i] = torch.stack(data_label[i]) + + holder = {i: [] for i in range(n_classes)} + for i in range(n_classes): + with torch.no_grad(): + data = feature_extractor(data_label[i]) + holder[i].append(data.squeeze()) + return holder + + + +class ImageNet32(object): + train_list = [ + ['train_data_batch_1', '27846dcaa50de8e21a7d1a35f30f0e91'], + ['train_data_batch_2', 'c7254a054e0e795c69120a5727050e3f'], + ['train_data_batch_3', '4333d3df2e5ffb114b05d2ffc19b1e87'], + ['train_data_batch_4', '1620cdf193304f4a92677b695d70d10f'], + ['train_data_batch_5', '348b3c2fdbb3940c4e9e834affd3b18d'], + ['train_data_batch_6', '6e765307c242a1b3d7d5ef9139b48945'], + ['train_data_batch_7', '564926d8cbf8fc4818ba23d2faac7564'], + ['train_data_batch_8', 'f4755871f718ccb653440b9dd0ebac66'], + ['train_data_batch_9', 'bb6dd660c38c58552125b1a92f86b5d4'], + ['train_data_batch_10', '8f03f34ac4b42271a294f91bf480f29b'], + ] + valid_list = [ + ['val_data', '3410e3017fdaefba8d5073aaa65e4bd6'], + ] + + def __init__(self, root, n_class, transform): + self.transform = transform + downloaded_list = self.train_list + self.n_class = n_class + self.data_label = {i: [] for i in range(n_class)} + self.data = [] + self.targets = [] + + for i, (file_name, checksum) in enumerate(downloaded_list): + file_path = os.path.join(root, file_name) + with open(file_path, 'rb') as f: + if sys.version_info[0] == 2: + entry = pickle.load(f) + else: + entry = pickle.load(f, encoding='latin1') + for j, k in enumerate(entry['labels']): + self.data_label[k - 1].append(entry['data'][j]) + + for i in range(n_class): + self.data_label[i] = np.vstack(self.data_label[i]).reshape(-1, 3, 32, 32) + self.data_label[i] = self.data_label[i].transpose((0, 2, 3, 1)) # convert to HWC + + def get(self, use_num_cls, max_num=None): + assert isinstance(use_num_cls, list) \ + and len(use_num_cls) > 0 and len(use_num_cls) < self.n_class, \ + 'invalid use_num_cls : {:}'.format(use_num_cls) + new_data, new_targets = [], [] + for i in use_num_cls: + new_data.append(self.data_label[i][:max_num] if max_num is not None else self.data_label[i]) + new_targets.extend([i] * max_num if max_num is not None + else [i] * len(self.data_label[i])) + self.data = np.concatenate(new_data) + self.targets = new_targets + + imgs = [] + for img in self.data: + img = Image.fromarray(img) + img = self.transform(img) + with torch.no_grad(): + imgs.append(feature_extractor(img.unsqueeze(0)).squeeze().unsqueeze(0)) + return torch.cat(imgs) + + +if __name__ == '__main__': + ncls = {'mnist': 10, 'svhn': 10, 'cifar10': 10, 'cifar100': 100, 'imagenet32': 1000} + transform = get_transform(args.dataset) + if args.dataset == 'imagenet32': + imgnet32 = ImageNet32(args.data, ncls[args.dataset], transform) + data_label = {i: [] for i in range(1000)} + for i in range(1000): + m = imgnet32.get([i]) + data_label[i].append(m) + if i % 10 == 0: + print(f'Currently saving features of {i}-th class') + torch.save(data_label, f'{args.save_path}/{args.dataset}bylabel.pt') + else: + if args.dataset == 'mnist': + data = dset.MNIST(args.data_path, train=True, transform=transform, download=True) + elif args.dataset == 'svhn': + data = dset.SVHN(args.data_path, split='train', transform=transform, download=True) + elif args.dataset == 'cifar10': + data = dset.CIFAR10(args.data_path, train=True, transform=transform, download=True) + elif args.dataset == 'cifar100': + data = dset.CIFAR100(args.data_path, train=True, transform=transform, download=True) + dataset = process(data, ncls[args.dataset]) + torch.save(dataset, f'{args.save_path}/{args.dataset}bylabel.pt') + diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/set_encoder/setenc_models.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/set_encoder/setenc_models.py new file mode 100644 index 0000000..e531f11 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/set_encoder/setenc_models.py @@ -0,0 +1,38 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from transfer_nag_lib.MetaD2A_nas_bench_201.set_encoder.setenc_modules import * + + +class SetPool(nn.Module): + def __init__(self, dim_input, num_outputs, dim_output, + num_inds=32, dim_hidden=128, num_heads=4, ln=False, mode=None): + super(SetPool, self).__init__() + if 'sab' in mode: # [32, 400, 128] + self.enc = nn.Sequential( + SAB(dim_input, dim_hidden, num_heads, ln=ln), # SAB? + SAB(dim_hidden, dim_hidden, num_heads, ln=ln)) + else: # [32, 400, 128] + self.enc = nn.Sequential( + ISAB(dim_input, dim_hidden, num_heads, num_inds, ln=ln), # SAB? + ISAB(dim_hidden, dim_hidden, num_heads, num_inds, ln=ln)) + if 'PF' in mode: # [32, 1, 501] + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), + nn.Linear(dim_hidden, dim_output)) + elif 'P' in mode: + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln)) + else: # torch.Size([32, 1, 501]) + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), # 32 1 128 + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + nn.Linear(dim_hidden, dim_output)) + # "", sm, sab, sabsm + + def forward(self, X): + x1 = self.enc(X) + x2 = self.dec(x1) + return x2 diff --git a/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/set_encoder/setenc_modules.py b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/set_encoder/setenc_modules.py new file mode 100644 index 0000000..54fe4d7 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/MetaD2A_nas_bench_201/set_encoder/setenc_modules.py @@ -0,0 +1,67 @@ +##################################################################################### +# Copyright (c) Juho Lee SetTransformer, ICML 2019 [GitHub set_transformer] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + +class MAB(nn.Module): + def __init__(self, dim_Q, dim_K, dim_V, num_heads, ln=False): + super(MAB, self).__init__() + self.dim_V = dim_V + self.num_heads = num_heads + self.fc_q = nn.Linear(dim_Q, dim_V) + self.fc_k = nn.Linear(dim_K, dim_V) + self.fc_v = nn.Linear(dim_K, dim_V) + if ln: + self.ln0 = nn.LayerNorm(dim_V) + self.ln1 = nn.LayerNorm(dim_V) + self.fc_o = nn.Linear(dim_V, dim_V) + + def forward(self, Q, K): + Q = self.fc_q(Q) + K, V = self.fc_k(K), self.fc_v(K) + + dim_split = self.dim_V // self.num_heads + Q_ = torch.cat(Q.split(dim_split, 2), 0) + K_ = torch.cat(K.split(dim_split, 2), 0) + V_ = torch.cat(V.split(dim_split, 2), 0) + + A = torch.softmax(Q_.bmm(K_.transpose(1,2))/math.sqrt(self.dim_V), 2) + O = torch.cat((Q_ + A.bmm(V_)).split(Q.size(0), 0), 2) + O = O if getattr(self, 'ln0', None) is None else self.ln0(O) + O = O + F.relu(self.fc_o(O)) + O = O if getattr(self, 'ln1', None) is None else self.ln1(O) + return O + +class SAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, ln=False): + super(SAB, self).__init__() + self.mab = MAB(dim_in, dim_in, dim_out, num_heads, ln=ln) + + def forward(self, X): + return self.mab(X, X) + +class ISAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, num_inds, ln=False): + super(ISAB, self).__init__() + self.I = nn.Parameter(torch.Tensor(1, num_inds, dim_out)) + nn.init.xavier_uniform_(self.I) + self.mab0 = MAB(dim_out, dim_in, dim_out, num_heads, ln=ln) + self.mab1 = MAB(dim_in, dim_out, dim_out, num_heads, ln=ln) + + def forward(self, X): + H = self.mab0(self.I.repeat(X.size(0), 1, 1), X) + return self.mab1(X, H) + +class PMA(nn.Module): + def __init__(self, dim, num_heads, num_seeds, ln=False): + super(PMA, self).__init__() + self.S = nn.Parameter(torch.Tensor(1, num_seeds, dim)) + nn.init.xavier_uniform_(self.S) + self.mab = MAB(dim, dim, dim, num_heads, ln=ln) + + def forward(self, X): + return self.mab(self.S.repeat(X.size(0), 1, 1), X) diff --git a/MobileNetV3/main_exp/transfer_nag_lib/Setconfig90.json b/MobileNetV3/main_exp/transfer_nag_lib/Setconfig90.json new file mode 100644 index 0000000..823563f --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/Setconfig90.json @@ -0,0 +1 @@ +{"units_C": 32, "output_size_A": [32, 32, 32, 32], "output_size_D": [32, 32, 32, 32], "output_size_B": [32, 32, 32, 32], "kernel": "52", "nu": 2.5, "ard": false, "epochs": 10000, "loss_tol": 0.0001, "dropout": 0.0, "lr": 0.001, "patience": 16} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/Setconfig_NAS.json b/MobileNetV3/main_exp/transfer_nag_lib/Setconfig_NAS.json new file mode 100644 index 0000000..665b27b --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/Setconfig_NAS.json @@ -0,0 +1 @@ +{"units_C": 72, "output_size_A": [72, 32, 32, 32], "output_size_D": [72, 32, 32, 32], "output_size_B": [72, 32, 32, 32], "kernel": "52", "nu": 2.5, "ard": false, "epochs": 10000, "loss_tol": 0.0001, "dropout": 0.0, "lr": 0.001, "patience": 16} \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/acquisition_functions.py b/MobileNetV3/main_exp/transfer_nag_lib/acquisition_functions.py new file mode 100644 index 0000000..bc2e6ff --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/acquisition_functions.py @@ -0,0 +1,99 @@ +import numpy as np +from scipy.stats import norm +import sys + +def UCB(incumbent, mu, stddev, support, return_score=False, beta=0.5): + return mu + beta * stddev + +def PI(incumbent, mu, stddev, support, return_score=False, eps=0.): + with np.errstate(divide='warn'): + imp = mu - incumbent - eps + Z = imp / stddev + score = norm.cdf(Z) + if not return_score: + return np.argmax(score) + else: + return score + +def EI(incumbent, mu, stddev, support, return_score=False, eps=0.): + + with np.errstate(divide='warn'): + imp = mu - incumbent - eps + Z = imp / stddev + score = imp * norm.cdf(Z) + stddev * norm.pdf(Z) + + if not return_score: + import pdb; pdb.set_trace() + score[support] = 0 + return np.argmax(score) + else: + # score[support] = 0 + return score + +# Different acquisition functions +def acq_fn(predictions, ytrain=None, stds=None, explore_type='its'): + predictions = np.array(predictions) + + if stds is None: + stds = np.sqrt(np.var(predictions, axis=0)) + + # Upper confidence bound (UCB) acquisition function + if explore_type == 'ucb': + explore_factor = 0.5 + mean = np.mean(predictions, axis=0) + ucb = mean - explore_factor * stds + sorted_indices = np.argsort(ucb) + + # Expected improvement (EI) acquisition function + elif explore_type == 'ei': + ei_calibration_factor = 5. + mean = list(np.mean(predictions, axis=0)) + factored_stds = list(stds / ei_calibration_factor) + min_y = ytrain.min() + gam = [(min_y - mean[i]) / factored_stds[i] for i in range(len(mean))] + ei = [-1 * factored_stds[i] * (gam[i] * norm.cdf(gam[i]) + norm.pdf(gam[i])) + for i in range(len(mean))] + sorted_indices = np.argsort(ei) + + # Probability of improvement (PI) acquisition function + elif explore_type == 'pi': + mean = list(np.mean(predictions, axis=0)) + stds = list(stds) + min_y = ytrain.min() + pi = [-1 * norm.cdf(min_y, loc=mean[i], scale=stds[i]) for i in range(len(mean))] + sorted_indices = np.argsort(pi) + + # Thompson sampling (TS) acquisition function + elif explore_type == 'ts': + rand_ind = np.random.randint(predictions.shape[0]) + ts = predictions[rand_ind,:] + sorted_indices = np.argsort(ts) + + # Top exploitation + elif explore_type == 'percentile': + min_prediction = np.min(predictions, axis=0) + sorted_indices = np.argsort(min_prediction) + + # Top mean + elif explore_type == 'mean': + mean = np.mean(predictions, axis=0) + sorted_indices = np.argsort(mean) + + elif explore_type == 'confidence': + confidence_factor = 2 + mean = np.mean(predictions, axis=0) + conf = mean + confidence_factor * stds + sorted_indices = np.argsort(conf) + + # Independent Thompson sampling (ITS) acquisition function + elif explore_type == 'its': + # predictions: (5, 119), mean: (119,), stds: (119,) + mean = np.mean(predictions, axis=0) + samples = np.random.normal(mean, stds) + sorted_indices = np.argsort(samples) + + else: + print('{} is not a valid exploration type'.format(explore_type)) + raise NotImplementedError() + + return sorted_indices \ No newline at end of file diff --git a/MobileNetV3/main_exp/transfer_nag_lib/encoder_FSBO_ofa.py b/MobileNetV3/main_exp/transfer_nag_lib/encoder_FSBO_ofa.py new file mode 100644 index 0000000..c52b1b7 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/encoder_FSBO_ofa.py @@ -0,0 +1,413 @@ +###################################################################################### +# Copyright (c) muhanzhang, D-VAE, NeurIPS 2019 [GitHub D-VAE] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +# import math +# import random +import torch +import json +from torch import nn +import os +from torch.nn import functional as F +import datetime + + +## Our packages +import gpytorch +import logging + +from transfer_nag_lib.DeepKernelGPHelpers import Metric +from transfer_nag_lib.DeepKernelGPModules import StandardDeepGP, ExactGPLayer +from transfer_nag_lib.MetaD2A_mobilenetV3.set_encoder.setenc_models import SetPool + + +class EncoderFSBO(nn.Module): + def __init__(self, args, graph_config, dgp_arch): + super(EncoderFSBO, self).__init__() + + ## GP parameters + space="OFA_MBV3" + c, D = 4230, 64 + dim = args.nz * 2 + rootdir = os.path.dirname(os.path.realpath(__file__)) + backbone_params = json.load(open(os.path.join(rootdir, "Setconfig90.json"), "rb")) + backbone_params.update({"dim": dim}) + backbone_params.update({"fixed_context_size": dim}) + backbone_params.update({"minibatch_size": 256}) + backbone_params.update({"n_inner_steps": 1}) + backbone_params.update({"output_size_A": dgp_arch}) + + checkpoint_path = os.path.join(rootdir, "checkpoints", "FSBO-metalearn", f"{space}", + datetime.datetime.now().strftime('meta-%Y-%m-%d-%H-%M-%S-%f')) + backbone_params.update({"checkpoint_path": checkpoint_path}) + self.fixed_context_size = backbone_params["fixed_context_size"] + self.minibatch_size = backbone_params["minibatch_size"] + self.n_inner_steps = backbone_params["n_inner_steps"] + self.checkpoint_path = backbone_params["checkpoint_path"] + os.makedirs(self.checkpoint_path, exist_ok=False) + json.dump(backbone_params, open(os.path.join(self.checkpoint_path, "configuration.json"), "w")) + # self.device = torch.device("cpu") # "cuda:0" if torch.cuda.is_available() else "cpu") + self.device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") + logging.basicConfig(filename=os.path.join(self.checkpoint_path, "log.txt"), level=logging.DEBUG) + self.config = backbone_params + self.likelihood = gpytorch.likelihoods.GaussianLikelihood() + self.gp = ExactGPLayer(train_x=None, train_y=None, likelihood=self.likelihood, config=self.config, + dims=self.fixed_context_size) + self.mll = gpytorch.mlls.ExactMarginalLogLikelihood(self.likelihood, self.gp).to(self.device) + self.gp.double() + self.likelihood.double() + self.mll.double() + self.mse = nn.MSELoss() + # self.mtrloader = get_meta_train_loader( + # args.batch_size, args.data_path, args.num_sample) + # self.get_tasks() + self.setup_writers() + + self.train_metrics = Metric() + self.valid_metrics = Metric(prefix="valid: ") + + self.max_n = graph_config['max_n'] # maximum number of vertices + self.nvt = graph_config['num_vertex_type'] if args.search_space == 'ofa' else args.nvt # number of vertex types + self.START_TYPE = graph_config['START_TYPE'] + self.END_TYPE = graph_config['END_TYPE'] + self.hs = args.hs # hidden state size of each vertex + self.nz = args.nz # size of latent representation z + self.gs = args.hs # size of graph state + self.bidir = True # whether to use bidirectional encoding + self.vid = True + self.input_type = 'DG' + self.num_sample = args.num_sample + + if self.vid: + self.vs = self.hs + self.max_n # vertex state size = hidden state + vid + else: + self.vs = self.hs + + # 0. encoding-related + self.grue_forward = nn.GRUCell(self.nvt, self.hs) # encoder GRU + self.grue_backward = nn.GRUCell(self.nvt, self.hs) # backward encoder GRU + self.fc1 = nn.Linear(self.gs, self.nz) # latent mean + self.fc2 = nn.Linear(self.gs, self.nz) # latent logvar + + # 2. gate-related + self.gate_forward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.gate_backward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.mapper_forward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) # disable bias to ensure padded zeros also mapped to zeros + self.mapper_backward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) + + # 3. bidir-related, to unify sizes + if self.bidir: + self.hv_unify = nn.Sequential( + nn.Linear(self.hs * 2, self.hs), + ) + self.hg_unify = nn.Sequential( + nn.Linear(self.gs * 2, self.gs), + ) + + # 4. other + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + self.tanh = nn.Tanh() + self.logsoftmax1 = nn.LogSoftmax(1) + + # 6. predictor + np = self.gs + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF').to(self.device) + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF').to(self.device) + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()).to(self.device) + + input_dim = 0 + if 'D' in self.input_type: + input_dim += self.nz + if 'G' in self.input_type: + input_dim += self.nz + + self.pred_fc = StandardDeepGP(backbone_params) + self.mseloss = nn.MSELoss(reduction='sum') + # self.nasbench201 = torch.load( + # os.path.join(args.data_path, 'nasbench201.pt')) + self.data_path = args.data_path + self.pred_fc.to(self.device) + self.inter_setpool.to(self.device) + self.intra_setpool.to(self.device) + self.grue_backward.to(self.device) + self.grue_forward.to(self.device) + self.gate_backward.to(self.device) + self.gate_forward.to(self.device) + self.mapper_backward.to(self.device) + self.mapper_forward.to(self.device) + self.hg_unify.to(self.device) + self.hv_unify.to(self.device) + self.fc1.to(self.device) + self.fc2.to(self.device) + + # def get_topk_idx(self, topk=1): + # self.mtrloader.dataset.set_mode('train') + # if self.nasbench201 is None: + # self.nasbench201 = torch.load( + # os.path.join(self.data_path, 'nasbench201.pt')) + # z_repr = [] + # g_repr = [] + # acc_repr = [] + # for x, g, acc in tqdm(self.mtrloader): + # str = decode_igraph_to_NAS_BENCH_201_string(g[0]) + # arch_idx = -1 + # for idx, arch_str in enumerate(self.nasbench201['arch']['str']): + # if arch_str == str: + # arch_idx = idx + # break + # g_repr.append(arch_idx) + # acc_repr.append(acc.detach().cpu().numpy()[0]) + # best = np.argsort(-1 * np.array(acc_repr))[:topk] + # self.nasbench201 = None + # return np.array(g_repr)[best], np.array(acc_repr)[best] + + def randomly_init_deepgp(self, ): + self.pred_fc = StandardDeepGP(self.config) + self.likelihood = gpytorch.likelihoods.GaussianLikelihood() + self.gp = ExactGPLayer(train_x=None, train_y=None, likelihood=self.likelihood, config=self.config, + dims=self.fixed_context_size) + self.mll = gpytorch.mlls.ExactMarginalLogLikelihood(self.likelihood, self.gp).to(self.device) + + + def setup_writers(self, ): + train_log_dir = os.path.join(self.checkpoint_path, "train") + os.makedirs(train_log_dir, exist_ok=True) + # self.train_summary_writer = SummaryWriter(train_log_dir) + + valid_log_dir = os.path.join(self.checkpoint_path, "valid") + os.makedirs(valid_log_dir, exist_ok=True) + # self.valid_summary_writer = SummaryWriter(valid_log_dir) + + def get_mu_and_std(self, x_support, y_support, x_query, y_query): + if x_support is not None: + x_support.to(self.device) + y_support.to(self.device) + + self.gp.set_train_data(inputs=x_support, targets=y_support, strict=False) + self.gp.to(self.device) + self.gp.eval() + self.likelihood.eval() + pred = self.likelihood(self.gp(x_query.to(self.device))) + mu = pred.mean.detach().to("cpu").numpy().reshape(-1, ) + stddev = pred.stddev.detach().to("cpu").numpy().reshape(-1, ) + return mu, stddev + + def predict_finetune(self, z, labels=None, train=False): + if len(labels) > 1: + z = torch.squeeze(z) + if train: + self.gp.set_train_data(inputs=z, targets=labels, strict=False) + y_dist = self.gp(z) + predictions = self.likelihood(y_dist) + return predictions.mean, y_dist + + def predict(self, D_mu, G_mu, labels=None, train=False): + input_vec = [] + if 'D' in self.input_type: + input_vec.append(D_mu) + if 'G' in self.input_type: + input_vec.append(G_mu) + print(input_vec) + input_vec = torch.cat(input_vec, dim=1) + z = self.pred_fc(input_vec).double() + if train: + self.gp.set_train_data(inputs=z, targets=labels, strict=False) + y_dist = self.gp(z.type(torch.DoubleTensor)) + predictions = self.likelihood(y_dist) + return predictions.mean, y_dist + + def get_data_and_graph_repr(self, x, g, matrix=False): + input_vec = [] + # self.pred_fc.to(self.device) + self.pred_fc.eval() + # self.inter_setpool.to(self.device) + self.inter_setpool.eval() + # self.intra_setpool.to(self.device) + self.intra_setpool.eval() + # self.grue_backward.to(self.device) + self.grue_backward.eval() + # self.grue_forward.to(self.device) + self.grue_forward.eval() + # self.gate_backward.to(self.device) + self.gate_backward.eval() + # self.gate_forward.to(self.device) + self.gate_forward.eval() + # self.mapper_backward.to(self.device) + self.mapper_backward.eval() + # self.mapper_forward.to(self.device) + self.mapper_forward.eval() + # self.hg_unify.to(self.device) + self.hg_unify.eval() + # self.hv_unify.to(self.device) + self.hv_unify.eval() + # self.fc1.to(self.device) + self.fc1.eval() + # self.fc2.to(self.device) + self.fc2.eval() + if 'D' in self.input_type: + input_vec.append(self.set_encode([x for i in range(len(g))]).to(self.device)) + if 'G' in self.input_type: + input_vec.append(self.graph_encode(g, matrix=matrix).squeeze()) + # print(input_vec) + if len(g) == 1: + input_vec = torch.cat(input_vec, dim=0) + print(input_vec) + else: + input_vec = torch.cat(input_vec, dim=1) + z = self.pred_fc(input_vec) + return z.detach().cpu().numpy().tolist() + + def get_device(self): + if self.device is None: + self.device = next(self.parameters()).device + return self.device + + def _get_zeros(self, n, length): + return torch.zeros(n, length).to(self.get_device()) # get a zero hidden state + + def _get_zero_hidden(self, n=1): + return self._get_zeros(n, self.hs) # get a zero hidden state + + def _one_hot(self, idx, length): + if type(idx) in [list, range]: + if idx == []: + return None + idx = torch.LongTensor(idx).unsqueeze(0).t() + x = torch.zeros((len(idx), length)).scatter_(1, idx, 1).to(self.get_device()) + else: + idx = torch.LongTensor([idx]).unsqueeze(0) + x = torch.zeros((1, length)).scatter_(1, idx, 1).to(self.get_device()) + return x + + def _gated(self, h, gate, mapper): + return gate(h) * mapper(h) + + def _collate_fn(self, G): + return [g.copy() for g in G] + + def _propagate_to(self, G, v, propagator, H=None, reverse=False, gate=None, mapper=None): + # propagate messages to vertex index v for all graphs in G + # return the new messages (states) at v + G = [g for g in G if g.vcount() > v] + if len(G) == 0: + return + if H is not None: + idx = [i for i, g in enumerate(G) if g.vcount() > v] + H = H[idx] + v_types = [g.vs[v]['type'] for g in G] + X = self._one_hot(v_types, self.nvt) + if reverse: + H_name = 'H_backward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.successors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.successors(v), self.max_n) for g in G] + gate, mapper = self.gate_backward, self.mapper_backward + else: + H_name = 'H_forward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.predecessors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.predecessors(v), self.max_n) for g in G] + if gate is None: + gate, mapper = self.gate_forward, self.mapper_forward + if self.vid: + H_pred = [[torch.cat([x[i], y[i:i + 1]], 1) for i in range(len(x))] for x, y in zip(H_pred, vids)] + # if h is not provided, use gated sum of v's predecessors' states as the input hidden state + if H is None: + max_n_pred = max([len(x) for x in H_pred]) # maximum number of predecessors + if max_n_pred == 0: + H = self._get_zero_hidden(len(G)) + else: + H_pred = [torch.cat(h_pred + + [self._get_zeros(max_n_pred - len(h_pred), self.vs)], 0).unsqueeze(0) + for h_pred in H_pred] # pad all to same length + H_pred = torch.cat(H_pred, 0) # batch * max_n_pred * vs + H = self._gated(H_pred, gate, mapper).sum(1) # batch * hs + Hv = propagator(X, H) + for i, g in enumerate(G): + g.vs[v][H_name] = Hv[i:i + 1] + return Hv + + def _propagate_from(self, G, v, propagator, H0=None, reverse=False): + # perform a series of propagation_to steps starting from v following a topo order + # assume the original vertex indices are in a topological order + if reverse: + prop_order = range(v, -1, -1) + else: + prop_order = range(v, self.max_n) + Hv = self._propagate_to(G, v, propagator, H0, reverse=reverse) # the initial vertex + for v_ in prop_order[1:]: + self._propagate_to(G, v_, propagator, reverse=reverse) + return Hv + + def _get_graph_state(self, G, decode=False): + # get the graph states + # when decoding, use the last generated vertex's state as the graph state + # when encoding, use the ending vertex state or unify the starting and ending vertex states + Hg = [] + for g in G: + hg = g.vs[g.vcount() - 1]['H_forward'] + if self.bidir and not decode: # decoding never uses backward propagation + hg_b = g.vs[0]['H_backward'] + hg = torch.cat([hg, hg_b], 1) + Hg.append(hg) + Hg = torch.cat(Hg, 0) + if self.bidir and not decode: + Hg = self.hg_unify(Hg) + return Hg + + def set_encode(self, X): + proto_batch = [] + for x in X: + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + return v + + def graph_encode(self, G, matrix=False): + # encode graphs G into latent vectors + if matrix: + mu = torch.Tensor([decode_igraph_to_NAS201_matrix(g).flatten() for g in G]).to(self.device) + else: + if type(G) != list: + G = [G] + self._propagate_from(G, 0, self.grue_forward, H0=self._get_zero_hidden(len(G)), + reverse=False) + if self.bidir: + self._propagate_from(G, self.max_n - 1, self.grue_backward, + H0=self._get_zero_hidden(len(G)), reverse=True) + Hg = self._get_graph_state(G) + mu = self.fc1(Hg) + # logvar = self.fc2(Hg) + return mu # , logvar + + def reparameterize(self, mu, logvar, eps_scale=0.01): + # return z ~ N(mu, std) + if self.training: + std = logvar.mul(0.5).exp_() + eps = torch.randn_like(std) * eps_scale + return eps.mul(std).add_(mu) + else: + return mu diff --git a/MobileNetV3/main_exp/transfer_nag_lib/ofa_net.py b/MobileNetV3/main_exp/transfer_nag_lib/ofa_net.py new file mode 100644 index 0000000..3f0fe29 --- /dev/null +++ b/MobileNetV3/main_exp/transfer_nag_lib/ofa_net.py @@ -0,0 +1,520 @@ +import numpy as np +import copy +import itertools +import random +import sys +import os +import pickle +import torch +from torch.nn.functional import one_hot + +# from ofa.imagenet_classification.run_manager import RunManager + +# from naszilla.nas_bench_201.distances import * + +# INPUT = 'input' +# OUTPUT = 'output' +# OPS = ['avg_pool_3x3', 'nor_conv_1x1', 'nor_conv_3x3', 'none', 'skip_connect'] +# NUM_OPS = len(OPS) +# OP_SPOTS = 20 +KS_LIST = [3, 5, 7] +EXPAND_LIST = [3, 4, 6] +DEPTH_LIST = [2, 3, 4] +NUM_STAGE = 5 +MAX_LAYER_PER_STAGE = 4 +MAX_N_BLOCK= NUM_STAGE * MAX_LAYER_PER_STAGE # 20 +OPS = { + '3-3': 0, '3-4': 1, '3-6': 2, + '5-3': 3, '5-4': 4, '5-6': 5, + '7-3': 6, '7-4': 7, '7-6': 8, + } + +OPS2STR = { + 0: '3-3', 1: '3-4', 2: '3-6', + 3: '5-3', 4: '5-4', 5: '5-6', + 6: '7-3', 7: '7-4', 8: '7-6', + } +NUM_OPS = len(OPS) +LONGEST_PATH_LENGTH = 20 +# OP_SPOTS = NUM_VERTICES - 2 + +# OPS = { +# '3-3': 0, '3-4': 1, '3-6': 2, +# '5-3': 3, '5-4': 4, '5-6': 5, +# '7-3': 6, '7-4': 7, '7-6': 8, +# } +# OFA evolution hyper-parameters +# self.arch_mutate_prob = kwargs.get("arch_mutate_prob", 0.1) +# self.resolution_mutate_prob = kwargs.get("resolution_mutate_prob", 0.5) +# self.population_size = kwargs.get("population_size", 100) +# self.max_time_budget = kwargs.get("max_time_budget", 500) +# self.parent_ratio = kwargs.get("parent_ratio", 0.25) +# self.mutation_ratio = kwargs.get("mutation_ratio", 0.5) + + +class OFASubNet: + + def __init__(self, string, accuracy_predictor=None): + self.string = string + self.accuracy_predictor = accuracy_predictor + # self.run_config = run_config + + def get_string(self): + return self.string + + def serialize(self): + return { + 'string':self.string + } + + @classmethod + def random_cell(cls, + nasbench, + max_nodes=None, + max_edges=None, + cutoff=None, + index_hash=None, + random_encoding=None): + """ + OFA sample random subnet + """ + # Randomly sample sub-networks from OFA network + random_subnet_config = nasbench.sample_active_subnet() + #{ + # "ks": ks_setting, + # "e": expand_setting, + # "d": depth_setting, + # } + + return {'string':cls.get_string_from_ops(random_subnet_config)} + + def encode(self, + predictor_encoding, + nasbench=None, + deterministic=True, + cutoff=None, + nasbench_ours=None, + dataset=None): + + if predictor_encoding == 'adj': + return self.encode_standard() + elif predictor_encoding == 'path': + raise NotImplementedError + return self.encode_paths() + elif predictor_encoding == 'trunc_path': + if not cutoff: + cutoff = 30 + dic = self.gcn_encoding(nasbench, + deterministic=deterministic, + nasbench_ours=nasbench_ours, + dataset=dataset) + dic['trunc_path'] = self.encode_freq_paths(cutoff=cutoff) + return dic + # return self.encode_freq_paths(cutoff=cutoff) + elif predictor_encoding == 'gcn': + return self.gcn_encoding(nasbench, + deterministic=deterministic, + nasbench_ours=nasbench_ours, + dataset=dataset) + else: + print('{} is an invalid predictor encoding'.format(predictor_encoding)) + raise NotImplementedError() + + def get_ops_onehot(self): + ops = self.get_op_dict() + # ops = [INPUT, *ops, OUTPUT] + node_types = torch.zeros(NUM_STAGE * MAX_LAYER_PER_STAGE).long() # w/o in / out + num_vertices = len(OPS.values()) + num_nodes = NUM_STAGE * MAX_LAYER_PER_STAGE + d_matrix = [] + # import pdb; pdb.set_trace() + for i in range(NUM_STAGE): + ds = ops['d'][i] + for j in range(ds): + d_matrix.append(ds) + + for j in range(MAX_LAYER_PER_STAGE - ds): + d_matrix.append('none') + + for i, (ks, e, d) in enumerate(zip( + ops['ks'], ops['e'], d_matrix)): + if d == 'none': + # node_types[i] = OPS[d] + pass + else: + node_types[i] = OPS[f'{ks}-{e}'] + + ops_onehot = one_hot(node_types, num_vertices).float() + return ops_onehot + + + def gcn_encoding(self, nasbench, deterministic, nasbench_ours=None, dataset=None): + + # op_map = [OUTPUT, INPUT, *OPS] + ops = self.get_op_dict() + # ops = [INPUT, *ops, OUTPUT] + node_types = torch.zeros(NUM_STAGE * MAX_LAYER_PER_STAGE).long() # w/o in / out + num_vertices = len(OPS.values()) + num_nodes = NUM_STAGE * MAX_LAYER_PER_STAGE + d_matrix = [] + # import pdb; pdb.set_trace() + for i in range(NUM_STAGE): + ds = ops['d'][i] + for j in range(ds): + d_matrix.append(ds) + + for j in range(MAX_LAYER_PER_STAGE - ds): + d_matrix.append('none') + + for i, (ks, e, d) in enumerate(zip( + ops['ks'], ops['e'], d_matrix)): + if d == 'none': + # node_types[i] = OPS[d] + pass + else: + node_types[i] = OPS[f'{ks}-{e}'] + + ops_onehot = one_hot(node_types, num_vertices).float() + val_loss = self.get_val_loss(nasbench, dataset=dataset) + test_loss = copy.deepcopy(val_loss) + # (num node, ops types) --> (20, 28) + def get_adj(): + adj = torch.zeros(num_nodes, num_nodes) + for i in range(num_nodes-1): + adj[i, i+1] = 1 + adj = np.array(adj) + return adj + + matrix = get_adj() + + dic = { + 'num_vertices': num_vertices, + 'adjacency': matrix, + 'operations': ops_onehot, + 'mask': np.array([i < num_vertices for i in range(num_vertices)], dtype=np.float32), + 'val_acc': 1.0 - val_loss, + 'test_acc': 1.0 - test_loss, + 'x': ops_onehot + } + + return dic + + def get_runtime(self, nasbench, dataset='cifar100'): + return nasbench.query_by_index(index, dataset).get_eval('x-valid')['time'] + + def get_val_loss(self, nasbench, deterministic=1, dataset='cifar100'): + assert dataset == 'imagenet1k' + # SuperNet version + # ops = self.get_op_dict() + # nasbench.set_active_subnet(ks=ops['ks'], e=ops['e'], d=ops['d']) + + # subnet = nasbench.get_active_subnet(preserve_weight=True) + # run_manager = RunManager(".tmp/eval_subnet", subnet, self.run_config, init=False) + # # assign image size: 128, 132, ..., 224 + # self.run_config.data_provider.assign_active_img_size(224) + # run_manager.reset_running_statistics(net=subnet) + + # loss, (top1, top5) = run_manager.validate(net=subnet) + # # print("Results: loss=%.5f,\t top1=%.1f,\t top5=%.1f" % (loss, top1, top5)) + # self.loss = loss + # self.top1 = top1 + # self.top5 = top5 + + # accuracy predictor version + ops = self.get_op_dict() + # resolutions = [160, 176, 192, 208, 224] + # ops['r'] = [random.choice(resolutions)] + ops['r'] = [224] + acc = self.accuracy_predictor.predict_accuracy([ops])[0][0].item() + return 1.0 - acc + + def get_test_loss(self, nasbench, dataset='cifar100', deterministic=1): + ops = self.get_op_dict() + # resolutions = [160, 176, 192, 208, 224] + # ops['r'] = [random.choice(resolutions)] + ops['r'] = [224] + acc = self.accuracy_predictor.predict_accuracy([ops])[0][0].item() + return 1.0 - acc + + def get_op_dict(self): + # given a string, get the list of operations + ops = { + "ks": [], "e": [], "d": [] + } + tokens = self.string.split('_') + for i, token in enumerate(tokens): + d, ks, e = token.split('-') + if i % MAX_LAYER_PER_STAGE == 0: + ops['d'].append(int(d)) + ops['ks'].append(int(ks)) + ops['e'].append(int(e)) + return ops + + def get_num(self): + # compute the unique number of the architecture, in [0, 15624] + ops = self.get_op_dict() + index = 0 + for i, op in enumerate(ops): + index += OPS.index(op) * NUM_OPS ** i + return index + + def get_random_hash(self): + num = self.get_num() + hashes = pickle.load(open('nas_bench_201/random_hash.pkl', 'rb')) + return hashes[num] + + @classmethod + def get_string_from_ops(cls, ops): + string = '' + for i, (ks, e) in enumerate(zip(ops['ks'], ops['e'])): + d = ops['d'][int(i/MAX_LAYER_PER_STAGE)] + string += f'{d}-{ks}-{e}_' + return string[:-1] + + def perturb(self, + nasbench, + mutation_rate=1): + # deterministic version of mutate + ops = self.get_op_dict() + new_ops = [] + num = np.random.choice(len(ops)) + for i, op in enumerate(ops): + if i == num: + available = [o for o in OPS if o != op] + new_ops.append(np.random.choice(available)) + else: + new_ops.append(op) + return {'string':self.get_string_from_ops(new_ops)} + + def mutate(self, + nasbench, + mutation_rate=0.1, + mutate_encoding='adj', + index_hash=None, + cutoff=30, + patience=5000): + p = 0 + mutation_rate = mutation_rate / 10 # OFA rate: 0.1 + arch_dict = self.get_op_dict() + + if mutate_encoding == 'adj': + # OFA version mutation + # https://github.com/mit-han-lab/once-for-all/blob/master/ofa/nas/search_algorithm/evolution.py + for i in range(MAX_N_BLOCK): + if random.random() < mutation_rate: + available_ks = [ks for ks in KS_LIST if ks != arch_dict["ks"][i]] + available_e = [e for e in EXPAND_LIST if e != arch_dict["e"][i]] + arch_dict["ks"][i] = random.choice(available_ks) + arch_dict["e"][i] = random.choice(available_e) + + for i in range(NUM_STAGE): + if random.random() < mutation_rate: + available_d = [d for d in DEPTH_LIST if d != arch_dict["d"][i]] + arch_dict["d"][i] = random.choice(available_d) + return {'string':self.get_string_from_ops(arch_dict)} + + elif mutate_encoding in ['path', 'trunc_path']: + raise NotImplementedError() + else: + print('{} is an invalid mutate encoding'.format(mutate_encoding)) + raise NotImplementedError() + + def encode_standard(self): + """ + compute the standard encoding + """ + ops = self.get_op_dict() + encoding = [] + for i, (ks, e) in enumerate(zip(ops['ks'], ops['e'])): + string = f'{ks}-{e}' + encoding.append(OPS[string]) + return encoding + + def encode_one_hot(self): + """ + compute the one-hot encoding + """ + encoding = self.encode_standard() + one_hot = [] + for num in encoding: + for i in range(len(OPS)): + if i == num: + one_hot.append(1) + else: + one_hot.append(0) + return one_hot + + def get_num_params(self, nasbench): + # todo: add this method + return 100 + + def get_paths(self): + """ + return all paths from input to output + """ + path_blueprints = [[3], [0,4], [1,5], [0,2,5]] + ops = self.get_op_dict() + paths = [] + for blueprint in path_blueprints: + paths.append([ops[node] for node in blueprint]) + return paths + + def get_path_indices(self): + """ + compute the index of each path + """ + paths = self.get_paths() + path_indices = [] + + for i, path in enumerate(paths): + if i == 0: + index = 0 + elif i in [1, 2]: + index = NUM_OPS + else: + index = NUM_OPS + NUM_OPS ** 2 + import pdb; pdb.set_trace() + for j, op in enumerate(path): + index += OPS.index(op) * NUM_OPS ** j + path_indices.append(index) + + return tuple(path_indices) + + def encode_paths(self): + """ output one-hot encoding of paths """ + num_paths = sum([NUM_OPS ** i for i in range(1, LONGEST_PATH_LENGTH + 1)]) + path_indices = self.get_path_indices() + encoding = np.zeros(num_paths) + for index in path_indices: + encoding[index] = 1 + return encoding + + def encode_freq_paths(self, cutoff=30): + # natural cutoffs 5, 30, 155 (last) + num_paths = sum([NUM_OPS ** i for i in range(1, LONGEST_PATH_LENGTH + 1)]) + path_indices = self.get_path_indices() + encoding = np.zeros(cutoff) + for index in range(min(num_paths, cutoff)): + if index in path_indices: + encoding[index] = 1 + return encoding + + def distance(self, other, dist_type, cutoff=30): + if dist_type == 'adj': + distance = adj_distance(self, other) + elif dist_type == 'path': + distance = path_distance(self, other) + elif dist_type == 'trunc_path': + distance = path_distance(self, other, cutoff=cutoff) + elif dist_type == 'nasbot': + distance = nasbot_distance(self, other) + else: + print('{} is an invalid distance'.format(distance)) + raise NotImplementedError() + return distance + + + def get_neighborhood(self, + nasbench, + mutate_encoding, + shuffle=True): + nbhd = [] + ops = self.get_op_dict() + + if mutate_encoding == 'adj': + # OFA version mutation variation + # https://github.com/mit-han-lab/once-for-all/blob/master/ofa/nas/search_algorithm/evolution.py + for i in range(MAX_N_BLOCK): + available_ks = [ks for ks in KS_LIST if ks != ops["ks"][i]] + for ks in available_ks: + new_ops = ops.copy() + new_ops["ks"][i] = ks + new_arch = {'string':self.get_string_from_ops(new_ops)} + nbhd.append(new_arch) + + available_e = [e for e in EXPAND_LIST if e != ops["e"][i]] + for e in available_e: + new_ops = ops.copy() + new_ops["e"][i] = e + new_arch = {'string':self.get_string_from_ops(new_ops)} + nbhd.append(new_arch) + # for i in range(MAX_N_BLOCK): + # available_ks = [ks for ks in KS_LIST if ks != ops["ks"][i]] + # available_e = [e for e in EXPAND_LIST if e != ops["e"][i]] + # for ks, e in zip(available_ks, available_e): + # new_ops = ops.copy() + # new_ops["ks"][i] = ks + # new_ops["e"][i] = e + # new_arch = {'string':self.get_string_from_ops(new_ops)} + # nbhd.append(new_arch) + + for i in range(NUM_STAGE): + available_d = [d for d in DEPTH_LIST if d != ops["d"][i]] + for d in available_d: + new_ops = ops.copy() + new_ops["d"][i] = d + new_arch = {'string':self.get_string_from_ops(new_ops)} + nbhd.append(new_arch) + + # if mutate_encoding == 'adj': + # for i in range(len(ops)): + # import pdb; pdb.set_trace() + # available = [op for op in OPS.keys() if op != ops[i]] + # for op in available: + # new_ops = ops.copy() + # new_ops[i] = op + # new_arch = {'string':self.get_string_from_ops(new_ops)} + # nbhd.append(new_arch) + + elif mutate_encoding in ['path', 'trunc_path']: + + if mutate_encoding == 'trunc_path': + path_blueprints = [[3], [0,4], [1,5]] + else: + path_blueprints = [[3], [0,4], [1,5], [0,2,5]] + ops = self.get_op_dict() + + for blueprint in path_blueprints: + for new_path in itertools.product(OPS, repeat=len(blueprint)): + new_ops = ops.copy() + + for i, op in enumerate(new_path): + new_ops[blueprint[i]] = op + + # check if it's the same + same = True + for j in range(len(ops)): + if ops[j] != new_ops[j]: + same = False + if not same: + new_arch = {'string':self.get_string_from_ops(new_ops)} + nbhd.append(new_arch) + else: + print('{} is an invalid mutate encoding'.format(mutate_encoding)) + raise NotImplementedError() + + if shuffle: + random.shuffle(nbhd) + return nbhd + + + def get_unique_string(self): + ops = self.get_op_dict() + d_matrix = [] + for i in range(NUM_STAGE): + ds = ops['d'][i] + for j in range(ds): + d_matrix.append(ds) + + for j in range(MAX_LAYER_PER_STAGE - ds): + d_matrix.append('none') + + string = '' + for i, (ks, e, d) in enumerate(zip(ops['ks'], ops['e'], d_matrix)): + if d == 'none': + string += f'0-0-0_' + else: + string += f'{d}-{ks}-{e}_' + return string[:-1] + + diff --git a/MobileNetV3/models/GDSS/attention.py b/MobileNetV3/models/GDSS/attention.py new file mode 100644 index 0000000..a2fed4e --- /dev/null +++ b/MobileNetV3/models/GDSS/attention.py @@ -0,0 +1,117 @@ +import math +import torch +from torch.nn import Parameter +import torch.nn.functional as F + +from models.GDSS.layers import DenseGCNConv, MLP +# from ..utils.graph_utils import mask_adjs, mask_x +from .graph_utils import mask_x, mask_adjs + + +# -------- Graph Multi-Head Attention (GMH) -------- +# -------- From Baek et al. (2021) -------- +class Attention(torch.nn.Module): + + def __init__(self, in_dim, attn_dim, out_dim, num_heads=4, conv='GCN'): + super(Attention, self).__init__() + self.num_heads = num_heads + self.attn_dim = attn_dim + self.out_dim = out_dim + self.conv = conv + + self.gnn_q, self.gnn_k, self.gnn_v = self.get_gnn(in_dim, attn_dim, out_dim, conv) + self.activation = torch.tanh + self.softmax_dim = 2 + + def forward(self, x, adj, flags, attention_mask=None): + + if self.conv == 'GCN': + Q = self.gnn_q(x, adj) + K = self.gnn_k(x, adj) + else: + Q = self.gnn_q(x) + K = self.gnn_k(x) + + V = self.gnn_v(x, adj) + dim_split = self.attn_dim // self.num_heads + Q_ = torch.cat(Q.split(dim_split, 2), 0) + K_ = torch.cat(K.split(dim_split, 2), 0) + + if attention_mask is not None: + attention_mask = torch.cat([attention_mask for _ in range(self.num_heads)], 0) + attention_score = Q_.bmm(K_.transpose(1,2))/math.sqrt(self.out_dim) + A = self.activation( attention_mask + attention_score ) + else: + A = self.activation( Q_.bmm(K_.transpose(1,2))/math.sqrt(self.out_dim) ) # (B x num_heads) x N x N + + # -------- (B x num_heads) x N x N -------- + A = A.view(-1, *adj.shape) + A = A.mean(dim=0) + A = (A + A.transpose(-1,-2))/2 + + return V, A + + def get_gnn(self, in_dim, attn_dim, out_dim, conv='GCN'): + + if conv == 'GCN': + gnn_q = DenseGCNConv(in_dim, attn_dim) + gnn_k = DenseGCNConv(in_dim, attn_dim) + gnn_v = DenseGCNConv(in_dim, out_dim) + + return gnn_q, gnn_k, gnn_v + + elif conv == 'MLP': + num_layers=2 + gnn_q = MLP(num_layers, in_dim, 2*attn_dim, attn_dim, activate_func=torch.tanh) + gnn_k = MLP(num_layers, in_dim, 2*attn_dim, attn_dim, activate_func=torch.tanh) + gnn_v = DenseGCNConv(in_dim, out_dim) + + return gnn_q, gnn_k, gnn_v + + else: + raise NotImplementedError(f'{conv} not implemented.') + + +# -------- Layer of ScoreNetworkA -------- +class AttentionLayer(torch.nn.Module): + + def __init__(self, num_linears, conv_input_dim, attn_dim, conv_output_dim, input_dim, output_dim, + num_heads=4, conv='GCN'): + + super(AttentionLayer, self).__init__() + + self.attn = torch.nn.ModuleList() + for _ in range(input_dim): + self.attn_dim = attn_dim + self.attn.append(Attention(conv_input_dim, self.attn_dim, conv_output_dim, + num_heads=num_heads, conv=conv)) + + self.hidden_dim = 2*max(input_dim, output_dim) + self.mlp = MLP(num_linears, 2*input_dim, self.hidden_dim, output_dim, use_bn=False, activate_func=F.elu) + self.multi_channel = MLP(2, input_dim*conv_output_dim, self.hidden_dim, conv_output_dim, + use_bn=False, activate_func=F.elu) + + def forward(self, x, adj, flags): + """ + + :param x: B x N x F_i + :param adj: B x C_i x N x N + :return: x_out: B x N x F_o, adj_out: B x C_o x N x N + """ + mask_list = [] + x_list = [] + for _ in range(len(self.attn)): + _x, mask = self.attn[_](x, adj[:,_,:,:], flags) + mask_list.append(mask.unsqueeze(-1)) + x_list.append(_x) + x_out = mask_x(self.multi_channel(torch.cat(x_list, dim=-1)), flags) + x_out = torch.tanh(x_out) + + mlp_in = torch.cat([torch.cat(mask_list, dim=-1), adj.permute(0,2,3,1)], dim=-1) + shape = mlp_in.shape + mlp_out = self.mlp(mlp_in.view(-1, shape[-1])) + _adj = mlp_out.view(shape[0], shape[1], shape[2], -1).permute(0,3,1,2) + _adj = _adj + _adj.transpose(-1,-2) + adj_out = mask_adjs(_adj, flags) + + return x_out, adj_out \ No newline at end of file diff --git a/MobileNetV3/models/GDSS/graph_utils.py b/MobileNetV3/models/GDSS/graph_utils.py new file mode 100644 index 0000000..8563daf --- /dev/null +++ b/MobileNetV3/models/GDSS/graph_utils.py @@ -0,0 +1,209 @@ +import torch +import torch.nn.functional as F +import networkx as nx +import numpy as np + + +# -------- Mask batch of node features with 0-1 flags tensor -------- +def mask_x(x, flags): + + if flags is None: + flags = torch.ones((x.shape[0], x.shape[1]), device=x.device) + return x * flags[:,:,None] + + +# -------- Mask batch of adjacency matrices with 0-1 flags tensor -------- +def mask_adjs(adjs, flags): + """ + :param adjs: B x N x N or B x C x N x N + :param flags: B x N + :return: + """ + if flags is None: + flags = torch.ones((adjs.shape[0], adjs.shape[-1]), device=adjs.device) + + if len(adjs.shape) == 4: + flags = flags.unsqueeze(1) # B x 1 x N + adjs = adjs * flags.unsqueeze(-1) + adjs = adjs * flags.unsqueeze(-2) + return adjs + + +# -------- Create flags tensor from graph dataset -------- +def node_flags(adj, eps=1e-5): + + flags = torch.abs(adj).sum(-1).gt(eps).to(dtype=torch.float32) + + if len(flags.shape)==3: + flags = flags[:,0,:] + return flags + + +# -------- Create initial node features -------- +def init_features(init, adjs=None, nfeat=10): + + if init=='zeros': + feature = torch.zeros((adjs.size(0), adjs.size(1), nfeat), dtype=torch.float32, device=adjs.device) + elif init=='ones': + feature = torch.ones((adjs.size(0), adjs.size(1), nfeat), dtype=torch.float32, device=adjs.device) + elif init=='deg': + feature = adjs.sum(dim=-1).to(torch.long) + num_classes = nfeat + try: + feature = F.one_hot(feature, num_classes=num_classes).to(torch.float32) + except: + print(feature.max().item()) + raise NotImplementedError(f'max_feat_num mismatch') + else: + raise NotImplementedError(f'{init} not implemented') + + flags = node_flags(adjs) + + return mask_x(feature, flags) + + +# -------- Sample initial flags tensor from the training graph set -------- +def init_flags(graph_list, config, batch_size=None): + if batch_size is None: + batch_size = config.data.batch_size + max_node_num = config.data.max_node_num + graph_tensor = graphs_to_tensor(graph_list, max_node_num) + idx = np.random.randint(0, len(graph_list), batch_size) + flags = node_flags(graph_tensor[idx]) + + return flags + + +# -------- Generate noise -------- +def gen_noise(x, flags, sym=True): + z = torch.randn_like(x) + if sym: + z = z.triu(1) + z = z + z.transpose(-1,-2) + z = mask_adjs(z, flags) + else: + z = mask_x(z, flags) + return z + + +# -------- Quantize generated graphs -------- +def quantize(adjs, thr=0.5): + adjs_ = torch.where(adjs < thr, torch.zeros_like(adjs), torch.ones_like(adjs)) + return adjs_ + + +# -------- Quantize generated molecules -------- +# adjs: 32 x 9 x 9 +def quantize_mol(adjs): + if type(adjs).__name__ == 'Tensor': + adjs = adjs.detach().cpu() + else: + adjs = torch.tensor(adjs) + adjs[adjs >= 2.5] = 3 + adjs[torch.bitwise_and(adjs >= 1.5, adjs < 2.5)] = 2 + adjs[torch.bitwise_and(adjs >= 0.5, adjs < 1.5)] = 1 + adjs[adjs < 0.5] = 0 + return np.array(adjs.to(torch.int64)) + + +def adjs_to_graphs(adjs, is_cuda=False): + graph_list = [] + for adj in adjs: + if is_cuda: + adj = adj.detach().cpu().numpy() + G = nx.from_numpy_matrix(adj) + G.remove_edges_from(nx.selfloop_edges(G)) + G.remove_nodes_from(list(nx.isolates(G))) + if G.number_of_nodes() < 1: + G.add_node(1) + graph_list.append(G) + return graph_list + + +# -------- Check if the adjacency matrices are symmetric -------- +def check_sym(adjs, print_val=False): + sym_error = (adjs-adjs.transpose(-1,-2)).abs().sum([0,1,2]) + if not sym_error < 1e-2: + raise ValueError(f'Not symmetric: {sym_error:.4e}') + if print_val: + print(f'{sym_error:.4e}') + + +# -------- Create higher order adjacency matrices -------- +def pow_tensor(x, cnum): + # x : B x N x N + x_ = x.clone() + xc = [x.unsqueeze(1)] + for _ in range(cnum-1): + x_ = torch.bmm(x_, x) + xc.append(x_.unsqueeze(1)) + xc = torch.cat(xc, dim=1) + + return xc + + +# -------- Create padded adjacency matrices -------- +def pad_adjs(ori_adj, node_number): + a = ori_adj + ori_len = a.shape[-1] + if ori_len == node_number: + return a + if ori_len > node_number: + raise ValueError(f'ori_len {ori_len} > node_number {node_number}') + a = np.concatenate([a, np.zeros([ori_len, node_number - ori_len])], axis=-1) + a = np.concatenate([a, np.zeros([node_number - ori_len, node_number])], axis=0) + return a + + +def graphs_to_tensor(graph_list, max_node_num): + adjs_list = [] + max_node_num = max_node_num + + for g in graph_list: + assert isinstance(g, nx.Graph) + node_list = [] + for v, feature in g.nodes.data('feature'): + node_list.append(v) + + adj = nx.to_numpy_matrix(g, nodelist=node_list) + padded_adj = pad_adjs(adj, node_number=max_node_num) + adjs_list.append(padded_adj) + + del graph_list + + adjs_np = np.asarray(adjs_list) + del adjs_list + + adjs_tensor = torch.tensor(adjs_np, dtype=torch.float32) + del adjs_np + + return adjs_tensor + + +def graphs_to_adj(graph, max_node_num): + max_node_num = max_node_num + + assert isinstance(graph, nx.Graph) + node_list = [] + for v, feature in graph.nodes.data('feature'): + node_list.append(v) + + adj = nx.to_numpy_matrix(graph, nodelist=node_list) + padded_adj = pad_adjs(adj, node_number=max_node_num) + + adj = torch.tensor(padded_adj, dtype=torch.float32) + del padded_adj + + return adj + + +def node_feature_to_matrix(x): + """ + :param x: BS x N x F + :return: + x_pair: BS x N x N x 2F + """ + x_b = x.unsqueeze(-2).expand(x.size(0), x.size(1), x.size(1), -1) # BS x N x N x F + x_pair = torch.cat([x_b, x_b.transpose(1, 2)], dim=-1) # BS x N x N x 2F + + return x_pair \ No newline at end of file diff --git a/MobileNetV3/models/GDSS/layers.py b/MobileNetV3/models/GDSS/layers.py new file mode 100644 index 0000000..3988d70 --- /dev/null +++ b/MobileNetV3/models/GDSS/layers.py @@ -0,0 +1,153 @@ +import torch +from torch.nn import Parameter +import torch.nn.functional as F +import math +from typing import Any + + +def glorot(tensor): + if tensor is not None: + stdv = math.sqrt(6.0 / (tensor.size(-2) + tensor.size(-1))) + tensor.data.uniform_(-stdv, stdv) + +def zeros(tensor): + if tensor is not None: + tensor.data.fill_(0) + +def reset(value: Any): + if hasattr(value, 'reset_parameters'): + value.reset_parameters() + else: + for child in value.children() if hasattr(value, 'children') else []: + reset(child) + +# -------- GCN layer -------- +class DenseGCNConv(torch.nn.Module): + r"""See :class:`torch_geometric.nn.conv.GCNConv`. + """ + def __init__(self, in_channels, out_channels, improved=False, bias=True): + super(DenseGCNConv, self).__init__() + + self.in_channels = in_channels + self.out_channels = out_channels + self.improved = improved + + self.weight = Parameter(torch.Tensor(self.in_channels, out_channels)) + + if bias: + self.bias = Parameter(torch.Tensor(out_channels)) + else: + self.register_parameter('bias', None) + + self.reset_parameters() + + def reset_parameters(self): + glorot(self.weight) + zeros(self.bias) + + + def forward(self, x, adj, mask=None, add_loop=True): + r""" + Args: + x (Tensor): Node feature tensor :math:`\mathbf{X} \in \mathbb{R}^{B + \times N \times F}`, with batch-size :math:`B`, (maximum) + number of nodes :math:`N` for each graph, and feature + dimension :math:`F`. + adj (Tensor): Adjacency tensor :math:`\mathbf{A} \in \mathbb{R}^{B + \times N \times N}`. The adjacency tensor is broadcastable in + the batch dimension, resulting in a shared adjacency matrix for + the complete batch. + mask (BoolTensor, optional): Mask matrix + :math:`\mathbf{M} \in {\{ 0, 1 \}}^{B \times N}` indicating + the valid nodes for each graph. (default: :obj:`None`) + add_loop (bool, optional): If set to :obj:`False`, the layer will + not automatically add self-loops to the adjacency matrices. + (default: :obj:`True`) + """ + x = x.unsqueeze(0) if x.dim() == 2 else x + adj = adj.unsqueeze(0) if adj.dim() == 2 else adj + B, N, _ = adj.size() + + if add_loop: + adj = adj.clone() + idx = torch.arange(N, dtype=torch.long, device=adj.device) + adj[:, idx, idx] = 1 if not self.improved else 2 + + out = torch.matmul(x, self.weight) + deg_inv_sqrt = adj.sum(dim=-1).clamp(min=1).pow(-0.5) + + adj = deg_inv_sqrt.unsqueeze(-1) * adj * deg_inv_sqrt.unsqueeze(-2) + out = torch.matmul(adj, out) + + if self.bias is not None: + out = out + self.bias + + if mask is not None: + out = out * mask.view(B, N, 1).to(x.dtype) + + return out + + + def __repr__(self): + return '{}({}, {})'.format(self.__class__.__name__, self.in_channels, + self.out_channels) + +# -------- MLP layer -------- +class MLP(torch.nn.Module): + def __init__(self, num_layers, input_dim, hidden_dim, output_dim, use_bn=False, activate_func=F.relu): + """ + num_layers: number of layers in the neural networks (EXCLUDING the input layer). If num_layers=1, this reduces to linear model. + input_dim: dimensionality of input features + hidden_dim: dimensionality of hidden units at ALL layers + output_dim: number of classes for prediction + num_classes: the number of classes of input, to be treated with different gains and biases, + (see the definition of class `ConditionalLayer1d`) + """ + + super(MLP, self).__init__() + + self.linear_or_not = True # default is linear model + self.num_layers = num_layers + self.use_bn = use_bn + self.activate_func = activate_func + + if num_layers < 1: + raise ValueError("number of layers should be positive!") + elif num_layers == 1: + # Linear model + self.linear = torch.nn.Linear(input_dim, output_dim) + else: + # Multi-layer model + self.linear_or_not = False + self.linears = torch.nn.ModuleList() + + self.linears.append(torch.nn.Linear(input_dim, hidden_dim)) + for layer in range(num_layers - 2): + self.linears.append(torch.nn.Linear(hidden_dim, hidden_dim)) + self.linears.append(torch.nn.Linear(hidden_dim, output_dim)) + + if self.use_bn: + self.batch_norms = torch.nn.ModuleList() + for layer in range(num_layers - 1): + self.batch_norms.append(torch.nn.BatchNorm1d(hidden_dim)) + + + def forward(self, x): + """ + :param x: [num_classes * batch_size, N, F_i], batch of node features + note that in self.cond_layers[layer], + `x` is splited into `num_classes` groups in dim=0, + and then treated with different gains and biases + """ + if self.linear_or_not: + # If linear model + return self.linear(x) + else: + # If MLP + h = x + for layer in range(self.num_layers - 1): + h = self.linears[layer](h) + if self.use_bn: + h = self.batch_norms[layer](h) + h = self.activate_func(h) + return self.linears[self.num_layers - 1](h) diff --git a/MobileNetV3/models/GDSS/scorenetx.py b/MobileNetV3/models/GDSS/scorenetx.py new file mode 100644 index 0000000..b48f921 --- /dev/null +++ b/MobileNetV3/models/GDSS/scorenetx.py @@ -0,0 +1,103 @@ +import torch +import torch.nn.functional as F + +from models.GDSS.layers import DenseGCNConv, MLP +from .graph_utils import mask_x, pow_tensor +from .attention import AttentionLayer +from .. import utils + +@utils.register_model(name='ScoreNetworkX') +class ScoreNetworkX(torch.nn.Module): + + # def __init__(self, max_feat_num, depth, nhid): + def __init__(self, config): + + super(ScoreNetworkX, self).__init__() + + self.nfeat = config.data.n_vocab + self.depth = config.model.depth + self.nhid = config.model.nhid + + self.layers = torch.nn.ModuleList() + for _ in range(self.depth): + if _ == 0: + self.layers.append(DenseGCNConv(self.nfeat, self.nhid)) + else: + self.layers.append(DenseGCNConv(self.nhid, self.nhid)) + + self.fdim = self.nfeat + self.depth * self.nhid + self.final = MLP(num_layers=3, input_dim=self.fdim, hidden_dim=2*self.fdim, output_dim=self.nfeat, + use_bn=False, activate_func=F.elu) + + self.activation = torch.tanh + + def forward(self, x, time_cond, maskX, flags=None): + + x_list = [x] + for _ in range(self.depth): + x = self.layers[_](x, maskX) + x = self.activation(x) + x_list.append(x) + + xs = torch.cat(x_list, dim=-1) # B x N x (F + num_layers x H) + out_shape = (x.shape[0], x.shape[1], -1) + x = self.final(xs).view(*out_shape) + + x = mask_x(x, flags) + return x + + +@utils.register_model(name='ScoreNetworkX_GMH') +class ScoreNetworkX_GMH(torch.nn.Module): + # def __init__(self, max_feat_num, depth, nhid, num_linears, + # c_init, c_hid, c_final, adim, num_heads=4, conv='GCN'): + def __init__(self, config): + super().__init__() + + self.max_feat_num = config.data.n_vocab + self.depth = config.model.depth + self.nhid = config.model.nhid + self.c_init = config.model.c_init + self.c_hid = config.model.c_hid + self.c_final = config.model.c_final + self.num_linears = config.model.num_linears + self.num_heads = config.model.num_heads + self.conv = config.model.conv + self.adim = config.model.adim + + self.layers = torch.nn.ModuleList() + for _ in range(self.depth): + if _ == 0: + self.layers.append(AttentionLayer(self.num_linears, self.max_feat_num, + self.nhid, self.nhid, self.c_init, + self.c_hid, self.num_heads, self.conv)) + elif _ == self.depth - 1: + self.layers.append(AttentionLayer(self.num_linears, self.nhid, self.adim, + self.nhid, self.c_hid, + self.c_final, self.num_heads, self.conv)) + else: + self.layers.append(AttentionLayer(self.num_linears, self.nhid, self.adim, + self.nhid, self.c_hid, + self.c_hid, self.num_heads, self.conv)) + + fdim = self.max_feat_num + self.depth * self.nhid + self.final = MLP(num_layers=3, input_dim=fdim, hidden_dim=2*fdim, output_dim=self.max_feat_num, + use_bn=False, activate_func=F.elu) + + self.activation = torch.tanh + + def forward(self, x, time_cond, maskX, flags=None): + adjc = pow_tensor(maskX, self.c_init) + + x_list = [x] + for _ in range(self.depth): + x, adjc = self.layers[_](x, adjc, flags) + x = self.activation(x) + x_list.append(x) + + xs = torch.cat(x_list, dim=-1) # B x N x (F + num_layers x H) + out_shape = (x.shape[0], x.shape[1], -1) + x = self.final(xs).view(*out_shape) + x = mask_x(x, flags) + + return x \ No newline at end of file diff --git a/MobileNetV3/models/__init__.py b/MobileNetV3/models/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/MobileNetV3/models/cate.py b/MobileNetV3/models/cate.py new file mode 100644 index 0000000..9118cb7 --- /dev/null +++ b/MobileNetV3/models/cate.py @@ -0,0 +1,352 @@ +import torch.nn as nn +import torch +import functools +from torch_geometric.utils import dense_to_sparse + +from . import utils, layers, gnns + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .transformer import Encoder, SemanticEmbedding +from models.GDSS.layers import MLP +from .set_encoder.setenc_models import SetPool + +""" Transformer Encoder """ +class GraphEncoder(nn.Module): + def __init__(self, config): + super(GraphEncoder, self).__init__() + # Forward Transformers + self.encoder_f = Encoder(config) + + def forward(self, x, mask): + h_f, hs_f, attns_f = self.encoder_f(x, mask) + h = torch.cat(hs_f, dim=-1) + return h + + + @staticmethod + def get_embeddings(h_x): + h_x = h_x.cpu() + return h_x[:, -1] + +class CLSHead(nn.Module): + def __init__(self, config, init_weights=None): + super(CLSHead, self).__init__() + self.layer_1 = nn.Linear(config.d_model, config.d_model) + self.dropout = nn.Dropout(p=config.dropout) + self.layer_2 = nn.Linear(config.d_model, config.n_vocab) + if init_weights is not None: + self.layer_2.weight = init_weights + + def forward(self, x): + x = self.dropout(torch.tanh(self.layer_1(x))) + return F.log_softmax(self.layer_2(x), dim=-1) + + +@utils.register_model(name='CATE') +class CATE(nn.Module): + def __init__(self, config): + super(CATE, self).__init__() + # Shared Embedding Layer + self.opEmb = SemanticEmbedding(config.model.graph_encoder) + self.dropout_op = nn.Dropout(p=config.model.dropout) + self.d_model = config.model.graph_encoder.d_model + self.act = act = get_act(config) + # Time + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + + # 2 GraphEncoder for X and Y + self.graph_encoder = GraphEncoder(config.model.graph_encoder) + + self.fdim = int(config.model.graph_encoder.n_layers * config.model.graph_encoder.d_model) + self.final = MLP(num_layers=3, input_dim=self.fdim, hidden_dim=2*self.fdim, output_dim=config.data.n_vocab, + use_bn=False, activate_func=F.elu) + + self.pos_enc_type = config.model.pos_enc_type + self.pos_encoder = PositionalEncoding_StageWise(d_model=self.d_model, max_len=config.data.max_node) + + def forward(self, X, time_cond, maskX): + + # Shared Embeddings + emb_x = self.dropout_op(self.opEmb(X)) + + if self.pos_encoder is not None: + emb_p = self.pos_encoder(emb_x) # [20, 64] + emb_x = emb_x + emb_p + # Time embedding + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model)# time embedding + emb_t = self.timeEmb1(emb_t) # [32, 512] + emb_t = self.timeEmb2(self.act(emb_t)) # [32, 64] + emb_t = emb_t.unsqueeze(1) + emb = emb_x + emb_t + + h_x = self.graph_encoder(emb, maskX) + h_x = self.final(h_x) + + """ + Shape: Batch Size, Length (with Pad), Feature Dim (forward) + Feature Dim (backward) + *HINT: X1 X2 X3 [PAD] [PAD] + """ + return h_x + + + +@utils.register_model(name='PredictorCATE') +class PredictorCATE(nn.Module): + def __init__(self, config): + super(PredictorCATE, self).__init__() + # Shared Embedding Layer + self.opEmb = SemanticEmbedding(config.model.graph_encoder) + self.dropout_op = nn.Dropout(p=config.model.dropout) + self.d_model = config.model.graph_encoder.d_model + self.act = act = get_act(config) + # Time + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + + # 2 GraphEncoder for X and Y + self.graph_encoder = GraphEncoder(config.model.graph_encoder) + + self.fdim = int(config.model.graph_encoder.n_layers * config.model.graph_encoder.d_model) + self.final = MLP(num_layers=3, input_dim=self.fdim, hidden_dim=2*self.fdim, output_dim=config.data.n_vocab, + use_bn=False, activate_func=F.elu) + + self.rdim = int(config.data.max_node * config.data.n_vocab) + self.regeress = MLP(num_layers=2, input_dim=self.rdim, hidden_dim=2*self.rdim, output_dim=1, + use_bn=False, activate_func=F.elu) + + + + def forward(self, X, time_cond, maskX): + + # Shared Embeddings + emb_x = self.dropout_op(self.opEmb(X)) + + # Time embedding + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model)# time embedding + emb_t = self.timeEmb1(emb_t) # [32, 512] + emb_t = self.timeEmb2(self.act(emb_t)) # [32, 64] + emb_t = emb_t.unsqueeze(1) + + emb = emb_x + emb_t + + # h_x = self.graph_encoder(emb_x, maskX) + h_x = self.graph_encoder(emb, maskX) + h_x = self.final(h_x) + + """ + Shape: Batch Size, Length (with Pad), Feature Dim (forward) + Feature Dim (backward) + *HINT: X1 X2 X3 [PAD] [PAD] + """ + h_x = h_x.reshape(h_x.size(0), -1) + h_x = self.regeress(h_x) + + return h_x + + +class PositionalEncoding_StageWise(nn.Module): + + def __init__(self, d_model, max_len): + + super(PositionalEncoding_StageWise, self).__init__() + + NUM_STAGE = 5 + max_len = int(max_len / NUM_STAGE) + self.encoding = torch.zeros(max_len, d_model) + + pos = torch.arange(0, max_len) + + + pos = pos.float().unsqueeze(dim=1) + + + _2i = torch.arange(0, d_model, step=2).float() + + # (max_len, 1) / (d_model/2 ) -> (max_len, d_model/2) + self.encoding[:, ::2] = torch.sin(pos / (10000 ** (_2i / d_model))) + self.encoding[:, 1::2] = torch.cos(pos / (10000 ** (_2i / d_model))) # (4, 64) + self.encoding = torch.cat([self.encoding] * NUM_STAGE, dim=0) + + def forward(self, x): + batch_size, seq_len, _ = x.size() + + return self.encoding[:seq_len, :].to(x.device) + + +@utils.register_model(name='MetaPredictorCATE') +class MetaPredictorCATE(nn.Module): + def __init__(self, config): + super(MetaPredictorCATE, self).__init__() + + self.input_type= config.model.input_type + self.hs = config.model.hs + + # Shared Embedding Layer + self.opEmb = SemanticEmbedding(config.model.graph_encoder) + self.dropout_op = nn.Dropout(p=config.model.dropout) + self.d_model = config.model.graph_encoder.d_model + self.act = act = get_act(config) + # Time + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + + # 2 GraphEncoder for X and Y + self.graph_encoder = GraphEncoder(config.model.graph_encoder) + + self.fdim = int(config.model.graph_encoder.n_layers * config.model.graph_encoder.d_model) + self.final = MLP(num_layers=3, input_dim=self.fdim, hidden_dim=2*self.fdim, output_dim=config.data.n_vocab, + use_bn=False, activate_func=F.elu) + + self.rdim = int(config.data.max_node * config.data.n_vocab) + self.regeress = MLP(num_layers=2, input_dim=self.rdim, hidden_dim=2*self.rdim, output_dim=2*self.rdim, + use_bn=False, activate_func=F.elu) + + # Set + self.nz = config.model.nz + self.num_sample = config.model.num_sample + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + input_dim = 0 + if 'D' in self.input_type: + input_dim += self.nz + if 'A' in self.input_type: + input_dim += 2*self.rdim + + self.pred_fc = nn.Sequential( + nn.Linear(input_dim, self.hs), + nn.Tanh(), + nn.Linear(self.hs, 1) + ) + + self.sample_state = False + self.D_mu = None + + + def arch_encode(self, X, time_cond, maskX): + # Shared Embeddings + emb_x = self.dropout_op(self.opEmb(X)) + + # Time embedding + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model)# time embedding + emb_t = self.timeEmb1(emb_t) # [32, 512] + emb_t = self.timeEmb2(self.act(emb_t)) # [32, 64] + emb_t = emb_t.unsqueeze(1) + emb = emb_x + emb_t + + h_x = self.graph_encoder(emb, maskX) + h_x = self.final(h_x) + + h_x = h_x.reshape(h_x.size(0), -1) + h_x = self.regeress(h_x) + return h_x + + def set_encode(self, task): + proto_batch = [] + for x in task: + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + return v + + def predict(self, D_mu, A_mu): + input_vec = [] + if 'D' in self.input_type: + input_vec.append(D_mu) + if 'A' in self.input_type: + input_vec.append(A_mu) + input_vec = torch.cat(input_vec, dim=1) + return self.pred_fc(input_vec) + + def forward(self, X, time_cond, maskX, task): + if self.sample_state: + if self.D_mu is None: + self.D_mu = self.set_encode(task) + D_mu = self.D_mu + else: + D_mu = self.set_encode(task) + A_mu = self.arch_encode(X, time_cond, maskX) + y_pred = self.predict(D_mu, A_mu) + return y_pred + + +class AttentionPool2d(nn.Module): + """ + Adapted from CLIP: https://github.com/openai/CLIP/blob/main/clip/model.py + """ + + def __init__( + self, + spacial_dim: int, + embed_dim: int, + num_heads_channels: int, + output_dim: int = None, + ): + super().__init__() + self.positional_embedding = nn.Parameter( + torch.randn(embed_dim, spacial_dim ** 2 + 1) / embed_dim ** 0.5 + ) + self.qkv_proj = conv_nd(1, embed_dim, 3 * embed_dim, 1) + self.c_proj = conv_nd(1, embed_dim, output_dim or embed_dim, 1) + self.num_heads = embed_dim // num_heads_channels + self.attention = QKVAttention(self.num_heads) + + def forward(self, x): + b, c, *_spatial = x.shape + x = x.reshape(b, c, -1) # NC(HW) + x = torch.cat([x.mean(dim=-1, keepdim=True), x], dim=-1) # NC(HW+1) + x = x + self.positional_embedding[None, :, :].to(x.dtype) # NC(HW+1) + x = self.qkv_proj(x) + x = self.attention(x) + x = self.c_proj(x) + return x[:, :, 0] + +import math +def get_timestep_embedding(timesteps, embedding_dim, max_positions=10000): + assert len(timesteps.shape) == 1 + half_dim = embedding_dim // 2 + # magic number 10000 is from transformers + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) * -emb) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = F.pad(emb, (0, 1), mode='constant') + assert emb.shape == (timesteps.shape[0], embedding_dim) + return emb + + +def get_act(config): + """Get actiuvation functions from the config file.""" + + if config.model.nonlinearity.lower() == 'elu': + return nn.ELU() + elif config.model.nonlinearity.lower() == 'relu': + return nn.ReLU() + elif config.model.nonlinearity.lower() == 'lrelu': + return nn.LeakyReLU(negative_slope=0.2) + elif config.model.nonlinearity.lower() == 'swish': + return nn.SiLU() + elif config.model.nonlinearity.lower() == 'tanh': + return nn.Tanh() + else: + raise NotImplementedError('activation function does not exist!') diff --git a/MobileNetV3/models/dagformer.py b/MobileNetV3/models/dagformer.py new file mode 100644 index 0000000..d81dff2 --- /dev/null +++ b/MobileNetV3/models/dagformer.py @@ -0,0 +1,355 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math +from einops.layers.torch import Rearrange +from einops import rearrange +import numpy as np + +from . import utils + + +class SinusoidalPositionalEmbedding(nn.Embedding): + + def __init__(self, num_positions, embedding_dim): + super().__init__(num_positions, embedding_dim) # torch.nn.Embedding(num_embeddings, embedding_dim) + self.weight = self._init_weight(self.weight) # self.weight => nn.Embedding(num_positions, embedding_dim).weight + + @staticmethod + def _init_weight(out: nn.Parameter): + n_pos, embed_dim = out.shape + pe = nn.Parameter(torch.zeros(out.shape)) + for pos in range(n_pos): + for i in range(0, embed_dim, 2): + pe[pos, i].data.copy_( torch.tensor( np.sin(pos / (10000 ** ( i / embed_dim)))) ) + pe[pos, i + 1].data.copy_( torch.tensor( np.cos(pos / (10000 ** ((i + 1) / embed_dim)))) ) + pe.detach_() + + return pe + + @torch.no_grad() + def forward(self, input_ids): + bsz, seq_len = input_ids.shape[:2] # for x, seq_len = max_node_num + positions = torch.arange(seq_len, dtype=torch.long, device=self.weight.device) + return super().forward(positions) + + +class MLP(nn.Module): + def __init__( + self, + dim_in, + dim_out, + *, + expansion_factor = 2., + depth = 2, + norm = False, + ): + super().__init__() + hidden_dim = int(expansion_factor * dim_out) + norm_fn = lambda: nn.LayerNorm(hidden_dim) if norm else nn.Identity() + + layers = [nn.Sequential( + nn.Linear(dim_in, hidden_dim), + nn.SiLU(), + norm_fn() + )] + + for _ in range(depth - 1): + layers.append(nn.Sequential( + nn.Linear(hidden_dim, hidden_dim), + nn.SiLU(), + norm_fn() + )) + + layers.append(nn.Linear(hidden_dim, dim_out)) + self.net = nn.Sequential(*layers) + + def forward(self, x): + return self.net(x.float()) + +class SinusoidalPosEmb(nn.Module): + def __init__(self, dim): + super().__init__() + self.dim = dim + + def forward(self, x): + dtype, device = x.dtype, x.device + assert is_float_dtype(dtype), 'input to sinusoidal pos emb must be a float type' + + half_dim = self.dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, device = device, dtype = dtype) * -emb) + emb = rearrange(x, 'i -> i 1') * rearrange(emb, 'j -> 1 j') + return torch.cat((emb.sin(), emb.cos()), dim = -1).type(dtype) + +def is_float_dtype(dtype): + return any([dtype == float_dtype for float_dtype in (torch.float64, torch.float32, torch.float16, torch.bfloat16)]) + + +class PositionWiseFeedForward(nn.Module): + + def __init__(self, emb_dim: int, d_ff: int, dropout: float = 0.1): + super(PositionWiseFeedForward, self).__init__() + + self.activation = nn.ReLU() + self.w_1 = nn.Linear(emb_dim, d_ff) + self.w_2 = nn.Linear(d_ff, emb_dim) + self.dropout = dropout + + def forward(self, x): + residual = x + x = self.activation(self.w_1(x)) + x = F.dropout(x, p=self.dropout, training=self.training) + + x = self.w_2(x) + x = F.dropout(x, p=self.dropout, training=self.training) + return x + residual # residual connection for preventing gradient vanishing + + +class MultiHeadAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + emb_dim, + num_heads, + dropout=0.0, + bias=False, + encoder_decoder_attention=False, # otherwise self_attention + causal = True + ): + super().__init__() + self.emb_dim = emb_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = emb_dim // num_heads + assert self.head_dim * num_heads == self.emb_dim, "emb_dim must be divisible by num_heads" + + self.encoder_decoder_attention = encoder_decoder_attention + self.causal = causal + self.q_proj = nn.Linear(emb_dim, emb_dim, bias=bias) + self.k_proj = nn.Linear(emb_dim, emb_dim, bias=bias) + self.v_proj = nn.Linear(emb_dim, emb_dim, bias=bias) + self.out_proj = nn.Linear(emb_dim, emb_dim, bias=bias) + + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + ( + self.num_heads, + self.head_dim, + ) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + # This is equivalent to + # return x.transpose(1,2) + + + def scaled_dot_product(self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.BoolTensor): + + attn_weights = torch.matmul(query, key.transpose(-1, -2)) / math.sqrt(self.emb_dim) # QK^T/sqrt(d) + + if attention_mask is not None: + attn_weights = attn_weights.masked_fill(attention_mask.unsqueeze(1), float("-inf")) + + attn_weights = F.softmax(attn_weights, dim=-1) # softmax(QK^T/sqrt(d)) + attn_probs = F.dropout(attn_weights, p=self.dropout, training=self.training) + attn_output = torch.matmul(attn_probs, value) # softmax(QK^T/sqrt(d))V + + return attn_output, attn_probs + + + def MultiHead_scaled_dot_product(self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.BoolTensor): + attention_mask = attention_mask.bool() + + attn_weights = torch.matmul(query, key.transpose(-1, -2)) / math.sqrt(self.head_dim) # QK^T/sqrt(d) # [6, 6] + + # Attention mask + if attention_mask is not None: + if self.causal: + # (seq_len x seq_len) + attn_weights = attn_weights.masked_fill(attention_mask.unsqueeze(0).unsqueeze(1), float("-inf")) + else: + # (batch_size x seq_len) + attn_weights = attn_weights.masked_fill(attention_mask.unsqueeze(1).unsqueeze(2), float("-inf")) + + attn_weights = F.softmax(attn_weights, dim=-1) # softmax(QK^T/sqrt(d)) + attn_probs = F.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.matmul(attn_probs, value) # softmax(QK^T/sqrt(d))V + attn_output = attn_output.permute(0, 2, 1, 3).contiguous() + concat_attn_output_shape = attn_output.size()[:-2] + (self.emb_dim,) + attn_output = attn_output.view(*concat_attn_output_shape) + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights + + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + attention_mask: torch.Tensor = None, + ): + + q = self.q_proj(query) + # Enc-Dec attention + if self.encoder_decoder_attention: + k = self.k_proj(key) + v = self.v_proj(key) + # Self attention + else: + k = self.k_proj(query) + v = self.v_proj(query) + + q = self.transpose_for_scores(q) + k = self.transpose_for_scores(k) + v = self.transpose_for_scores(v) + + attn_output, attn_weights = self.MultiHead_scaled_dot_product(q,k,v,attention_mask) + return attn_output, attn_weights + +class EncoderLayer(nn.Module): + def __init__(self, emb_dim, ffn_dim, attention_heads, + attention_dropout, dropout): + super().__init__() + self.emb_dim = emb_dim + self.ffn_dim = ffn_dim + self.self_attn = MultiHeadAttention( + emb_dim=self.emb_dim, + num_heads=attention_heads, + dropout=attention_dropout) + self.self_attn_layer_norm = nn.LayerNorm(self.emb_dim) + self.dropout = dropout + self.activation_fn = nn.ReLU() + self.PositionWiseFeedForward = PositionWiseFeedForward(self.emb_dim, self.ffn_dim, dropout) + self.final_layer_norm = nn.LayerNorm(self.emb_dim) + + def forward(self, x, encoder_padding_mask): + + residual = x + x, attn_weights = self.self_attn(query=x, key=x, attention_mask=encoder_padding_mask) + + x = F.dropout(x, p=self.dropout, training=self.training) + x = residual + x + x = self.self_attn_layer_norm(x) + x = self.PositionWiseFeedForward(x) + x = self.final_layer_norm(x) + if torch.isinf(x).any() or torch.isnan(x).any(): + clamp_value = torch.finfo(x.dtype).max - 1000 + x = torch.clamp(x, min=-clamp_value, max=clamp_value) + return x, attn_weights + + +@utils.register_model(name='DAGformer') +class DAGformer(torch.nn.Module): + def __init__(self, config): + # max_feat_num, + # max_node_num, + # emb_dim, + # ffn_dim, + # encoder_layers, + # attention_heads, + # attention_dropout, + # dropout, + # hs, + # time_dep=True, + # num_timesteps=None, + # return_attn=False, + # except_inout=False, + # connect_prev=True + # ): + super().__init__() + + self.dropout = config.model.dropout + self.time_dep = config.model.time_dep + self.return_attn = config.model.return_attn + max_feat_num = config.data.n_vocab + max_node_num = config.data.max_node + emb_dim = config.model.emb_dim + # num_timesteps = config.model.num_scales + num_timesteps = None + + self.x_embedding = MLP(max_feat_num, emb_dim) + # position embedding with topological order + self.position_embedding = SinusoidalPositionalEmbedding(max_node_num, emb_dim) + + if self.time_dep: + self.time_embedding = nn.Sequential( + nn.Embedding(num_timesteps, emb_dim) if num_timesteps is not None + else nn.Sequential(SinusoidalPosEmb(emb_dim), MLP(emb_dim, emb_dim)), # also offer a continuous version of timestep embeddings, with a 2 layer MLP + Rearrange('b (n d) -> b n d', n=1) + ) + + self.layers = nn.ModuleList([EncoderLayer(emb_dim, + config.model.ffn_dim, + config.model.attention_heads, + config.model.attention_dropout, + config.model.dropout) + for _ in range(config.model.encoder_layers)]) + + self.pred_fc = nn.Sequential( + nn.Linear(emb_dim, config.model.hs), + nn.Tanh(), + nn.Linear(config.model.hs, 1), + # nn.Sigmoid() + ) + + # -------- Load Constant Adj Matrix (START) --------- # + self.device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') + # from utils.graph_utils import get_const_adj + # mat = get_const_adj( + # except_inout=except_inout, + # shape_adj=(1, max_node_num, max_node_num), + # device=torch.device('cpu'), + # connect_prev=connect_prev)[0].cpu() + # is_triu_ = is_triu(mat) + # if is_triu_: + # self.adj_ = mat.T.to(self.device) + # else: + # self.adj_ = mat.to(self.device) + # -------- Load Constant Adj Matrix (END) --------- # + + def forward(self, x, t, adj, flags=None): + """ + :param x: B x N x F_i + :param adjs: B x C_i x N x N + :return: x_o: B x N x F_o, new_adjs: B x C_o x N x N + """ + + assert len(x.shape) == 3 + + self_attention_mask = torch.eye(adj.size(1)).to(self.device) + # attention_mask = 1. - (self_attention_mask + self.adj_) + attention_mask = 1. - (self_attention_mask + adj[0]) + + # -------- Generate input for DAGformer ------- # + x_embed = self.x_embedding(x) + # x_embed = x + x_pos = self.position_embedding(x).unsqueeze(0) + if self.time_dep: + time_embed = self.time_embedding(t) + + x = x_embed + x_pos + if self.time_dep: + x = x + time_embed + x = F.dropout(x, p=self.dropout, training=self.training) + + self_attn_scores = [] + for encoder_layer in self.layers: + x, attn = encoder_layer(x, attention_mask) + self_attn_scores.append(attn.detach()) + + x = self.pred_fc(x[:, -1, :]) # [256, 16] + + if self.return_attn: + return x, self_attn_scores + else: + return x \ No newline at end of file diff --git a/MobileNetV3/models/digcn.py b/MobileNetV3/models/digcn.py new file mode 100644 index 0000000..3f99a1d --- /dev/null +++ b/MobileNetV3/models/digcn.py @@ -0,0 +1,142 @@ +# Most of this code is from https://github.com/ultmaster/neuralpredictor.pytorch +# which was authored by Yuge Zhang, 2020 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from . import utils +from models.cate import PositionalEncoding_StageWise + + +def normalize_adj(adj): + # Row-normalize matrix + last_dim = adj.size(-1) + rowsum = adj.sum(2, keepdim=True).repeat(1, 1, last_dim) + return torch.div(adj, rowsum) + + +def graph_pooling(inputs, num_vertices): + num_vertices = num_vertices.to(inputs.device) + out = inputs.sum(1) + return torch.div(out, num_vertices.unsqueeze(-1).expand_as(out)) + + +class DirectedGraphConvolution(nn.Module): + def __init__(self, in_features, out_features): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.weight1 = nn.Parameter(torch.zeros((in_features, out_features))) + self.weight2 = nn.Parameter(torch.zeros((in_features, out_features))) + self.dropout = nn.Dropout(0.1) + self.reset_parameters() + + def reset_parameters(self): + nn.init.xavier_uniform_(self.weight1.data) + nn.init.xavier_uniform_(self.weight2.data) + + def forward(self, inputs, adj): + inputs = inputs.to(self.weight1.device) + adj = adj.to(self.weight1.device) + norm_adj = normalize_adj(adj) + output1 = F.relu(torch.matmul(norm_adj, torch.matmul(inputs, self.weight1))) + inv_norm_adj = normalize_adj(adj.transpose(1, 2)) + output2 = F.relu(torch.matmul(inv_norm_adj, torch.matmul(inputs, self.weight2))) + out = (output1 + output2) / 2 + out = self.dropout(out) + return out + + def __repr__(self): + return self.__class__.__name__ + ' (' \ + + str(self.in_features) + ' -> ' \ + + str(self.out_features) + ')' + +# if nasbench-101: initial_hidden=5. if nasbench-201: initial_hidden=7 +@utils.register_model(name='NeuralPredictor') +class NeuralPredictor(nn.Module): + # def __init__(self, initial_hidden=5, gcn_hidden=144, gcn_layers=4, linear_hidden=128): + def __init__(self, config): + super().__init__() + self.gcn = [DirectedGraphConvolution(config.model.graph_encoder.initial_hidden if i == 0 else config.model.graph_encoder.gcn_hidden, + config.model.graph_encoder.gcn_hidden) + for i in range(config.model.graph_encoder.gcn_layers)] + self.gcn = nn.ModuleList(self.gcn) + self.dropout = nn.Dropout(0.1) + self.fc1 = nn.Linear(config.model.graph_encoder.gcn_hidden, config.model.graph_encoder.linear_hidden, bias=False) + self.fc2 = nn.Linear(config.model.graph_encoder.linear_hidden, 1, bias=False) + # Time + self.d_model = config.model.graph_encoder.gcn_hidden + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + self.act = act = get_act(config) + + # self.pos_enc_type = config.model.pos_enc_type + # if self.pos_enc_type == 1: + # raise NotImplementedError + # elif self.pos_enc_type == 2: + # self.pos_encoder = PositionalEncoding_StageWise(d_model=config.model.graph_encoder.gcn_hidden, max_len=config.data.max_node) + # elif self.pos_enc_type == 3: + # raise NotImplementedError + # else: + # self.pos_encoder = None + + # def forward(self, inputs): + def forward(self, X, time_cond, maskX): + # numv, adj, out = inputs["num_vertices"], inputs["adjacency"], inputs["operations"] + out = X # (5, 20, 10) + adj = maskX # (1, 20, 20) + + # # pos embedding + # if self.pos_encoder is not None: + # emb_p = self.pos_encoder(out) # [20, 64] + # out = out + emb_p + numv = torch.tensor([adj.size(1)] * adj.size(0)).to(out.device) # 20 + gs = adj.size(1) # graph node number + + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model)# time embedding + emb_t = self.timeEmb1(emb_t) + emb_t = self.timeEmb2(self.act(emb_t)) # (5, 144) + + adj_with_diag = normalize_adj(adj + torch.eye(gs, device=adj.device)) # assuming diagonal is not 1 + for layer in self.gcn: + out = layer(out, adj_with_diag) + out = graph_pooling(out, numv) # out: 5, 20, 144 + # time + out = out + emb_t + out = self.fc1(out) # (5, 128) + out = self.dropout(out) + # out = self.fc2(out).view(-1) + out = self.fc2(out) + return out + +import math +def get_timestep_embedding(timesteps, embedding_dim, max_positions=10000): + assert len(timesteps.shape) == 1 + half_dim = embedding_dim // 2 + # magic number 10000 is from transformers + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) * -emb) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = F.pad(emb, (0, 1), mode='constant') + assert emb.shape == (timesteps.shape[0], embedding_dim) + return emb + +def get_act(config): + """Get actiuvation functions from the config file.""" + + if config.model.nonlinearity.lower() == 'elu': + return nn.ELU() + elif config.model.nonlinearity.lower() == 'relu': + return nn.ReLU() + elif config.model.nonlinearity.lower() == 'lrelu': + return nn.LeakyReLU(negative_slope=0.2) + elif config.model.nonlinearity.lower() == 'swish': + return nn.SiLU() + elif config.model.nonlinearity.lower() == 'tanh': + return nn.Tanh() + else: + raise NotImplementedError('activation function does not exist!') \ No newline at end of file diff --git a/MobileNetV3/models/digcn_meta.py b/MobileNetV3/models/digcn_meta.py new file mode 100644 index 0000000..bdf2a60 --- /dev/null +++ b/MobileNetV3/models/digcn_meta.py @@ -0,0 +1,194 @@ +# Most of this code is from https://github.com/ultmaster/neuralpredictor.pytorch +# which was authored by Yuge Zhang, 2020 + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from . import utils +from .set_encoder.setenc_models import SetPool + +def normalize_adj(adj): + # Row-normalize matrix + last_dim = adj.size(-1) + rowsum = adj.sum(2, keepdim=True).repeat(1, 1, last_dim) + return torch.div(adj, rowsum) + + +def graph_pooling(inputs, num_vertices): + num_vertices = num_vertices.to(inputs.device) + out = inputs.sum(1) + return torch.div(out, num_vertices.unsqueeze(-1).expand_as(out)) + + +class DirectedGraphConvolution(nn.Module): + def __init__(self, in_features, out_features): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.weight1 = nn.Parameter(torch.zeros((in_features, out_features))) + self.weight2 = nn.Parameter(torch.zeros((in_features, out_features))) + self.dropout = nn.Dropout(0.1) + self.reset_parameters() + + def reset_parameters(self): + nn.init.xavier_uniform_(self.weight1.data) + nn.init.xavier_uniform_(self.weight2.data) + + def forward(self, inputs, adj): + inputs = inputs.to(self.weight1.device) + adj = adj.to(self.weight1.device) + norm_adj = normalize_adj(adj) + output1 = F.relu(torch.matmul(norm_adj, torch.matmul(inputs, self.weight1))) + inv_norm_adj = normalize_adj(adj.transpose(1, 2)) + output2 = F.relu(torch.matmul(inv_norm_adj, torch.matmul(inputs, self.weight2))) + out = (output1 + output2) / 2 + out = self.dropout(out) + return out + + def __repr__(self): + return self.__class__.__name__ + ' (' \ + + str(self.in_features) + ' -> ' \ + + str(self.out_features) + ')' + +# if nasbench-101: initial_hidden=5. if nasbench-201: initial_hidden=7 +@utils.register_model(name='MetaNeuralPredictor') +class MetaeuralPredictor(nn.Module): + # def __init__(self, initial_hidden=5, gcn_hidden=144, gcn_layers=4, linear_hidden=128): + def __init__(self, config): + super().__init__() + # Arch + self.gcn = [DirectedGraphConvolution(config.model.graph_encoder.initial_hidden if i == 0 else config.model.graph_encoder.gcn_hidden, + config.model.graph_encoder.gcn_hidden) + for i in range(config.model.graph_encoder.gcn_layers)] + self.gcn = nn.ModuleList(self.gcn) + self.dropout = nn.Dropout(0.1) + self.fc1 = nn.Linear(config.model.graph_encoder.gcn_hidden, config.model.graph_encoder.linear_hidden, bias=False) + # self.fc2 = nn.Linear(config.model.graph_encoder.linear_hidden, 1, bias=False) + + # Time + self.d_model = config.model.graph_encoder.gcn_hidden + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + + self.act = act = get_act(config) + self.input_type = config.model.input_type + self.hs = config.model.hs + + # Set + self.nz = config.model.nz + self.num_sample = config.model.num_sample + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + input_dim = 0 + if 'D' in self.input_type: + input_dim += self.nz + if 'A' in self.input_type: + input_dim += config.model.graph_encoder.linear_hidden + + self.pred_fc = nn.Sequential( + nn.Linear(input_dim, self.hs), + nn.Tanh(), + nn.Linear(self.hs, 1) + ) + + self.sample_state = False + self.D_mu = None + + def arch_encode(self, X, time_cond, maskX): + # numv, adj, out = inputs["num_vertices"], inputs["adjacency"], inputs["operations"] + out = X + adj = maskX + numv = torch.tensor([adj.size(1)] * adj.size(0)).to(out.device) + gs = adj.size(1) # graph node number + + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model)# time embedding + emb_t = self.timeEmb1(emb_t) + emb_t = self.timeEmb2(self.act(emb_t)) + + adj_with_diag = normalize_adj(adj + torch.eye(gs, device=adj.device)) # assuming diagonal is not 1 + for layer in self.gcn: + out = layer(out, adj_with_diag) + out = graph_pooling(out, numv) + # time + out = out + emb_t + out = self.fc1(out) + out = self.dropout(out) + + # out = self.fc2(out).view(-1) + # out = self.fc2(out) + return out + + def set_encode(self, task): + proto_batch = [] + for x in task: + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + return v + + def predict(self, D_mu, A_mu): + input_vec = [] + if 'D' in self.input_type: + input_vec.append(D_mu) + if 'A' in self.input_type: + input_vec.append(A_mu) + input_vec = torch.cat(input_vec, dim=1) + return self.pred_fc(input_vec) + + def forward(self, X, time_cond, maskX, task): + if self.sample_state: + if self.D_mu is None: + self.D_mu = self.set_encode(task) + D_mu = self.D_mu + else: + D_mu = self.set_encode(task) + A_mu = self.arch_encode(X, time_cond, maskX) + y_pred = self.predict(D_mu, A_mu) + return y_pred + + +import math +def get_timestep_embedding(timesteps, embedding_dim, max_positions=10000): + assert len(timesteps.shape) == 1 + half_dim = embedding_dim // 2 + # magic number 10000 is from transformers + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) * -emb) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = F.pad(emb, (0, 1), mode='constant') + assert emb.shape == (timesteps.shape[0], embedding_dim) + return emb + +def get_act(config): + """Get actiuvation functions from the config file.""" + + if config.model.nonlinearity.lower() == 'elu': + return nn.ELU() + elif config.model.nonlinearity.lower() == 'relu': + return nn.ReLU() + elif config.model.nonlinearity.lower() == 'lrelu': + return nn.LeakyReLU(negative_slope=0.2) + elif config.model.nonlinearity.lower() == 'swish': + return nn.SiLU() + elif config.model.nonlinearity.lower() == 'tanh': + return nn.Tanh() + else: + raise NotImplementedError('activation function does not exist!') \ No newline at end of file diff --git a/MobileNetV3/models/ema.py b/MobileNetV3/models/ema.py new file mode 100644 index 0000000..5eca0b4 --- /dev/null +++ b/MobileNetV3/models/ema.py @@ -0,0 +1,85 @@ +import torch + + +class ExponentialMovingAverage: + """ + Maintains (exponential) moving average of a set of parameters. + """ + + def __init__(self, parameters, decay, use_num_updates=True): + """ + Args: + parameters: Iterable of `torch.nn.Parameter`; usually the result of `model.parameters()`. + decay: The exponential decay. + use_num_updates: Whether to use number of updates when computing averages. + """ + if decay < 0.0 or decay > 1.0: + raise ValueError('Decay must be between 0 and 1') + self.decay = decay + self.num_updates = 0 if use_num_updates else None + self.shadow_params = [p.clone().detach() + for p in parameters if p.requires_grad] + self.collected_params = [] + + def update(self, parameters): + """ + Update currently maintained parameters. + + Call this every time the parameters are updated, such as the result of the `optimizer.step()` call. + + Args: + parameters: Iterable of `torch.nn.Parameter`; usually the same set of parameters used to + initialize this object. + """ + decay = self.decay + if self.num_updates is not None: + self.num_updates += 1 + decay = min(decay, (1 + self.num_updates) / (10 + self.num_updates)) + one_minus_decay = 1.0 - decay + with torch.no_grad(): + parameters = [p for p in parameters if p.requires_grad] + for s_param, param in zip(self.shadow_params, parameters): + s_param.sub_(one_minus_decay * (s_param - param)) + + def copy_to(self, parameters): + """ + Copy current parameters into given collection of parameters. + + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be + updated with the stored moving averages. + """ + parameters = [p for p in parameters if p.requires_grad] + for s_param, param in zip(self.shadow_params, parameters): + if param.requires_grad: + param.data.copy_(s_param.data) + + def store(self, parameters): + """ + Save the current parameters for restoring later. + + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be temporarily stored. + """ + self.collected_params = [param.clone() for param in parameters] + + def restore(self, parameters): + """ + Restore the parameters stored with the `store` method. + Useful to validate the model with EMA parameters without affecting the original optimization process. + Store the parameters before the `copy_to` method. + After validation (or model saving), use this to restore the former parameters. + + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be updated with the stored parameters. + """ + for c_param, param in zip(self.collected_params, parameters): + param.data.copy_(c_param.data) + + def state_dict(self): + return dict(decay=self.decay, num_updates=self.num_updates, shadow_params=self.shadow_params) + + def load_state_dict(self, state_dict): + self.decay = state_dict['decay'] + self.num_updates = state_dict['num_updates'] + self.shadow_params = state_dict['shadow_params'] diff --git a/MobileNetV3/models/gnns.py b/MobileNetV3/models/gnns.py new file mode 100644 index 0000000..2ba0351 --- /dev/null +++ b/MobileNetV3/models/gnns.py @@ -0,0 +1,82 @@ +import torch.nn as nn +import torch +from .trans_layers import * + + +class pos_gnn(nn.Module): + def __init__(self, act, x_ch, pos_ch, out_ch, max_node, graph_layer, n_layers=3, edge_dim=None, heads=4, + temb_dim=None, dropout=0.1, attn_clamp=False): + super().__init__() + self.out_ch = out_ch + self.Dropout_0 = nn.Dropout(dropout) + self.act = act + self.max_node = max_node + self.n_layers = n_layers + + if temb_dim is not None: + self.Dense_node0 = nn.Linear(temb_dim, x_ch) + self.Dense_node1 = nn.Linear(temb_dim, pos_ch) + self.Dense_edge0 = nn.Linear(temb_dim, edge_dim) + self.Dense_edge1 = nn.Linear(temb_dim, edge_dim) + + self.convs = nn.ModuleList() + self.edge_convs = nn.ModuleList() + self.edge_layer = nn.Linear(edge_dim * 2 + self.out_ch, edge_dim) + + for i in range(n_layers): + if i == 0: + self.convs.append(eval(graph_layer)(x_ch, pos_ch, self.out_ch//heads, heads, edge_dim=edge_dim*2, + act=act, attn_clamp=attn_clamp)) + else: + self.convs.append(eval(graph_layer) + (self.out_ch, pos_ch, self.out_ch//heads, heads, edge_dim=edge_dim*2, act=act, + attn_clamp=attn_clamp)) + self.edge_convs.append(nn.Linear(self.out_ch, edge_dim*2)) + + def forward(self, x_degree, x_pos, edge_index, dense_ori, dense_spd, dense_index, temb=None): + """ + Args: + x_degree: node degree feature [B*N, x_ch] + x_pos: node rwpe feature [B*N, pos_ch] + edge_index: [2, edge_length] + dense_ori: edge feature [B, N, N, nf//2] + dense_spd: edge shortest path distance feature [B, N, N, nf//2] # Do we need this part? # TODO + dense_index + temb: [B, temb_dim] + """ + + B, N, _, _ = dense_ori.shape + + if temb is not None: + dense_ori = dense_ori + self.Dense_edge0(self.act(temb))[:, None, None, :] + dense_spd = dense_spd + self.Dense_edge1(self.act(temb))[:, None, None, :] + + temb = temb.unsqueeze(1).repeat(1, self.max_node, 1) + temb = temb.reshape(-1, temb.shape[-1]) + x_degree = x_degree + self.Dense_node0(self.act(temb)) + x_pos = x_pos + self.Dense_node1(self.act(temb)) + + dense_edge = torch.cat([dense_ori, dense_spd], dim=-1) + + ori_edge_attr = dense_edge + h = x_degree + h_pos = x_pos + + for i_layer in range(self.n_layers): + h_edge = dense_edge[dense_index] + # update node feature + h, h_pos = self.convs[i_layer](h, h_pos, edge_index, h_edge) + h = self.Dropout_0(h) + h_pos = self.Dropout_0(h_pos) + + # update dense edge feature + h_dense_node = h.reshape(B, N, -1) + cur_edge_attr = h_dense_node.unsqueeze(1) + h_dense_node.unsqueeze(2) # [B, N, N, nf] + dense_edge = (dense_edge + self.act(self.edge_convs[i_layer](cur_edge_attr))) / math.sqrt(2.) + dense_edge = self.Dropout_0(dense_edge) + + # Concat edge attribute + h_dense_edge = torch.cat([ori_edge_attr, dense_edge], dim=-1) + h_dense_edge = self.edge_layer(h_dense_edge).permute(0, 3, 1, 2) + + return h_dense_edge diff --git a/MobileNetV3/models/layers.py b/MobileNetV3/models/layers.py new file mode 100644 index 0000000..a74efee --- /dev/null +++ b/MobileNetV3/models/layers.py @@ -0,0 +1,44 @@ +"""Common layers""" + +import torch.nn as nn +import torch +import torch.nn.functional as F +import math + + +def get_act(config): + """Get actiuvation functions from the config file.""" + + if config.model.nonlinearity.lower() == 'elu': + return nn.ELU() + elif config.model.nonlinearity.lower() == 'relu': + return nn.ReLU() + elif config.model.nonlinearity.lower() == 'lrelu': + return nn.LeakyReLU(negative_slope=0.2) + elif config.model.nonlinearity.lower() == 'swish': + return nn.SiLU() + elif config.model.nonlinearity.lower() == 'tanh': + return nn.Tanh() + else: + raise NotImplementedError('activation function does not exist!') + + +def conv1x1(in_planes, out_planes, stride=1, bias=True, dilation=1, padding=0): + conv = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=bias, dilation=dilation, + padding=padding) + return conv + + +# from DDPM +def get_timestep_embedding(timesteps, embedding_dim, max_positions=10000): + assert len(timesteps.shape) == 1 + half_dim = embedding_dim // 2 + # magic number 10000 is from transformers + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) * -emb) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = F.pad(emb, (0, 1), mode='constant') + assert emb.shape == (timesteps.shape[0], embedding_dim) + return emb diff --git a/MobileNetV3/models/pgsn.py b/MobileNetV3/models/pgsn.py new file mode 100644 index 0000000..ccf6002 --- /dev/null +++ b/MobileNetV3/models/pgsn.py @@ -0,0 +1,171 @@ +import torch.nn as nn +import torch +import functools +from torch_geometric.utils import dense_to_sparse + +from . import utils, layers, gnns + +get_act = layers.get_act +conv1x1 = layers.conv1x1 + + +@utils.register_model(name='PGSN') +class PGSN(nn.Module): + """Position enhanced graph score network.""" + + def __init__(self, config): + super().__init__() + + self.config = config + self.act = act = get_act(config) + + # get model construction paras + self.nf = nf = config.model.nf + self.num_gnn_layers = num_gnn_layers = config.model.num_gnn_layers + dropout = config.model.dropout + self.embedding_type = embedding_type = config.model.embedding_type.lower() + self.rw_depth = rw_depth = config.model.rw_depth + self.edge_th = config.model.edge_th + + modules = [] + # timestep/noise_level embedding; only for continuous training + if embedding_type == 'positional': + embed_dim = nf + else: + raise ValueError(f'embedding type {embedding_type} unknown.') + + # timestep embedding layers + modules.append(nn.Linear(embed_dim, nf * 4)) + modules.append(nn.Linear(nf * 4, nf * 4)) + + # graph size condition embedding + self.size_cond = size_cond = config.model.size_cond + if size_cond: + self.size_onehot = functools.partial(nn.functional.one_hot, num_classes=config.data.max_node + 1) + modules.append(nn.Linear(config.data.max_node + 1, nf * 4)) + modules.append(nn.Linear(nf * 4, nf * 4)) + + channels = config.data.num_channels + assert channels == 1, "Without edge features." + + # degree onehot + self.degree_max = self.config.data.max_node // 2 + self.degree_onehot = functools.partial( + nn.functional.one_hot, + num_classes=self.degree_max + 1) + + # project edge features + modules.append(conv1x1(channels, nf // 2)) + modules.append(conv1x1(rw_depth + 1, nf // 2)) + + # project node features + self.x_ch = nf + self.pos_ch = nf // 2 + modules.append(nn.Linear(self.degree_max + 1, self.x_ch)) + modules.append(nn.Linear(rw_depth, self.pos_ch)) + + # GNN + modules.append(gnns.pos_gnn(act, self.x_ch, self.pos_ch, nf, config.data.max_node, + config.model.graph_layer, num_gnn_layers, + heads=config.model.heads, edge_dim=nf//2, temb_dim=nf * 4, + dropout=dropout, attn_clamp=config.model.attn_clamp)) + + # output + modules.append(conv1x1(nf // 2, nf // 2)) + modules.append(conv1x1(nf // 2, channels)) + + self.all_modules = nn.ModuleList(modules) + + def forward(self, x, time_cond, *args, **kwargs): + mask = kwargs['mask'] + modules = self.all_modules + m_idx = 0 + + # Sinusoidal positional embeddings + timesteps = time_cond + temb = layers.get_timestep_embedding(timesteps, self.nf) + + # time embedding + temb = modules[m_idx](temb) # [32, 512] + m_idx += 1 + temb = modules[m_idx](self.act(temb)) # [32, 512] + m_idx += 1 + + if self.size_cond: + with torch.no_grad(): + node_mask = utils.mask_adj2node(mask.squeeze(1)) # [B, N] + num_node = torch.sum(node_mask, dim=-1) # [B] + num_node = self.size_onehot(num_node.to(torch.long)).to(torch.float) + num_node_emb = modules[m_idx](num_node) + m_idx += 1 + num_node_emb = modules[m_idx](self.act(num_node_emb)) + m_idx += 1 + temb = temb + num_node_emb + + if not self.config.data.centered: + # rescale the input data to [-1, 1] + x = x * 2. - 1. + + with torch.no_grad(): + # continuous-valued graph adjacency matrices + cont_adj = ((x + 1.) / 2.).clone() + cont_adj = (cont_adj * mask).squeeze(1) # [B, N, N] + cont_adj = cont_adj.clamp(min=0., max=1.) + if self.edge_th > 0.: + cont_adj[cont_adj < self.edge_th] = 0. + + # discretized graph adjacency matrices + adj = x.squeeze(1).clone() # [B, N, N] + adj[adj >= 0.] = 1. + adj[adj < 0.] = 0. + adj = adj * mask.squeeze(1) + + # extract RWSE and Shortest-Path Distance + x_pos, spd_onehot = utils.get_rw_feat(self.rw_depth, adj) + # x_pos: [32, 20, 16], spd_onehot: [32, 17, 20, 20] + + # edge [B, N, N, F] + dense_edge_ori = modules[m_idx](x).permute(0, 2, 3, 1) # [32, 20, 20, 64] + m_idx += 1 + dense_edge_spd = modules[m_idx](spd_onehot).permute(0, 2, 3, 1) # [32, 20, 20, 64] + m_idx += 1 + + # Use Degree as node feature + x_degree = torch.sum(cont_adj, dim=-1) # [B, N] # [32, 20] + x_degree = x_degree.clamp(max=float(self.degree_max)) # [B, N] # [32, 20] + x_degree = self.degree_onehot(x_degree.to(torch.long)).to(torch.float) # [B, N, max_node] # [32, 20, 11] + x_degree = modules[m_idx](x_degree) # projection layer [B, N, nf] # [32, 20, 128] + m_idx += 1 + import pdb; pdb.set_trace() + + # pos encoding + # x_pos: [32, 20, 16] + x_pos = modules[m_idx](x_pos) # [32, 20, 64] + m_idx += 1 + + # Dense to sparse node [BxN, -1] + x_degree = x_degree.reshape(-1, self.x_ch) # [640, 128] + x_pos = x_pos.reshape(-1, self.pos_ch) # [640, 64] + dense_index = cont_adj.nonzero(as_tuple=True) + edge_index, _ = dense_to_sparse(cont_adj) # [2, 5386] + + # Run GNN layers + h_dense_edge = modules[m_idx](x_degree, x_pos, edge_index, dense_edge_ori, dense_edge_spd, dense_index, temb) + m_idx += 1 + import pdb; pdb.set_trace() + + # Output + h = self.act(modules[m_idx](self.act(h_dense_edge))) + m_idx += 1 + import pdb; pdb.set_trace() + h = modules[m_idx](h) + m_idx += 1 + import pdb; pdb.set_trace() + + # make edge estimation symmetric + h = (h + h.transpose(2, 3)) / 2. * mask + import pdb; pdb.set_trace() + + assert m_idx == len(modules) + + return h diff --git a/MobileNetV3/models/regressor.py b/MobileNetV3/models/regressor.py new file mode 100644 index 0000000..35c6fc9 --- /dev/null +++ b/MobileNetV3/models/regressor.py @@ -0,0 +1,27 @@ +import torch +import torch.nn as nn +import torch.optim as optim + +from . import utils + +@utils.register_model(name='MLPRegressor') +class MLPRegressor(nn.Module): + # def __init__(self, input_size, hidden_size, output_size): + def __init__(self, config): + super().__init__() + input_size = int(config.data.max_node * config.data.n_vocab) + hidden_size = config.model.hidden_size + output_size = config.model.output_size + self.fc1 = nn.Linear(input_size, hidden_size) + self.fc2 = nn.Linear(hidden_size, hidden_size) + self.fc3 = nn.Linear(hidden_size, hidden_size) + self.fc4 = nn.Linear(hidden_size, output_size) + self.activation = nn.ReLU() + + def forward(self, X, time_cond, maskX): + x = X.view(X.size(0), -1) + x = self.activation(self.fc1(x)) + x= self.activation(self.fc2(x)) + x= self.activation(self.fc3(x)) + x= self.fc4(x) + return x \ No newline at end of file diff --git a/MobileNetV3/models/set_encoder/setenc_models.py b/MobileNetV3/models/set_encoder/setenc_models.py new file mode 100644 index 0000000..61fab26 --- /dev/null +++ b/MobileNetV3/models/set_encoder/setenc_models.py @@ -0,0 +1,38 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from .setenc_modules import * + + +class SetPool(nn.Module): + def __init__(self, dim_input, num_outputs, dim_output, + num_inds=32, dim_hidden=128, num_heads=4, ln=False, mode=None): + super(SetPool, self).__init__() + if 'sab' in mode: # [32, 400, 128] + self.enc = nn.Sequential( + SAB(dim_input, dim_hidden, num_heads, ln=ln), # SAB? + SAB(dim_hidden, dim_hidden, num_heads, ln=ln)) + else: # [32, 400, 128] + self.enc = nn.Sequential( + ISAB(dim_input, dim_hidden, num_heads, num_inds, ln=ln), # SAB? + ISAB(dim_hidden, dim_hidden, num_heads, num_inds, ln=ln)) + if 'PF' in mode: # [32, 1, 501] + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), + nn.Linear(dim_hidden, dim_output)) + elif 'P' in mode: + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln)) + else: # torch.Size([32, 1, 501]) + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), # 32 1 128 + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + nn.Linear(dim_hidden, dim_output)) + # "", sm, sab, sabsm + + def forward(self, X): + x1 = self.enc(X) + x2 = self.dec(x1) + return x2 diff --git a/MobileNetV3/models/set_encoder/setenc_modules.py b/MobileNetV3/models/set_encoder/setenc_modules.py new file mode 100644 index 0000000..1e09c70 --- /dev/null +++ b/MobileNetV3/models/set_encoder/setenc_modules.py @@ -0,0 +1,67 @@ +##################################################################################### +# Copyright (c) Juho Lee SetTransformer, ICML 2019 [GitHub set_transformer] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + +class MAB(nn.Module): + def __init__(self, dim_Q, dim_K, dim_V, num_heads, ln=False): + super(MAB, self).__init__() + self.dim_V = dim_V + self.num_heads = num_heads + self.fc_q = nn.Linear(dim_Q, dim_V) + self.fc_k = nn.Linear(dim_K, dim_V) + self.fc_v = nn.Linear(dim_K, dim_V) + if ln: + self.ln0 = nn.LayerNorm(dim_V) + self.ln1 = nn.LayerNorm(dim_V) + self.fc_o = nn.Linear(dim_V, dim_V) + + def forward(self, Q, K): + Q = self.fc_q(Q) + K, V = self.fc_k(K), self.fc_v(K) + + dim_split = self.dim_V // self.num_heads + Q_ = torch.cat(Q.split(dim_split, 2), 0) + K_ = torch.cat(K.split(dim_split, 2), 0) + V_ = torch.cat(V.split(dim_split, 2), 0) + + A = torch.softmax(Q_.bmm(K_.transpose(1,2))/math.sqrt(self.dim_V), 2) + O = torch.cat((Q_ + A.bmm(V_)).split(Q.size(0), 0), 2) + O = O if getattr(self, 'ln0', None) is None else self.ln0(O) + O = O + F.relu(self.fc_o(O)) + O = O if getattr(self, 'ln1', None) is None else self.ln1(O) + return O + +class SAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, ln=False): + super(SAB, self).__init__() + self.mab = MAB(dim_in, dim_in, dim_out, num_heads, ln=ln) + + def forward(self, X): + return self.mab(X, X) + +class ISAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, num_inds, ln=False): + super(ISAB, self).__init__() + self.I = nn.Parameter(torch.Tensor(1, num_inds, dim_out)) + nn.init.xavier_uniform_(self.I) + self.mab0 = MAB(dim_out, dim_in, dim_out, num_heads, ln=ln) + self.mab1 = MAB(dim_in, dim_out, dim_out, num_heads, ln=ln) + + def forward(self, X): + H = self.mab0(self.I.repeat(X.size(0), 1, 1), X) + return self.mab1(X, H) + +class PMA(nn.Module): + def __init__(self, dim, num_heads, num_seeds, ln=False): + super(PMA, self).__init__() + self.S = nn.Parameter(torch.Tensor(1, num_seeds, dim)) + nn.init.xavier_uniform_(self.S) + self.mab = MAB(dim, dim, dim, num_heads, ln=ln) + + def forward(self, X): + return self.mab(self.S.repeat(X.size(0), 1, 1), X) \ No newline at end of file diff --git a/MobileNetV3/models/trans_layers.py b/MobileNetV3/models/trans_layers.py new file mode 100644 index 0000000..576f74d --- /dev/null +++ b/MobileNetV3/models/trans_layers.py @@ -0,0 +1,144 @@ +import math +from typing import Union, Tuple, Optional +from torch_geometric.typing import PairTensor, Adj, OptTensor + +import torch +import torch.nn as nn +from torch import Tensor +import torch.nn.functional as F +from torch.nn import Linear +from torch_scatter import scatter +from torch_geometric.nn.conv import MessagePassing +from torch_geometric.utils import softmax +import numpy as np + + +class PosTransLayer(MessagePassing): + """Involving the edge feature and updating position feature. Multiply Msg.""" + + _alpha: OptTensor + + def __init__(self, x_channels: int, pos_channels: int, out_channels: int, + heads: int = 1, dropout: float = 0., edge_dim: Optional[int] = None, + bias: bool = True, act=None, attn_clamp: bool = False, **kwargs): + kwargs.setdefault('aggr', 'add') + super(PosTransLayer, self).__init__(node_dim=0, **kwargs) + + self.x_channels = x_channels + self.pos_channels = pos_channels + self.in_channels = in_channels = x_channels + pos_channels + self.out_channels = out_channels + self.heads = heads + self.dropout = dropout + self.edge_dim = edge_dim + self.attn_clamp = attn_clamp + + if act is None: + self.act = nn.LeakyReLU(negative_slope=0.2) + else: + self.act = act + + self.lin_key = Linear(in_channels, heads * out_channels) + self.lin_query = Linear(in_channels, heads * out_channels) + self.lin_value = Linear(in_channels, heads * out_channels) + + self.lin_edge0 = Linear(edge_dim, heads * out_channels, bias=False) + self.lin_edge1 = Linear(edge_dim, heads * out_channels, bias=False) + + self.lin_pos = Linear(heads * out_channels, pos_channels, bias=False) + + self.lin_skip = Linear(x_channels, heads * out_channels, bias=bias) + self.norm1 = nn.GroupNorm(num_groups=min(heads * out_channels // 4, 32), + num_channels=heads * out_channels, eps=1e-6) + self.norm2 = nn.GroupNorm(num_groups=min(heads * out_channels // 4, 32), + num_channels=heads * out_channels, eps=1e-6) + # FFN + self.FFN = nn.Sequential(Linear(heads * out_channels, heads * out_channels), + self.act, + Linear(heads * out_channels, heads * out_channels)) + + self.reset_parameters() + + def reset_parameters(self): + self.lin_key.reset_parameters() + self.lin_query.reset_parameters() + self.lin_value.reset_parameters() + self.lin_skip.reset_parameters() + self.lin_edge0.reset_parameters() + self.lin_edge1.reset_parameters() + self.lin_pos.reset_parameters() + + def forward(self, x: OptTensor, + pos: Tensor, + edge_index: Adj, + edge_attr: OptTensor = None + ) -> Tuple[Tensor, Tensor]: + """""" + + H, C = self.heads, self.out_channels + + x_feat = torch.cat([x, pos], -1) + query = self.lin_query(x_feat).view(-1, H, C) + key = self.lin_key(x_feat).view(-1, H, C) + value = self.lin_value(x_feat).view(-1, H, C) + + # propagate_type: (x: PairTensor, edge_attr: OptTensor) + out_x, out_pos = self.propagate(edge_index, query=query, key=key, value=value, pos=pos, edge_attr=edge_attr, + size=None) + + out_x = out_x.view(-1, self.heads * self.out_channels) + + # skip connection for x + x_r = self.lin_skip(x) + out_x = (out_x + x_r) / math.sqrt(2) + out_x = self.norm1(out_x) + + # FFN + out_x = (out_x + self.FFN(out_x)) / math.sqrt(2) + out_x = self.norm2(out_x) + + # skip connection for pos + out_pos = pos + torch.tanh(pos + out_pos) + + return out_x, out_pos + + def message(self, query_i: Tensor, key_j: Tensor, value_j: Tensor, + pos_j: Tensor, + edge_attr: OptTensor, + index: Tensor, ptr: OptTensor, + size_i: Optional[int]) -> Tuple[Tensor, Tensor]: + + edge_attn = self.lin_edge0(edge_attr).view(-1, self.heads, self.out_channels) + alpha = (query_i * key_j * edge_attn).sum(dim=-1) / math.sqrt(self.out_channels) + if self.attn_clamp: + alpha = alpha.clamp(min=-5., max=5.) + + alpha = softmax(alpha, index, ptr, size_i) + alpha = F.dropout(alpha, p=self.dropout, training=self.training) + + # node feature message + msg = value_j + msg = msg * self.lin_edge1(edge_attr).view(-1, self.heads, self.out_channels) + msg = msg * alpha.view(-1, self.heads, 1) + + # node position message + pos_msg = pos_j * self.lin_pos(msg.reshape(-1, self.heads * self.out_channels)) + + return msg, pos_msg + + def aggregate(self, inputs: Tuple[Tensor, Tensor], index: Tensor, + ptr: Optional[Tensor] = None, + dim_size: Optional[int] = None) -> Tuple[Tensor, Tensor]: + if ptr is not None: + raise NotImplementedError("Not implement Ptr in aggregate") + else: + return (scatter(inputs[0], index, 0, dim_size=dim_size, reduce=self.aggr), + scatter(inputs[1], index, 0, dim_size=dim_size, reduce="mean")) + + def update(self, inputs: Tuple[Tensor, Tensor]) -> Tuple[Tensor, Tensor]: + return inputs + + def __repr__(self): + return '{}({}, {}, heads={})'.format(self.__class__.__name__, + self.in_channels, + self.out_channels, self.heads) diff --git a/MobileNetV3/models/transformer.py b/MobileNetV3/models/transformer.py new file mode 100755 index 0000000..b7b0929 --- /dev/null +++ b/MobileNetV3/models/transformer.py @@ -0,0 +1,248 @@ +from copy import deepcopy as cp + +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + +def clones(module, N): + return nn.ModuleList([cp(module) for _ in range(N)]) + +def attention(query, key, value, mask = None, dropout = None): + d_k = query.size(-1) + scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k) + if mask is not None: + scores = scores.masked_fill(mask == 0, -1e9) + attn = F.softmax(scores, dim = -1) + if dropout is not None: + attn = dropout(attn) + return torch.matmul(attn, value), attn + +class MultiHeadAttention(nn.Module): + def __init__(self, config): + super(MultiHeadAttention, self).__init__() + + self.d_model = config.d_model + self.n_head = config.n_head + self.d_k = config.d_model // config.n_head + + self.linears = clones(nn.Linear(self.d_model, self.d_model), 4) + self.dropout = nn.Dropout(p=config.dropout) + + def forward(self, query, key, value, mask = None): + if mask is not None: + mask = mask.unsqueeze(1) + batch_size = query.size(0) + + query, key , value = [l(x).view(batch_size, -1, self.n_head, self.d_k).transpose(1,2) for l, x in zip(self.linears, (query, key, value))] + x, attn = attention(query, key, value, mask = mask, dropout = self.dropout) + x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.n_head * self.d_k) + return self.linears[3](x), attn + +class PositionwiseFeedForward(nn.Module): + def __init__(self, config): + super(PositionwiseFeedForward, self).__init__() + + self.w_1 = nn.Linear(config.d_model, config.d_ff) + self.w_2 = nn.Linear(config.d_ff, config.d_model) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x): + return self.w_2(self.dropout(F.relu(self.w_1(x)))) + +class PositionwiseFeedForwardLast(nn.Module): + def __init__(self, config): + super(PositionwiseFeedForwardLast, self).__init__() + + self.w_1 = nn.Linear(config.d_model, config.d_ff) + self.w_2 = nn.Linear(config.d_ff, config.n_vocab) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x): + return self.w_2(self.dropout(F.relu(self.w_1(x)))) + +class SelfAttentionBlock(nn.Module): + def __init__(self, config): + super(SelfAttentionBlock, self).__init__() + + self.norm = nn.LayerNorm(config.d_model) + self.attn = MultiHeadAttention(config) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x, mask): + x_ = self.norm(x) + x_ , attn = self.attn(x_, x_, x_, mask) + return self.dropout(x_) + x, attn + +class SourceAttentionBlock(nn.Module): + def __init__(self, config): + super(SourceAttentionBlock, self).__init__() + + self.norm = nn.LayerNorm(config.d_model) + self.attn = MultiHeadAttention(config) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x, m, mask): + x_ = self.norm(x) + x_, attn = self.attn(x_, m, m, mask) + return self.dropout(x_) + x, attn + +class FeedForwardBlock(nn.Module): + def __init__(self, config): + super(FeedForwardBlock, self).__init__() + + self.norm = nn.LayerNorm(config.d_model) + self.feed_forward = PositionwiseFeedForward(config) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x): + x_ = self.norm(x) + x_ = self.feed_forward(x_) + return self.dropout(x_) + x + +class FeedForwardBlockLast(nn.Module): + def __init__(self, config): + super(FeedForwardBlockLast, self).__init__() + + self.norm = nn.LayerNorm(config.d_model) + self.feed_forward = PositionwiseFeedForwardLast(config) + self.dropout = nn.Dropout(p = config.dropout) + # Only for the last layer + self.proj_fc = nn.Linear(config.d_model, config.n_vocab) + + def forward(self, x): + x_ = self.norm(x) + x_ = self.feed_forward(x_) + # return self.dropout(x_) + x + return self.dropout(x_) + self.proj_fc(x) + +class EncoderBlock(nn.Module): + def __init__(self, config): + super(EncoderBlock, self).__init__() + self.self_attn = SelfAttentionBlock(config) + self.feed_forward = FeedForwardBlock(config) + + def forward(self, x, mask): + x, attn = self.self_attn(x, mask) + x = self.feed_forward(x) + return x, attn + +class EncoderBlockLast(nn.Module): + def __init__(self, config): + super(EncoderBlockLast, self).__init__() + self.self_attn = SelfAttentionBlock(config) + self.feed_forward = FeedForwardBlockLast(config) + + def forward(self, x, mask): + x, attn = self.self_attn(x, mask) + x = self.feed_forward(x) + return x, attn + +class DecoderBlock(nn.Module): + def __init__(self, config): + super(DecoderBlock, self).__init__() + + self.self_attn = SelfAttentionBlock(config) + self.src_attn = SourceAttentionBlock(config) + self.feed_forward = FeedForwardBlock(config) + + def forward(self, x, m, src_mask, tgt_mask): + x, attn_tgt = self.self_attn(x, tgt_mask) + x, attn_src = self.src_attn(x, m, src_mask) + x = self.feed_forward(x) + return x, attn_src, attn_tgt + +class Encoder(nn.Module): + def __init__(self, config): + super(Encoder, self).__init__() + + # self.layers = clones(EncoderBlock(config), config.n_layers - 1) + # self.layers.append(EncoderBlockLast(config)) + # self.norms = clones(nn.LayerNorm(config.d_model), config.n_layers - 1) + # self.norms.append(nn.LayerNorm(config.n_vocab)) + + self.layers = clones(EncoderBlock(config), config.n_layers) + self.norms = clones(nn.LayerNorm(config.d_model), config.n_layers) + + def forward(self, x, mask): + outputs = [] + attns = [] + for layer, norm in zip(self.layers, self.norms): + x, attn = layer(x, mask) + outputs.append(norm(x)) + attns.append(attn) + return outputs[-1], outputs, attns + +class PositionalEmbedding(nn.Module): + def __init__(self, config): + super(PositionalEmbedding, self).__init__() + + p2e = torch.zeros(config.max_len, config.d_model) + position = torch.arange(0.0, config.max_len).unsqueeze(1) + div_term = torch.exp(torch.arange(0.0, config.d_model, 2) * (- math.log(10000.0) / config.d_model)) + p2e[:, 0::2] = torch.sin(position * div_term) + p2e[:, 1::2] = torch.cos(position * div_term) + + self.register_buffer('p2e', p2e) + + def forward(self, x): + shp = x.size() + with torch.no_grad(): + emb = torch.index_select(self.p2e, 0, x.view(-1)).view(shp + (-1,)) + return emb + +class Transformer(nn.Module): + def __init__(self, config): + super(Transformer, self).__init__() + self.p2e = PositionalEmbedding(config) + self.encoder = Encoder(config) + + def forward(self, input_emb, position_ids, attention_mask): + # position embedding projection + projection = self.p2e(position_ids) + input_emb + return self.encoder(projection, attention_mask) + + +class TokenTypeEmbedding(nn.Module): + def __init__(self, config): + super(TokenTypeEmbedding, self).__init__() + self.t2e = nn.Embedding(config.n_token_type, config.d_model) + self.d_model = config.d_model + + def forward(self, x): + return self.t2e(x) * math.sqrt(self.d_model) + +class SemanticEmbedding(nn.Module): + def __init__(self, config): + super(SemanticEmbedding, self).__init__() + # self.w2e = nn.Embedding(config.n_vocab, config.d_model) + self.d_model = config.d_model + self.fc = nn.Linear(config.n_vocab, config.d_model) + + def forward(self, x): + # return self.w2e(x) * math.sqrt(self.d_model) + return self.fc(x) * math.sqrt(self.d_model) + +class Embeddings(nn.Module): + def __init__(self, config): + super(Embeddings, self).__init__() + + self.w2e = SemanticEmbedding(config) + self.p2e = PositionalEmbedding(config) + self.t2e = TokenTypeEmbedding(config) + + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, input_ids, position_ids = None, token_type_ids = None): + if position_ids is None: + batch_size, length = input_ids.size() + with torch.no_grad(): + position_ids = torch.arange(0, length).repeat(batch_size, 1) + if torch.cuda.is_available(): + position_ids = position_ids.cuda(device=input_ids.device) + + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + embeddings = self.w2e(input_ids) + self.p2e(position_ids) + self.t2e(token_type_ids) + return self.dropout(embeddings) \ No newline at end of file diff --git a/MobileNetV3/models/utils.py b/MobileNetV3/models/utils.py new file mode 100644 index 0000000..c1efd29 --- /dev/null +++ b/MobileNetV3/models/utils.py @@ -0,0 +1,301 @@ +import torch +import torch.nn.functional as F +import sde_lib +import numpy as np + +_MODELS = {} + + +def register_model(cls=None, *, name=None): + """A decorator for registering model classes.""" + + def _register(cls): + if name is None: + local_name = cls.__name__ + else: + local_name = name + if local_name in _MODELS: + raise ValueError( + f'Already registered model with name: {local_name}') + _MODELS[local_name] = cls + return cls + + if cls is None: + return _register + else: + return _register(cls) + + +def get_model(name): + return _MODELS[name] + + +def create_model(config): + """Create the score model.""" + model_name = config.model.name + score_model = get_model(model_name)(config) + score_model = score_model.to(config.device) + if 'load_pretrained' in config['training'].keys() and config.training.load_pretrained: + from utils import restore_checkpoint_partial + score_model = restore_checkpoint_partial(score_model, torch.load(config.training.pretrained_model_path, map_location=config.device)['model']) + # score_model = torch.nn.DataParallel(score_model) + return score_model + + +def get_model_fn(model, train=False): + """Create a function to give the output of the score-based model. + + Args: + model: The score model. + train: `True` for training and `False` for evaluation. + + Returns: + A model function. + """ + + def model_fn(x, labels, *args, **kwargs): + """Compute the output of the score-based model. + + Args: + x: A mini-batch of input data (Adjacency matrices). + labels: A mini-batch of conditioning variables for time steps. Should be interpreted differently + for different models. + mask: Mask for adjacency matrices. + + Returns: + A tuple of (model output, new mutable states) + """ + if not train: + model.eval() + return model(x, labels, *args, **kwargs) + else: + model.train() + return model(x, labels, *args, **kwargs) + + return model_fn + + +def get_score_fn(sde, model, train=False, continuous=False): + """Wraps `score_fn` so that the model output corresponds to a real time-dependent score function. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + model: A score model. + train: `True` for training and `False` for evaluation. + continuous: If `True`, the score-based model is expected to directly take continuous time steps. + + Returns: + A score function. + """ + model_fn = get_model_fn(model, train=train) + + if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + def score_fn(x, t, *args, **kwargs): + # Scale neural network output by standard deviation and flip sign + if continuous or isinstance(sde, sde_lib.subVPSDE): + # For VP-trained models, t=0 corresponds to the lowest noise level + # The maximum value of time embedding is assumed to 999 for continuously-trained models. + labels = t * 999 + score = model_fn(x, labels, *args, **kwargs) + std = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VP-trained models, t=0 corresponds to the lowest noise level + labels = t * (sde.N - 1) + score = model_fn(x, labels, *args, **kwargs) + std = sde.sqrt_1m_alpha_cumprod.to(labels.device)[ + labels.long()] + + score = -score / std[:, None, None] + return score + + elif isinstance(sde, sde_lib.VESDE): + def score_fn(x, t, *args, **kwargs): + if continuous: + labels = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VE-trained models, t=0 corresponds to the highest noise level + labels = sde.T - t + labels *= sde.N - 1 + labels = torch.round(labels).long() + + score = model_fn(x, labels, *args, **kwargs) + return score + + else: + raise NotImplementedError( + f"SDE class {sde.__class__.__name__} not yet supported.") + + return score_fn + + +def get_classifier_grad_fn(sde, classifier, train=False, continuous=False, + regress=True, labels='max'): + logit_fn = get_logit_fn(sde, classifier, train, continuous) + + def classifier_grad_fn(x, t, *args, **kwargs): + with torch.enable_grad(): + x_in = x.detach().requires_grad_(True) + if regress: + assert labels in ['max', 'min'] + logit = logit_fn(x_in, t, *args, **kwargs) + prob = logit.sum() + else: + logit = logit_fn(x_in, t, *args, **kwargs) + # prob = torch.nn.functional.log_softmax(logit, dim=-1)[torch.arange(labels.shape[0]), labels].sum() + log_prob = F.log_softmax(logit, dim=-1) + prob = log_prob[range(len(logit)), labels.view(-1)].sum() + # prob.backward() + # classifier_grad = x_in.grad + classifier_grad = torch.autograd.grad(prob, x_in)[0] + return classifier_grad + + return classifier_grad_fn + + +def get_logit_fn(sde, classifier, train=False, continuous=False): + classifier_fn = get_model_fn(classifier, train=train) + + if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + def logit_fn(x, t, *args, **kwargs): + # Scale neural network output by standard deviation and flip sign + if continuous or isinstance(sde, sde_lib.subVPSDE): + # For VP-trained models, t=0 corresponds to the lowest noise level + # The maximum value of time embedding is assumed to 999 for continuously-trained models. + labels = t * 999 + logit = classifier_fn(x, labels, *args, **kwargs) + # std = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VP-trained models, t=0 corresponds to the lowest noise level + labels = t * (sde.N - 1) + logit = classifier_fn(x, labels, *args, **kwargs) + # std = sde.sqrt_1m_alpha_cumprod.to(labels.device)[ + # labels.long()] + + # score = -score / std[:, None, None] + return logit + + elif isinstance(sde, sde_lib.VESDE): + def logit_fn(x, t, *args, **kwargs): + if continuous: + labels = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VE-trained models, t=0 corresponds to the highest noise level + labels = sde.T - t + labels *= sde.N - 1 + labels = torch.round(labels).long() + + logit = classifier_fn(x, labels, *args, **kwargs) + return logit + + return logit_fn + + +def get_predictor_fn(sde, model, train=False, continuous=False): + """Wraps `score_fn` so that the model output corresponds to a real time-dependent score function. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + model: A predictor model. + train: `True` for training and `False` for evaluation. + continuous: If `True`, the score-based model is expected to directly take continuous time steps. + + Returns: + A score function. + """ + model_fn = get_model_fn(model, train=train) + + if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + def predictor_fn(x, t, *args, **kwargs): + # Scale neural network output by standard deviation and flip sign + if continuous or isinstance(sde, sde_lib.subVPSDE): + # For VP-trained models, t=0 corresponds to the lowest noise level + # The maximum value of time embedding is assumed to 999 for continuously-trained models. + labels = t * 999 + pred = model_fn(x, labels, *args, **kwargs) + std = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VP-trained models, t=0 corresponds to the lowest noise level + labels = t * (sde.N - 1) + pred = model_fn(x, labels, *args, **kwargs) + std = sde.sqrt_1m_alpha_cumprod.to(labels.device)[ + labels.long()] + + # score = -score / std[:, None, None] + return pred + + elif isinstance(sde, sde_lib.VESDE): + def predictor_fn(x, t, *args, **kwargs): + if continuous: + labels = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VE-trained models, t=0 corresponds to the highest noise level + labels = sde.T - t + labels *= sde.N - 1 + labels = torch.round(labels).long() + + pred = model_fn(x, labels, *args, **kwargs) + return pred + + else: + raise NotImplementedError( + f"SDE class {sde.__class__.__name__} not yet supported.") + + return predictor_fn + + +def to_flattened_numpy(x): + """Flatten a torch tensor `x` and convert it to numpy.""" + return x.detach().cpu().numpy().reshape((-1,)) + + +def from_flattened_numpy(x, shape): + """Form a torch tensor with the given `shape` from a flattened numpy array `x`.""" + return torch.from_numpy(x.reshape(shape)) + + +@torch.no_grad() +def mask_adj2node(adj_mask): + """Convert batched adjacency mask matrices to batched node mask matrices. + + Args: + adj_mask: [B, N, N] Batched adjacency mask matrices without self-loop edge. + + Output: + node_mask: [B, N] Batched node mask matrices indicating the valid nodes. + """ + + batch_size, max_num_nodes, _ = adj_mask.shape + + node_mask = adj_mask[:, 0, :].clone() + node_mask[:, 0] = 1 + + return node_mask + + +@torch.no_grad() +def get_rw_feat(k_step, dense_adj): + """Compute k_step Random Walk for given dense adjacency matrix.""" + + rw_list = [] + deg = dense_adj.sum(-1, keepdims=True) + AD = dense_adj / (deg + 1e-8) + rw_list.append(AD) + + for _ in range(k_step): + rw = torch.bmm(rw_list[-1], AD) + rw_list.append(rw) + rw_map = torch.stack(rw_list[1:], dim=1) # [B, k_step, N, N] + + rw_landing = torch.diagonal( + rw_map, offset=0, dim1=2, dim2=3) # [B, k_step, N] + rw_landing = rw_landing.permute(0, 2, 1) # [B, N, rw_depth] + + # get the shortest path distance indices + tmp_rw = rw_map.sort(dim=1)[0] + spd_ind = (tmp_rw <= 0).sum(dim=1) # [B, N, N] + + spd_onehot = torch.nn.functional.one_hot( + spd_ind, num_classes=k_step+1).to(torch.float) + spd_onehot = spd_onehot.permute(0, 3, 1, 2) # [B, kstep, N, N] + + return rw_landing, spd_onehot diff --git a/MobileNetV3/run_lib.py b/MobileNetV3/run_lib.py new file mode 100644 index 0000000..aa22bc6 --- /dev/null +++ b/MobileNetV3/run_lib.py @@ -0,0 +1,448 @@ +import os +import torch +import numpy as np +import random +import logging +import time + +from absl import flags + +from torch_geometric.loader import DataLoader +import pickle +from scipy.stats import pearsonr, spearmanr +import wandb +import pandas as pd +import torch +from torch.utils.data import DataLoader #, Subset + +from models import pgsn +from models import cate +from models import dagformer +from models import digcn +from models import digcn_meta +from models import regressor +from models.GDSS import scorenetx +import losses +import sampling +from models import utils as mutils +from models.ema import ExponentialMovingAverage +import datasets_nas +import sde_lib +from utils import * +from logger import Logger +from analysis.arch_metrics import SamplingArchMetrics, SamplingArchMetricsMeta + +FLAGS = flags.FLAGS + + +def set_exp_name(config, classifier_config_nf=None): + exp_name = f'./exp/{config.task}/{config.folder_name}' + wandb_exp_name = exp_name + + os.makedirs(exp_name, exist_ok=True) + + config.exp_name = exp_name + + set_random_seed(config) + + return exp_name, wandb_exp_name + + +def set_random_seed(config): + seed = config.seed + os.environ['PYTHONHASHSEED'] = str(seed) + + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + np.random.seed(seed) + random.seed(seed) + + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def sde_train(config): + """Runs the training pipeline. + + Args: + config: Configuration to use. + workdir: Working directory for checkpoints and TF summaries. + If this contains checkpoint training will be resumed from the latest checkpoint. + """ + # Wandb logger + exp_name, wandb_exp_name = set_exp_name(config) + wandb_logger = Logger( + log_dir=exp_name, + exp_name=wandb_exp_name, + write_textfile=True, + use_wandb=config.log.use_wandb, + wandb_project_name=config.log.wandb_project_name) + wandb_logger.update_config(config, is_args=True) + wandb_logger.write_str(str(vars(config))) + wandb_logger.write_str('-' * 100) + + # Create directories for experimental logs + sample_dir = os.path.join(exp_name, "samples") + os.makedirs(sample_dir, exist_ok=True) + + # Initialize model. + score_model = mutils.create_model(config) + ema = ExponentialMovingAverage(score_model.parameters(), decay=config.model.ema_rate) + optimizer = losses.get_optimizer(config, score_model.parameters()) + state = dict(optimizer=optimizer, model=score_model, ema=ema, step=0, config=config) + + # Create checkpoints directly + checkpoint_dir = os.path.join(exp_name, "checkpoints") + # Intermediate checkpoints to resume training + checkpoint_meta_dir = os.path.join(exp_name, "checkpoints-meta", "checkpoint.pth") + os.makedirs(checkpoint_dir, exist_ok=True) + os.makedirs(os.path.dirname(checkpoint_meta_dir), exist_ok=True) + # Resume training when intermediate checkpoints are detected + if config.resume: + state = restore_checkpoint(config.resume_ckpt_path, state, config.device, resume=config.resume) + initial_step = int(state['step']) + + train_ds, eval_ds, test_ds = datasets_nas.get_dataset(config) + train_loader, eval_loader, test_loader = datasets_nas.get_dataloader(config, train_ds, eval_ds, test_ds) + n_node_pmf = None # temp + print(f'==> # of training elem: {len(train_ds)}') + train_iter = iter(train_loader) + # create data normalizer and its inverse + scaler = datasets_nas.get_data_scaler(config) + inverse_scaler = datasets_nas.get_data_inverse_scaler(config) + + # Setup SDEs + if config.training.sde.lower() == 'vpsde': + sde = sde_lib.VPSDE(beta_min=config.model.beta_min, beta_max=config.model.beta_max, N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'vesde': + sde = sde_lib.VESDE(sigma_min=config.model.sigma_min, sigma_max=config.model.sigma_max, N=config.model.num_scales) + sampling_eps = 1e-5 + else: + raise NotImplementedError(f"SDE {config.training.sde} unknown.") + + # Build one-step training and evaluation functions + optimize_fn = losses.optimization_manager(config) + continuous = config.training.continuous + reduce_mean = config.training.reduce_mean + likelihood_weighting = config.training.likelihood_weighting + train_step_fn = losses.get_step_fn(sde, train=True, optimize_fn=optimize_fn, + reduce_mean=reduce_mean, continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name) + eval_step_fn = losses.get_step_fn(sde, train=False, optimize_fn=optimize_fn, + reduce_mean=reduce_mean, continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name) + + # Build sampling functions + if config.training.snapshot_sampling: + sampling_shape = (config.training.eval_batch_size, config.data.max_node, config.data.n_vocab) + sampling_fn = sampling.get_sampling_fn( + config, sde, sampling_shape, inverse_scaler, sampling_eps, config.data.name) + + num_train_steps = config.training.n_iters + + # Build analysis tools + sampling_metrics = SamplingArchMetrics(config, train_ds, exp_name) + # visualization_tools = ArchVisualization(config, remove_none=False, exp_name=exp_name) + + # -------- Train --------- # + logging.info("Starting training loop at step %d." % (initial_step,)) + element = {'train': ['training_loss'], + 'eval': ['eval_loss'], + 'test': ['test_loss'], + 'sample': ['r_valid', 'r_unique', 'r_novel'], + 'valid_error': ['multi_node_type', 'INVALID_1OR2', 'INVALID_3AND4', 'x_elem_sum']} + + is_best = False + min_test_loss = 1e05 + for step in range(initial_step, num_train_steps + 1): + try: + x, adj, extra = next(train_iter) + except StopIteration: + train_iter = train_loader.__iter__() + x, adj, extra = next(train_iter) + mask = aug_mask(adj, algo=config.data.aug_mask_algo, data=config.data.name) + x, adj, mask = scaler(x.to(config.device)), adj.to(config.device), mask.to(config.device) + # mask = cate_mask(adj) + # adj, mask = dense_adj(graphs, config.data.max_node, scaler, config.data.dequantization) + batch = (x, adj, mask) + # Execute one training step + loss = train_step_fn(state, batch) + wandb_logger.update(key="training_loss", v=loss.item()) + if step % config.training.log_freq == 0: + logging.info("step: %d, training_loss: %.5e" % (step, loss.item())) + + # Report the loss on evaluation dataset periodically + if step % config.training.eval_freq == 0: + for eval_x, eval_adj, eval_extra in eval_loader: + eval_mask = aug_mask(eval_adj, algo=config.data.aug_mask_algo, data=config.data.name) + eval_x, eval_adj, eval_mask = scaler(eval_x.to(config.device)), eval_adj.to(config.device), eval_mask.to(config.device) + eval_batch = (eval_x, eval_adj, eval_mask) + eval_loss = eval_step_fn(state, eval_batch) + logging.info("step: %d, eval_loss: %.5e" % (step, eval_loss.item())) + wandb_logger.update(key="eval_loss", v=eval_loss.item()) + for test_x, test_adj, test_extra in test_loader: + test_mask = aug_mask(test_adj, algo=config.data.aug_mask_algo, data=config.data.name) + test_x, test_adj, test_mask = scaler(test_x.to(config.device)), test_adj.to(config.device), test_mask.to(config.device) + test_batch = (test_x, test_adj, test_mask) + test_loss = eval_step_fn(state, test_batch) + logging.info("step: %d, test_loss: %.5e" % (step, test_loss.item())) + wandb_logger.update(key="test_loss", v=test_loss.item()) + + if wandb_logger.logs['test_loss'].avg < min_test_loss: + is_best = True + + # Save a checkpoint periodically and generate samples + if step != 0 and step % config.training.snapshot_freq == 0 or step == num_train_steps: + # Save the checkpoint. + save_step = step // config.training.snapshot_freq + # save_checkpoint(os.path.join(checkpoint_dir, f'checkpoint_{save_step}.pth'), state) + save_checkpoint(checkpoint_dir, state, step, save_step, is_best) + + # Generate and save samples + if config.training.snapshot_sampling: + ema.store(score_model.parameters()) + ema.copy_to(score_model.parameters()) + + sample, sample_steps, _ = sampling_fn(score_model, mask) # sample: [batch_size, num_node, n_vocab] + sample_list = quantize(sample, adj, + alpha=config.sampling.alpha, qtype=config.sampling.qtype) # quantization + this_sample_dir = os.path.join(sample_dir, "iter_{}".format(step)) + os.makedirs(this_sample_dir, exist_ok=True) + # check samples + arch_metric = sampling_metrics(arch_list=sample_list, adj=adj, mask=mask, this_sample_dir=this_sample_dir, test=False) + r_valid, r_unique, r_novel = arch_metric[0][0], arch_metric[0][1], arch_metric[0][2] + if len(arch_metric[0]) > 3: + error_type_1 = arch_metric[0][3] + error_type_2 = arch_metric[0][4] + error_type_3 = arch_metric[0][5] + x_elem_sum = int(torch.sum(torch.tensor(sample_list))) + else: + error_type_1 = None + + logging.info("step: %d, r_valid: %.5e" % (step, r_valid)) + logging.info("step: %d, r_unique: %.5e" % (step, r_unique)) + logging.info("step: %d, r_novel: %.5e" % (step, r_novel)) + if error_type_1 is not None: + logging.info("step: %d, multi_node_type: %.5e" % (step, error_type_1)) + logging.info("step: %d, INVALID_1OR2: %.5e" % (step, error_type_2)) + logging.info("step: %d, INVALID_3AND4: %.5e" % (step, error_type_3)) + logging.info("step: %d, x_elem_sum: %d" % (step, x_elem_sum)) + # writer.add_scalar("r_valid", r_valid, step) + # res = nasbench201.get_prop(sample_valid_str_list=sample_valid_str) + if config.log.use_wandb: + # wandb_logger.log_sample(sample) + wandb_logger.update(key="r_valid", v=r_valid) + wandb_logger.update(key="r_unique", v=r_unique) + wandb_logger.update(key="r_novel", v=r_novel) + if error_type_1 is not None: + wandb_logger.update(key="multi_node_type", v=error_type_1) + wandb_logger.update(key="INVALID_1OR2", v=error_type_2) + wandb_logger.update(key="INVALID_3AND4", v=error_type_3) + wandb_logger.update(key="x_elem_sum", v=x_elem_sum) + if config.log.log_valid_sample_prop: + wandb_logger.log_valid_sample_prop(arch_metric, x_axis='latency', y_axis='test_acc') + + if step % config.training.eval_freq == 0: + wandb_logger.write_log(element=element, step=step) + else: + wandb_logger.write_log(element={'train': ['training_loss']}, step=step) + wandb_logger.reset() + wandb_logger.save_log() + + +def meta_predictor_train(config): + + # Wandb logger + exp_name, wandb_exp_name = set_exp_name(config) + wandb_logger = Logger( + log_dir=exp_name, + exp_name=wandb_exp_name, + write_textfile=True, + use_wandb=config.log.use_wandb, + wandb_project_name=config.log.wandb_project_name) + wandb_logger.update_config(config, is_args=True) + wandb_logger.write_str(str(vars(config))) + wandb_logger.write_str('-' * 100) + + # Create directories for experimental logs + sample_dir = os.path.join(exp_name, "samples") + os.makedirs(sample_dir, exist_ok=True) + + # Initialize model. + predictor_model = mutils.create_model(config) + optimizer = losses.get_optimizer(config, predictor_model.parameters()) + state = dict(optimizer=optimizer, model=predictor_model, step=0, config=config) + # Create checkpoints directly + + checkpoint_dir = os.path.join(exp_name, "checkpoints") + # Intermediate checkpoints to resume training + checkpoint_meta_dir = os.path.join(exp_name, "checkpoints-meta", "checkpoint.pth") + os.makedirs(checkpoint_dir, exist_ok=True) + os.makedirs(os.path.dirname(checkpoint_meta_dir), exist_ok=True) + # Resume training when intermediate checkpoints are detected + state = restore_checkpoint(checkpoint_meta_dir, state, config.device, resume=config.resume) + initial_step = int(state['step']) + + # Build dataloader and iterators + train_ds, eval_ds, test_ds = datasets_nas.get_meta_dataset(config) + train_loader, eval_loader, test_loader = datasets_nas.get_dataloader(config, train_ds, eval_ds, test_ds) + + train_iter = iter(train_loader) + # create data normalizer and its inverse + scaler = datasets_nas.get_data_scaler(config) + inverse_scaler = datasets_nas.get_data_inverse_scaler(config) + + # Setup SDEs + if config.training.sde.lower() == 'vpsde': + sde = sde_lib.VPSDE(beta_min=config.model.beta_min, beta_max=config.model.beta_max, N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'vesde': + sde = sde_lib.VESDE(sigma_min=config.model.sigma_min, sigma_max=config.model.sigma_max, N=config.model.num_scales) + sampling_eps = 1e-5 + else: + raise NotImplementedError(f"SDE {config.training.sde} unknown.") + + # Build one-step training and evaluation functions + optimize_fn = losses.optimization_manager(config) + continuous = config.training.continuous + reduce_mean = config.training.reduce_mean + likelihood_weighting = config.training.likelihood_weighting + train_step_fn = losses.get_step_fn_predictor(sde, train=True, optimize_fn=optimize_fn, + reduce_mean=reduce_mean, continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name, label_list=config.data.label_list, + noised=config.training.noised, + t_spot=config.training.t_spot, + is_meta=True) + eval_step_fn = losses.get_step_fn_predictor(sde, train=False, optimize_fn=optimize_fn, + reduce_mean=reduce_mean, continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name, label_list=config.data.label_list, + noised=config.training.noised, + t_spot=config.training.t_spot, + is_meta=True) + + # Build sampling functions and Load pre-trained score network + if config.training.snapshot_sampling: + sampling_shape = (config.training.eval_batch_size, config.data.max_node, config.data.n_vocab) + sampling_fn = sampling.get_sampling_fn(config, sde, sampling_shape, inverse_scaler, + sampling_eps, config.data.name, conditional=True, + is_meta=True, data_name='cifar10', num_sample=config.model.num_sample) + # Score model + score_config = torch.load(config.scorenet_ckpt_path)['config'] + check_config(score_config, config) + score_model = mutils.create_model(score_config) + score_ema = ExponentialMovingAverage(score_model.parameters(), decay=score_config.model.ema_rate) + score_state = dict(model=score_model, ema=score_ema, step=0, config=score_config) + score_state = restore_checkpoint(config.scorenet_ckpt_path, score_state, device=config.device, resume=True) + score_ema.copy_to(score_model.parameters()) + + num_train_steps = config.training.n_iters + + # Build analysis tools + sampling_metrics = SamplingArchMetricsMeta(config, train_ds, exp_name) + + # -------- Train --------- # + logging.info("Starting training loop at step %d." % (initial_step,)) + element = {'train': ['training_loss'], + 'eval': ['eval_loss']} + + is_best = False + max_eval_p_corr = -1 + for step in range(initial_step, num_train_steps + 1): + try: + x, adj, extra, task = next(train_iter) + except StopIteration: + train_iter = train_loader.__iter__() + x, adj, extra, task = next(train_iter) + mask = aug_mask(adj, algo=config.data.aug_mask_algo, data=config.data.name) + x, adj, mask = scaler(x.to(config.device)), adj.to(config.device), mask.to(config.device) + # task = task.to(config.device) if config.data.name == 'NASBench201' else [_.to(config.device) for _ in task] + task = [_.to(config.device) for _ in task] if config.data.name == 'ofa' else task.to(config.device) + # mask = cate_mask(adj) + # adj, mask = dense_adj(graphs, config.data.max_node, scaler, config.data.dequantization) + batch = (x, adj, mask, extra, task) + # Execute one training step + loss, pred, labels = train_step_fn(state, batch) + wandb_logger.update(key="training_loss", v=loss.item()) + if step % config.training.log_freq == 0: + logging.info("step: %d, training_loss: %.5e" % (step, loss.item())) + + # Save a temporary checkpoint to resume training after pre-emption periodically + if step != 0 and step % config.training.snapshot_freq_for_preemption == 0: + save_checkpoint(checkpoint_meta_dir, state, step, save_step, is_best) + + # Report the loss on evaluation dataset periodically + if step % config.training.eval_freq == 0: + eval_pred_list, eval_labels_list = list(), list() + for eval_x, eval_adj, eval_extra, eval_task in eval_loader: + eval_mask = aug_mask(eval_adj, algo=config.data.aug_mask_algo, data=config.data.name) + eval_x, eval_adj, eval_mask = scaler(eval_x.to(config.device)), eval_adj.to(config.device), eval_mask.to(config.device) + eval_task = [_.to(config.device) for _ in eval_task] + eval_batch = (eval_x, eval_adj, eval_mask, eval_extra, eval_task) + eval_loss, eval_pred, eval_labels = eval_step_fn(state, eval_batch) + eval_pred_list += [v.detach().item() for v in eval_pred.squeeze()] + eval_labels_list += [v.detach().item() for v in eval_labels.squeeze()] + logging.info("step: %d, eval_loss: %.5e" % (step, eval_loss.item())) + wandb_logger.update(key="eval_loss", v=eval_loss.item()) + + eval_p_corr = pearsonr(np.array(eval_pred_list), np.array(eval_labels_list))[0] + eval_s_corr = spearmanr(np.array(eval_pred_list), np.array(eval_labels_list))[0] + + if eval_p_corr > max_eval_p_corr: + is_best = True + max_eval_p_corr = eval_p_corr + + # Save a checkpoint periodically and generate samples + if step != 0 and step % config.training.snapshot_freq == 0 or step == num_train_steps: + # Save the checkpoint. + save_step = step // config.training.snapshot_freq + save_checkpoint(checkpoint_dir, state, step, save_step, is_best) + + # Generate and save samples + if config.training.snapshot_sampling: + score_ema.store(score_model.parameters()) + score_ema.copy_to(score_model.parameters()) + + sample, sample_steps, sample_chain, (score_grad_norm_p, classifier_grad_norm_p, score_grad_norm_c, classifier_grad_norm_c) = \ + sampling_fn(score_model, mask, predictor_model, eval_chain=False, number_chain_steps=config.sampling.number_chain_steps, + classifier_scale=config.sampling.classifier_scale) + sample_list = quantize(sample, adj) # quantization + this_sample_dir = os.path.join(sample_dir, "iter_{}".format(step)) + os.makedirs(this_sample_dir, exist_ok=True) + arch_metric = sampling_metrics(arch_list=sample_list, adj=adj, mask=mask, + this_sample_dir=this_sample_dir, test=False, + check_dataname=config.sampling.check_dataname) + + r_valid, r_unique, r_novel = arch_metric[0][0], arch_metric[0][1], arch_metric[0][2] + test_acc_list = arch_metric[2]['test_acc_list'] + + if step % config.training.eval_freq == 0: + wandb_logger.write_log(element=element, step=step) + else: + wandb_logger.write_log(element={'train': ['training_loss']}, step=step) + wandb_logger.reset() + + +def check_config(config1, config2): + assert config1.training.sde == config2.training.sde + assert config1.training.continuous == config2.training.continuous + assert config1.data.centered == config2.data.centered + assert config1.data.max_node == config2.data.max_node + assert config1.data.n_vocab == config2.data.n_vocab + +run_train_dict = { + 'sde': sde_train, + 'meta_predictor': meta_predictor_train +} + + +def train(config): + run_train_dict[config.model_type](config) + + diff --git a/MobileNetV3/sampling.py b/MobileNetV3/sampling.py new file mode 100644 index 0000000..f3b2761 --- /dev/null +++ b/MobileNetV3/sampling.py @@ -0,0 +1,1214 @@ +"""Various sampling methods.""" + +import functools + +import torch +import numpy as np +import abc +import sys +import os + +from models.utils import from_flattened_numpy, to_flattened_numpy, get_score_fn + +from scipy import integrate +from torchdiffeq import odeint +import sde_lib +from models import utils as mutils +from tqdm import trange + +from datasets_nas import MetaTestDataset +# from configs.ckpt import META_DATAROOT_NB201, META_DATAROOT_OFA +from all_path import PROCESSED_DATA_PATH + +_CORRECTORS = {} +_PREDICTORS = {} + + +def register_predictor(cls=None, *, name=None): + """A decorator for registering predictor classes.""" + + def _register(cls): + if name is None: + local_name = cls.__name__ + else: + local_name = name + if local_name in _PREDICTORS: + raise ValueError(f'Already registered predictor with name: {local_name}') + _PREDICTORS[local_name] = cls + return cls + + if cls is None: + return _register + else: + return _register(cls) + + +def register_corrector(cls=None, *, name=None): + """A decorator for registering corrector classes.""" + + def _register(cls): + if name is None: + local_name = cls.__name__ + else: + local_name = name + if local_name in _CORRECTORS: + raise ValueError(f'Already registered corrector with name: {local_name}') + _CORRECTORS[local_name] = cls + return cls + + if cls is None: + return _register + else: + return _register(cls) + + +def get_predictor(name): + return _PREDICTORS[name] + + +def get_corrector(name): + return _CORRECTORS[name] + + +def get_sampling_fn( + config, sde, shape, inverse_scaler, eps, data, conditional=False, + p=1, prod_w=False, weight_ratio_abs=False, + is_meta=False, data_name='cifar10', num_sample=20, is_multi_obj=False): + """Create a sampling function. + + Args: + config: A `ml_collections.ConfigDict` object that contains all configuration information. + sde: A `sde_lib.SDE` object that represents the forward SDE. + shape: A sequence of integers representing the expected shape of a single sample. + inverse_scaler: The inverse data normalizer function. + eps: A `float` number. The reverse-time SDE is only integrated to `eps` for numerical stability. + + Returns: + A function that takes random states and a replicated training state and outputs samples with the + trailing dimensions matching `shape`. + """ + + sampler_name = config.sampling.method + # Probability flow ODE sampling with black-box ODE solvers + if sampler_name.lower() == 'ode': + sampling_fn = get_ode_sampler(sde=sde, + shape=shape, + inverse_scaler=inverse_scaler, + denoise=config.sampling.noise_removal, + eps=eps, + rtol=config.sampling.rtol, + atol=config.sampling.atol, + device=config.device) + elif sampler_name.lower() == 'diffeq': + sampling_fn = get_diffeq_sampler(sde=sde, + shape=shape, + inverse_scaler=inverse_scaler, + denoise=config.sampling.noise_removal, + eps=eps, + rtol=config.sampling.rtol, + atol=config.sampling.atol, + step_size=config.sampling.ode_step, + method=config.sampling.ode_method, + device=config.device) + # Predictor-Corrector sampling. Predictor-only and Corrector-only samplers are special cases. + elif sampler_name.lower() == 'pc': + predictor = get_predictor(config.sampling.predictor.lower()) + corrector = get_corrector(config.sampling.corrector.lower()) + # print(config.sampling.predictor.lower(), config.sampling.corrector.lower()) + if data in ['NASBench201', 'ofa']: + if is_meta: + sampling_fn = get_pc_conditional_sampler_meta_nas(sde=sde, + shape=shape, + predictor=predictor, + corrector=corrector, + inverse_scaler=inverse_scaler, + snr=config.sampling.snr, + n_steps=config.sampling.n_steps_each, + probability_flow=config.sampling.probability_flow, + continuous=config.training.continuous, + denoise=config.sampling.noise_removal, + eps=eps, + device=config.device, + regress=config.sampling.regress, + labels=config.sampling.labels, + classifier_scale=config.sampling.classifier_scale, + weight_scheduling=config.sampling.weight_scheduling, + weight_ratio=config.sampling.weight_ratio, + t_spot=config.sampling.t_spot, + t_spot_end=config.sampling.t_spot_end, + p=p, + prod_w=prod_w, + weight_ratio_abs=weight_ratio_abs, + data_name=data_name, + num_sample=num_sample, + search_space=config.data.name) + elif is_multi_obj: + sampling_fn = get_pc_conditional_sampler_nas(sde=sde, + shape=shape, + predictor=predictor, + corrector=corrector, + inverse_scaler=inverse_scaler, + snr=config.sampling.snr, + n_steps=config.sampling.n_steps_each, + probability_flow=config.sampling.probability_flow, + continuous=config.training.continuous, + denoise=config.sampling.noise_removal, + eps=eps, + device=config.device, + regress=config.sampling.regress, + labels=config.sampling.labels, + classifier_scale=config.sampling.classifier_scale, + weight_scheduling=config.sampling.weight_scheduling, + weight_ratio=config.sampling.weight_ratio, + t_spot=config.sampling.t_spot, + t_spot_end=config.sampling.t_spot_end, + p=p, + prod_w=prod_w, + weight_ratio_abs=weight_ratio_abs) + elif conditional: + sampling_fn = get_pc_conditional_sampler_nas(sde=sde, + shape=shape, + predictor=predictor, + corrector=corrector, + inverse_scaler=inverse_scaler, + snr=config.sampling.snr, + n_steps=config.sampling.n_steps_each, + probability_flow=config.sampling.probability_flow, + continuous=config.training.continuous, + denoise=config.sampling.noise_removal, + eps=eps, + device=config.device, + regress=config.sampling.regress, + labels=config.sampling.labels, + classifier_scale=config.sampling.classifier_scale, + weight_scheduling=config.sampling.weight_scheduling, + weight_ratio=config.sampling.weight_ratio, + t_spot=config.sampling.t_spot, + t_spot_end=config.sampling.t_spot_end, + p=p, + prod_w=prod_w, + weight_ratio_abs=weight_ratio_abs) + else: + sampling_fn = get_pc_sampler_nas(sde=sde, + shape=shape, + predictor=predictor, + corrector=corrector, + inverse_scaler=inverse_scaler, + snr=config.sampling.snr, + n_steps=config.sampling.n_steps_each, + probability_flow=config.sampling.probability_flow, + continuous=config.training.continuous, + denoise=config.sampling.noise_removal, + eps=eps, + device=config.device) + + else: + sampling_fn = get_pc_sampler(sde=sde, + shape=shape, + predictor=predictor, + corrector=corrector, + inverse_scaler=inverse_scaler, + snr=config.sampling.snr, + n_steps=config.sampling.n_steps_each, + probability_flow=config.sampling.probability_flow, + continuous=config.training.continuous, + denoise=config.sampling.noise_removal, + eps=eps, + device=config.device) + else: + raise ValueError(f"Sampler name {sampler_name} unknown.") + + return sampling_fn + + +class Predictor(abc.ABC): + """The abstract class for a predictor algorithm.""" + + def __init__(self, sde, score_fn, probability_flow=False): + super().__init__() + self.sde = sde + # Compute the reverse SDE/ODE + if isinstance(sde, tuple): + self.rsde = (sde[0].reverse(score_fn, probability_flow), sde[1].reverse(score_fn, probability_flow)) + else: + self.rsde = sde.reverse(score_fn, probability_flow) + self.score_fn = score_fn + + @abc.abstractmethod + def update_fn(self, x, t, *args, **kwargs): + """One update of the predictor. + + Args: + x: A PyTorch tensor representing the current state. + t: A PyTorch tensor representing the current time step. + + Returns: + x: A PyTorch tensor of the next state. + x_mean: A PyTorch tensor. The next state without random noise. Useful for denoising. + """ + pass + + +class Corrector(abc.ABC): + """The abstract class for a corrector algorithm.""" + + def __init__(self, sde, score_fn, snr, n_steps): + super().__init__() + self.sde = sde + self.score_fn = score_fn + self.snr = snr + self.n_steps = n_steps + + @abc.abstractmethod + def update_fn(self, x, t, *args, **kwargs): + """One update of the corrector. + + Args: + x: A PyTorch tensor representing the current state. + t: A PyTorch tensor representing the current time step. + + Returns: + x: A PyTorch tensor of the next state. + x_mean: A PyTorch tensor. The next state without random noise. Useful for denoising. + """ + pass + + +@register_predictor(name='euler_maruyama') +class EulerMaruyamaPredictor(Predictor): + def __init__(self, sde, score_fn, probability_flow=False): + super().__init__(sde, score_fn, probability_flow) + + # def update_fn(self, x, t, *args, **kwargs): + # dt = -1. / self.rsde.N + # z = torch.randn_like(x) + # z = torch.tril(z, -1) + # z = z + z.transpose(-1, -2) + # drift, diffusion = self.rsde.sde(x, t, *args, **kwargs) + # drift = torch.tril(drift, -1) + # drift = drift + drift.transpose(-1, -2) + # x_mean = x + drift * dt + # x = x_mean + diffusion[:, None, None, None] * np.sqrt(-dt) * z + # return x, x_mean + + def update_fn(self, x, t, *args, **kwargs): + dt = -1. / self.rsde.N + z = torch.randn_like(x) + # z = torch.tril(z, -1) + # z = z + z.transpose(-1, -2) + drift, diffusion = self.rsde.sde(x, t, *args, **kwargs) + # drift = torch.tril(drift, -1) + # drift = drift + drift.transpose(-1, -2) + x_mean = x + drift * dt + x = x_mean + diffusion[:, None, None] * np.sqrt(-dt) * z + return x, x_mean + + +@register_predictor(name='reverse_diffusion') +class ReverseDiffusionPredictor(Predictor): + def __init__(self, sde, score_fn, probability_flow=False): + super().__init__(sde, score_fn, probability_flow) + + # def update_fn(self, x, t, *args, **kwargs): + # f, G = self.rsde.discretize(x, t, *args, **kwargs) + # f = torch.tril(f, -1) + # f = f + f.transpose(-1, -2) + # z = torch.randn_like(x) + # z = torch.tril(z, -1) + # z = z + z.transpose(-1, -2) + + # x_mean = x - f + # x = x_mean + G[:, None, None, None] * z + # return x, x_mean + + def update_fn(self, x, t, *args, **kwargs): + f, G = self.rsde.discretize(x, t, *args, **kwargs) + # f = torch.tril(f, -1) + # f = f + f.transpose(-1, -2) + z = torch.randn_like(x) + # z = torch.tril(z, -1) + # z = z + z.transpose(-1, -2) + + x_mean = x - f + x = x_mean + G[:, None, None] * z + return x, x_mean + + +@register_predictor(name='none') +class NonePredictor(Predictor): + """An empty predictor that does nothing.""" + + def __init__(self, sde, score_fn, probability_flow=False): + pass + + def update_fn(self, x, t, *args, **kwargs): + return x, x + + +@register_corrector(name='langevin') +class LangevinCorrector(Corrector): + def __init__(self, sde, score_fn, snr, n_steps): + super().__init__(sde, score_fn, snr, n_steps) + + # def update_fn(self, x, t, *args, **kwargs): + # sde = self.sde + # score_fn = self.score_fn + # n_steps = self.n_steps + # target_snr = self.snr + # if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + # timestep = (t * (sde.N - 1) / sde.T).long() + # # Note: it seems that subVPSDE doesn't set alphas + # alpha = sde.alphas.to(t.device)[timestep] + # else: + # alpha = torch.ones_like(t) + + # for i in range(n_steps): + + # grad = score_fn(x, t, *args, **kwargs) + # noise = torch.randn_like(x) + + # noise = torch.tril(noise, -1) + # noise = noise + noise.transpose(-1, -2) + + # mask = kwargs['mask'] + + # # mask invalid elements and calculate norm + # mask_tmp = mask.reshape(mask.shape[0], -1) + + # grad_norm = torch.norm(mask_tmp * grad.reshape(grad.shape[0], -1), dim=-1).mean() + # noise_norm = torch.norm(mask_tmp * noise.reshape(noise.shape[0], -1), dim=-1).mean() + + # step_size = (target_snr * noise_norm / grad_norm) ** 2 * 2 * alpha + # x_mean = x + step_size[:, None, None, None] * grad + # x = x_mean + torch.sqrt(step_size * 2)[:, None, None, None] * noise + + # return x, x_mean + + def update_fn(self, x, t, *args, **kwargs): + sde = self.sde + score_fn = self.score_fn + n_steps = self.n_steps + target_snr = self.snr + if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + timestep = (t * (sde.N - 1) / sde.T).long() + # Note: it seems that subVPSDE doesn't set alphas + alpha = sde.alphas.to(t.device)[timestep] + else: + alpha = torch.ones_like(t) + + for i in range(n_steps): + + grad = score_fn(x, t, *args, **kwargs) + noise = torch.randn_like(x) + + # noise = torch.tril(noise, -1) + # noise = noise + noise.transpose(-1, -2) + + # mask = kwargs['maskX'] + + # mask invalid elements and calculate norm + # mask_tmp = mask.reshape(mask.shape[0], -1) + + # grad_norm = torch.norm(mask_tmp * grad.reshape(grad.shape[0], -1), dim=-1).mean() + # noise_norm = torch.norm(mask_tmp * noise.reshape(noise.shape[0], -1), dim=-1).mean() + grad_norm = torch.norm(grad.reshape(grad.shape[0], -1), dim=-1).mean() + noise_norm = torch.norm(noise.reshape(noise.shape[0], -1), dim=-1).mean() + + step_size = (target_snr * noise_norm / grad_norm) ** 2 * 2 * alpha + x_mean = x + step_size[:, None, None] * grad + x = x_mean + torch.sqrt(step_size * 2)[:, None, None] * noise + + return x, x_mean + + +@register_corrector(name='none') +class NoneCorrector(Corrector): + """An empty corrector that does nothing.""" + + def __init__(self, sde, score_fn, snr, n_steps): + pass + + def update_fn(self, x, t, *args, **kwargs): + return x, x + + +def shared_predictor_update_fn(x, t, sde, model, + predictor, probability_flow, continuous, *args, **kwargs): + """A wrapper that configures and returns the update function of predictors.""" + score_fn = mutils.get_score_fn(sde, model, train=False, continuous=continuous) + if predictor is None: + # Corrector-only sampler + predictor_obj = NonePredictor(sde, score_fn, probability_flow) + else: + predictor_obj = predictor(sde, score_fn, probability_flow) + + return predictor_obj.update_fn(x, t, *args, **kwargs) + + +def shared_corrector_update_fn(x, t, sde, model, + corrector, continuous, snr, n_steps, *args, **kwargs): + """A wrapper that configures and returns the update function of correctors.""" + score_fn = mutils.get_score_fn(sde, model, train=False, continuous=continuous) + + if corrector is None: + # Predictor-only sampler + corrector_obj = NoneCorrector(sde, score_fn, snr, n_steps) + else: + corrector_obj = corrector(sde, score_fn, snr, n_steps) + + return corrector_obj.update_fn(x, t, *args, **kwargs) + + +def get_pc_sampler(sde, shape, predictor, corrector, inverse_scaler, snr, + n_steps=1, probability_flow=False, continuous=False, + denoise=True, eps=1e-3, device='cuda'): + """Create a Predictor-Corrector (PC) sampler. + + Args: + sde: An `sde_lib.SDE` object representing the forward SDE. + shape: A sequence of integers. The expected shape of a single sample. + predictor: A subclass of `sampling.Predictor` representing the predictor algorithm. + corrector: A subclass of `sampling.Corrector` representing the corrector algorithm. + inverse_scaler: The inverse data normalizer. + snr: A `float` number. The signal-to-noise ratio for configuring correctors. + n_steps: An integer. The number of corrector steps per predictor update. + probability_flow: If `True`, solve the reverse-time probability flow ODE when running the predictor. + continuous: `True` indicates that the score model was continuously trained. + denoise: If `True`, add one-step denoising to the final samples. + eps: A `float` number. The reverse-time SDE and ODE are integrated to `epsilon` to avoid numerical issues. + device: PyTorch device. + + Returns: + A sampling function that returns samples and the number of function evaluations during sampling. + """ + # Create predictor & corrector update functions + predictor_update_fn = functools.partial(shared_predictor_update_fn, + sde=sde, + predictor=predictor, + probability_flow=probability_flow, + continuous=continuous) + corrector_update_fn = functools.partial(shared_corrector_update_fn, + sde=sde, + corrector=corrector, + continuous=continuous, + snr=snr, + n_steps=n_steps) + + def pc_sampler(model, n_nodes_pmf): + """The PC sampler function. + + Args: + model: A score model. + n_nodes_pmf: Probability mass function of graph nodes. + + Returns: + Samples, number of function evaluations. + """ + with torch.no_grad(): + # Initial sample + x = sde.prior_sampling(shape).to(device) + timesteps = torch.linspace(sde.T, eps, sde.N, device=device) + + # Sample the number of nodes + n_nodes = torch.multinomial(n_nodes_pmf, shape[0], replacement=True) + mask = torch.zeros((shape[0], shape[-1]), device=device) + for i in range(shape[0]): + mask[i][:n_nodes[i]] = 1. + mask = (mask[:, None, :] * mask[:, :, None]).unsqueeze(1) + mask = torch.tril(mask, -1) + mask = mask + mask.transpose(-1, -2) + + x = x * mask + + for i in range(sde.N): + t = timesteps[i] + vec_t = torch.ones(shape[0], device=t.device) * t + x, x_mean = corrector_update_fn(x, vec_t, model=model, mask=mask) + x = x * mask + x, x_mean = predictor_update_fn(x, vec_t, model=model, mask=mask) + x = x * mask + + return inverse_scaler(x_mean if denoise else x) * mask, sde.N * (n_steps + 1), n_nodes + + return pc_sampler + +def get_pc_sampler_nas(sde, shape, predictor, corrector, inverse_scaler, snr, + n_steps=1, probability_flow=False, continuous=False, + denoise=True, eps=1e-3, device='cuda'): + """Create a Predictor-Corrector (PC) sampler. + + Args: + sde: An `sde_lib.SDE` object representing the forward SDE. + shape: A sequence of integers. The expected shape of a single sample. + predictor: A subclass of `sampling.Predictor` representing the predictor algorithm. + corrector: A subclass of `sampling.Corrector` representing the corrector algorithm. + inverse_scaler: The inverse data normalizer. + snr: A `float` number. The signal-to-noise ratio for configuring correctors. + n_steps: An integer. The number of corrector steps per predictor update. + probability_flow: If `True`, solve the reverse-time probability flow ODE when running the predictor. + continuous: `True` indicates that the score model was continuously trained. + denoise: If `True`, add one-step denoising to the final samples. + eps: A `float` number. The reverse-time SDE and ODE are integrated to `epsilon` to avoid numerical issues. + device: PyTorch device. + + Returns: + A sampling function that returns samples and the number of function evaluations during sampling. + """ + # Create predictor & corrector update functions + predictor_update_fn = functools.partial(shared_predictor_update_fn, + sde=sde, + predictor=predictor, + probability_flow=probability_flow, + continuous=continuous) + corrector_update_fn = functools.partial(shared_corrector_update_fn, + sde=sde, + corrector=corrector, + continuous=continuous, + snr=snr, + n_steps=n_steps) + + def pc_sampler(model, mask): + """The PC sampler function. + + Args: + model: A score model. + n_nodes_pmf: Probability mass function of graph nodes. + + Returns: + Samples, number of function evaluations. + """ + with torch.no_grad(): + # Initial sample + x = sde.prior_sampling(shape).to(device) + timesteps = torch.linspace(sde.T, eps, sde.N, device=device) + + # Sample the number of nodes + # n_nodes = torch.multinomial(n_nodes_pmf, shape[0], replacement=True) + # mask = torch.zeros((shape[0], shape[-1]), device=device) + # for i in range(shape[0]): + # mask[i][:n_nodes[i]] = 1. + # mask = (mask[:, None, :] * mask[:, :, None]).unsqueeze(1) + # mask = torch.tril(mask, -1) + # mask = mask + mask.transpose(-1, -2) + # x = x * mask + mask = mask[0].unsqueeze(0).repeat(x.size(0), 1, 1) + + for i in trange(sde.N, desc='[PC sampling]', position=1, leave=False): + t = timesteps[i] + vec_t = torch.ones(shape[0], device=t.device) * t + x, x_mean = corrector_update_fn(x, vec_t, model=model, maskX=mask) + # x = x * mask + x, x_mean = predictor_update_fn(x, vec_t, model=model, maskX=mask) + # x = x * mask + + # return inverse_scaler(x_mean if denoise else x) * mask, sde.N * (n_steps + 1), n_nodes + return inverse_scaler(x_mean if denoise else x), sde.N * (n_steps + 1), None + + return pc_sampler + + +def get_pc_conditional_sampler_nas(sde, shape, + predictor, corrector, inverse_scaler, snr, + n_steps=1, probability_flow=False, + continuous=False, denoise=True, eps=1e-5, device='cuda', + regress=True, labels='max', classifier_scale=0.5, + weight_scheduling=True, weight_ratio=True, t_spot=1., t_spot_end=None, + p=1, prod_w=False, weight_ratio_abs=False): + """Class-conditional sampling with Predictor-Corrector (PC) samplers. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + score_model: A `torch.nn.Module` object that represents the architecture of the score-based model. + classifier: A `torch.nn.Module` object that represents the architecture of the noise-dependent classifier. + # classifier_params: A dictionary that contains the weights of the classifier. + shape: A sequence of integers. The expected shape of a single sample. + predictor: A subclass of `sampling.predictor` that represents a predictor algorithm. + corrector: A subclass of `sampling.corrector` that represents a corrector algorithm. + inverse_scaler: The inverse data normalizer. + snr: A `float` number. The signal-to-noise ratio for correctors. + n_steps: An integer. The number of corrector steps per update of the predictor. + probability_flow: If `True`, solve the probability flow ODE for sampling with the predictor. + continuous: `True` indicates the score-based model was trained with continuous time. + denoise: If `True`, add one-step denoising to final samples. + eps: A `float` number. The SDE/ODE will be integrated to `eps` to avoid numerical issues. + + Returns: A pmapped class-conditional image sampler. + """ + score_grad_norm_p, classifier_grad_norm_p = [], [] + score_grad_norm_c, classifier_grad_norm_c = [], [] + if t_spot_end is None or t_spot_end == 0.: + t_spot_end = eps + + def weight_scheduling_fn(w, t): + return w * 0.1 ** t + + def conditional_predictor_update_fn(score_model, classifier, x, t, labels, maskX, *args, **kwargs): + """The predictor update function for class-conditional sampling.""" + score_fn = mutils.get_score_fn(sde, score_model, train=False, continuous=continuous) + # The gradient function of the noise-dependent classifier + classifier_grad_fn = mutils.get_classifier_grad_fn(sde, classifier, train=False, continuous=continuous, + regress=regress, labels=labels) + + def total_grad_fn(x, t, *args, **kwargs): + + # score = score_fn(x, t, *args, **kwargs) + score = score_fn(x, t, maskX) + classifier_grad = classifier_grad_fn(x, t, maskX, *args, **kwargs) + + # Sample weight + if weight_scheduling: + w = weight_scheduling_fn(classifier_scale, t[0].item()) + else: + w = classifier_scale + + if weight_ratio: + if prod_w: + ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / (w * classifier_grad).view(x.shape[0], -1).norm(p=p, dim=-1) + else: + ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) + # ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) + w *= ratio[:, None, None] + + if weight_ratio_abs: + assert not weight_ratio + ratio = torch.div(torch.abs(score), torch.abs(classifier_grad)) + w *= ratio + + score_grad_norm_p.append(torch.mean(score.view(x.shape[0], -1).norm(p=p, dim=-1)).item()) + + if weight_ratio: # ratio per sample + classifier_grad_norm_p.append(torch.mean(classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) * ratio[:, None, None]).item()) + elif weight_ratio_abs: # ratio per element + classifier_grad_norm_p.append(torch.mean((classifier_grad * ratio).norm(p=p)).item()) + else: + classifier_grad_norm_p.append(torch.mean(classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1)).item()) + + if t_spot < 1.: + if t[0].item() <= t_spot and t[0] >= t_spot_end: + return score + w * classifier_grad + # return (1 - w) * score + w * classifier_grad + else: + return score + else: + # return (1 - w) * score + w * classifier_grad + return score + w * classifier_grad + + if predictor is None: + predictor_obj = NonePredictor(sde, total_grad_fn, probability_flow) + else: + predictor_obj = predictor(sde, total_grad_fn, probability_flow) + return predictor_obj.update_fn(x, t, *args, **kwargs) + + def conditional_corrector_update_fn(score_model, classifier, x, t, labels, maskX, *args, **kwargs): + """The corrector update function for class-conditional sampling.""" + score_fn = mutils.get_score_fn(sde, score_model, train=False, continuous=continuous) + classifier_grad_fn = mutils.get_classifier_grad_fn(sde, classifier, train=False, continuous=continuous, + regress=regress, labels=labels) + + def total_grad_fn(x, t, *args, **kwargs): + # score = score_fn(x, t, *args, **kwargs) + score = score_fn(x, t, maskX) + classifier_grad = classifier_grad_fn(x, t, maskX, *args, **kwargs) + + # Sample weight + if weight_scheduling: + w = weight_scheduling_fn(classifier_scale, t[0].item()) + else: + w = classifier_scale + + if weight_ratio: + if prod_w: + ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / (w * classifier_grad).view(x.shape[0], -1).norm(p=p, dim=-1) + else: + ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) + # ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) + w *= ratio[:, None, None] + + score_grad_norm_c.append(torch.mean(score.view(x.shape[0], -1).norm(p=p, dim=-1)).item()) + + if weight_ratio: + classifier_grad_norm_c.append(torch.mean(classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) * ratio[:, None, None]).item()) + else: + classifier_grad_norm_c.append(torch.mean(classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1)).item()) + + if t_spot < 1.: + if t[0].item() <= t_spot and t[0] >= t_spot_end: + return score + w * classifier_grad + # return (1 - w) * score + w * classifier_grad + else: + return score + else: + return score + w * classifier_grad + # return (1 - w) * score + w * classifier_grad + + if corrector is None: + corrector_obj = NoneCorrector(sde, total_grad_fn, snr, n_steps) + else: + corrector_obj = corrector(sde, total_grad_fn, snr, n_steps) + return corrector_obj.update_fn(x, t, *args, **kwargs) + + def pc_conditional_sampler(score_model, mask, classifier, + eval_chain=False, keep_chain=None, number_chain_steps=None): + """Generate class-conditional samples with Predictor-Corrector (PC) samplers. + + Args: + score_model: A `torch.nn.Module` object that represents the training state + of the score-based model. + labels: A JAX array of integers that represent the target label of each sample. + + Returns: + Class-conditional samples. + """ + chain_x = None + if eval_chain: + if number_chain_steps is None: + number_chain_steps = sde.N + if keep_chain is None: + keep_chain = shape[0] # all sample + assert number_chain_steps <= sde.N + chain_x_size = torch.Size((number_chain_steps, keep_chain, *shape[1:])) + chain_x = torch.zeros(chain_x_size) + + with torch.no_grad(): + # Initial sample + x = sde.prior_sampling(shape).to(device) + timesteps = torch.linspace(sde.T, eps, sde.N, device=device) + + if len(mask.shape) == 3: + mask = mask[0] + mask = mask.unsqueeze(0).repeat(x.size(0), 1, 1) # adj + + for i in trange(sde.N, desc='[PC conditional sampling]', position=1, leave=False): + t = timesteps[i] + vec_t = torch.ones(shape[0], device=t.device) * t + # x, x_mean = conditional_corrector_update_fn(x, vec_t, model=model, maskX=mask) + x, x_mean = conditional_corrector_update_fn(score_model, classifier, x, vec_t, labels=labels, maskX=mask) + # x = x * mask + x, x_mean = conditional_predictor_update_fn(score_model, classifier, x, vec_t, labels=labels, maskX=mask) + # x = x * mask + + if eval_chain: + # arch_metric = sampling_metrics(arch_list=inverse_scaler(x_mean if denoise else x), + # adj=adj, mask=mask, + # this_sample_dir=os.path.join(sampling_metrics.exp_name), + # test=False) + # r_valid, r_unique, r_novel = arch_metric[0][0], arch_metric[0][1], arch_metric[0][2] + # Save the first keep_chain graphs + write_index = number_chain_steps - 1 - int((i * number_chain_steps) // sde.N) + # write_index = int((t * number_chain_steps) // sde.T) + chain_x[write_index] = inverse_scaler(x_mean if denoise else x)[:keep_chain] + + # Overwrite last frame with the resulting x + # if keep_chain > 0: + # final_x_chain = inverse_scaler(x_mean if denoise else x)[:keep_chain] + # chain_x[0] = final_x_chain + # # Repeat last frame to see final sample better + # import pdb; pdb.set_trace() + # chain_x = torch.cat([chain_x, chain_x[-1:].repeat(10, 1, 1)], dim=0) + # import pdb; pdb.set_trace() + # assert chain_x.size(0) == (number_chain_steps + 10) + + return inverse_scaler(x_mean if denoise else x), sde.N * (n_steps + 1), chain_x, (score_grad_norm_p, classifier_grad_norm_p, score_grad_norm_c, classifier_grad_norm_c) + + return pc_conditional_sampler + + +def get_pc_conditional_sampler_meta_nas(sde, shape, + predictor, corrector, inverse_scaler, snr, + n_steps=1, probability_flow=False, + continuous=False, denoise=True, eps=1e-5, device='cuda', + regress=True, labels='max', classifier_scale=0.5, + weight_scheduling=True, weight_ratio=True, t_spot=1., t_spot_end=None, + p=1, prod_w=False, weight_ratio_abs=False, + data_name='cifar10', num_sample=20, search_space=None): + """Class-conditional sampling with Predictor-Corrector (PC) samplers. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + score_model: A `torch.nn.Module` object that represents the architecture of the score-based model. + classifier: A `torch.nn.Module` object that represents the architecture of the noise-dependent classifier. + # classifier_params: A dictionary that contains the weights of the classifier. + shape: A sequence of integers. The expected shape of a single sample. + predictor: A subclass of `sampling.predictor` that represents a predictor algorithm. + corrector: A subclass of `sampling.corrector` that represents a corrector algorithm. + inverse_scaler: The inverse data normalizer. + snr: A `float` number. The signal-to-noise ratio for correctors. + n_steps: An integer. The number of corrector steps per update of the predictor. + probability_flow: If `True`, solve the probability flow ODE for sampling with the predictor. + continuous: `True` indicates the score-based model was trained with continuous time. + denoise: If `True`, add one-step denoising to final samples. + eps: A `float` number. The SDE/ODE will be integrated to `eps` to avoid numerical issues. + + Returns: A pmapped class-conditional image sampler. + """ + + # --------- Meta-NAS (START) ---------- # + test_dataset = MetaTestDataset( + data_path=PROCESSED_DATA_PATH, + data_name=data_name, + num_sample=num_sample + ) + # --------- Meta-NAS (END) ---------- # + + score_grad_norm_p, classifier_grad_norm_p = [], [] + score_grad_norm_c, classifier_grad_norm_c = [], [] + if t_spot_end is None or t_spot_end == 0.: + t_spot_end = eps + + def weight_scheduling_fn(w, t): + return w * 0.1 ** t + + def conditional_predictor_update_fn(score_model, classifier, x, t, labels, maskX, classifier_scale, *args, **kwargs): + """The predictor update function for class-conditional sampling.""" + score_fn = mutils.get_score_fn(sde, score_model, train=False, continuous=continuous) + # The gradient function of the noise-dependent classifier + classifier_grad_fn = mutils.get_classifier_grad_fn(sde, classifier, train=False, continuous=continuous, + regress=regress, labels=labels) + + def total_grad_fn(x, t, *args, **kwargs): + + # score = score_fn(x, t, *args, **kwargs) + score = score_fn(x, t, maskX) + classifier_grad = classifier_grad_fn(x, t, maskX, *args, **kwargs) + + # Sample weight + if weight_scheduling: + w = weight_scheduling_fn(classifier_scale, t[0].item()) + else: + w = classifier_scale + + if weight_ratio: + if prod_w: + ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / (w * classifier_grad).view(x.shape[0], -1).norm(p=p, dim=-1) + else: + ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) + # ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) + w *= ratio[:, None, None] + + if weight_ratio_abs: + assert not weight_ratio + ratio = torch.div(torch.abs(score), torch.abs(classifier_grad)) + w *= ratio + + score_grad_norm_p.append(torch.mean(score.view(x.shape[0], -1).norm(p=p, dim=-1)).item()) + + if weight_ratio: # ratio per sample + classifier_grad_norm_p.append(torch.mean(classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) * ratio[:, None, None]).item()) + elif weight_ratio_abs: # ratio per element + classifier_grad_norm_p.append(torch.mean((classifier_grad * ratio).norm(p=p)).item()) + else: + classifier_grad_norm_p.append(torch.mean(classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1)).item()) + + if t_spot < 1.: + if t[0].item() <= t_spot and t[0] >= t_spot_end: + return score + w * classifier_grad + # return (1 - w) * score + w * classifier_grad + else: + return score + else: + # return (1 - w) * score + w * classifier_grad + return score + w * classifier_grad + + if predictor is None: + predictor_obj = NonePredictor(sde, total_grad_fn, probability_flow) + else: + predictor_obj = predictor(sde, total_grad_fn, probability_flow) + return predictor_obj.update_fn(x, t, *args, **kwargs) + + def conditional_corrector_update_fn(score_model, classifier, x, t, labels, maskX, classifier_scale, *args, **kwargs): + """The corrector update function for class-conditional sampling.""" + score_fn = mutils.get_score_fn(sde, score_model, train=False, continuous=continuous) + classifier_grad_fn = mutils.get_classifier_grad_fn(sde, classifier, train=False, continuous=continuous, + regress=regress, labels=labels) + + def total_grad_fn(x, t, *args, **kwargs): + # score = score_fn(x, t, *args, **kwargs) + score = score_fn(x, t, maskX) + classifier_grad = classifier_grad_fn(x, t, maskX, *args, **kwargs) + + # Sample weight + if weight_scheduling: + w = weight_scheduling_fn(classifier_scale, t[0].item()) + else: + w = classifier_scale + + if weight_ratio: + if prod_w: + ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / (w * classifier_grad).view(x.shape[0], -1).norm(p=p, dim=-1) + else: + ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) + # ratio = score.view(x.shape[0], -1).norm(p=p, dim=-1) / classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) + w *= ratio[:, None, None] + + score_grad_norm_c.append(torch.mean(score.view(x.shape[0], -1).norm(p=p, dim=-1)).item()) + + if weight_ratio: + classifier_grad_norm_c.append(torch.mean(classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1) * ratio[:, None, None]).item()) + else: + classifier_grad_norm_c.append(torch.mean(classifier_grad.view(x.shape[0], -1).norm(p=p, dim=-1)).item()) + + if t_spot < 1.: + if t[0].item() <= t_spot and t[0] >= t_spot_end: + return score + w * classifier_grad + # return (1 - w) * score + w * classifier_grad + else: + return score + else: + return score + w * classifier_grad + # return (1 - w) * score + w * classifier_grad + if corrector is None: + corrector_obj = NoneCorrector(sde, total_grad_fn, snr, n_steps) + else: + corrector_obj = corrector(sde, total_grad_fn, snr, n_steps) + return corrector_obj.update_fn(x, t, *args, **kwargs) + + def pc_conditional_sampler(score_model, mask, classifier, + eval_chain=False, keep_chain=None, + number_chain_steps=None, classifier_scale=None, + task=None, sample_bs=None): + """Generate class-conditional samples with Predictor-Corrector (PC) samplers. + + Args: + score_model: A `torch.nn.Module` object that represents the training state + of the score-based model. + labels: A JAX array of integers that represent the target label of each sample. + + Returns: + Class-conditional samples. + """ + + chain_x = None + if eval_chain: + if number_chain_steps is None: + number_chain_steps = sde.N + if keep_chain is None: + keep_chain = shape[0] # all sample + assert number_chain_steps <= sde.N + chain_x_size = torch.Size((number_chain_steps, keep_chain, *shape[1:])) + chain_x = torch.zeros(chain_x_size) + + with torch.no_grad(): + + # ----------- Meta-NAS (START) ---------- # + # different task embedding in a batch + # task_batch = [] + # for _ in range(shape[0]): + # task_batch.append(test_dataset[0]) + # task = torch.stack(task_batch, dim=0) + + if task is None: + # same task embedding in a batch + task = test_dataset[0] + task = task.repeat(shape[0], 1, 1) + task = task.to(device) + else: + task = task.repeat(shape[0], 1, 1) + task = task.to(device) + # print(f'Sampling stage') + # import pdb; pdb.set_trace() + + # for accerlerating sampling + classifier.sample_state = True + classifier.D_mu = None + # ----------- Meta-NAS (END) ---------- # + # import pdb; pdb.set_trace() + # Initial sample + x = sde.prior_sampling(shape).to(device) + timesteps = torch.linspace(sde.T, eps, sde.N, device=device) + + if len(mask.shape) == 3: + mask = mask[0] + mask = mask.unsqueeze(0).repeat(x.size(0), 1, 1) # adj + + for i in trange(sde.N, desc='[PC conditional sampling]', position=1, leave=False): + t = timesteps[i] + vec_t = torch.ones(shape[0], device=t.device) * t + + x, x_mean = conditional_corrector_update_fn(score_model, classifier, x, vec_t, labels=labels, maskX=mask, task=task, classifier_scale=classifier_scale) + x, x_mean = conditional_predictor_update_fn(score_model, classifier, x, vec_t, labels=labels, maskX=mask, task=task, classifier_scale=classifier_scale) + + if eval_chain: + # Save the first keep_chain graphs + write_index = number_chain_steps - 1 - int((i * number_chain_steps) // sde.N) + # write_index = int((t * number_chain_steps) // sde.T) + chain_x[write_index] = inverse_scaler(x_mean if denoise else x)[:keep_chain] + + classifier.sample_state = False + return inverse_scaler(x_mean if denoise else x), sde.N * (n_steps + 1), chain_x, (score_grad_norm_p, classifier_grad_norm_p, score_grad_norm_c, classifier_grad_norm_c) + + return pc_conditional_sampler + + + +def get_ode_sampler(sde, shape, inverse_scaler, denoise=False, + rtol=1e-5, atol=1e-5, method='RK45', eps=1e-3, device='cuda'): + """Probability flow ODE sampler with the black-box ODE solver. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + shape: A sequence of integers. The expected shape of a single sample. + inverse_scaler: The inverse data normalizer. + denoise: If `True`, add one-step denoising to final samples. + rtol: A `float` number. The relative tolerance level of the ODE solver. + atol: A `float` number. The absolute tolerance level of the ODE solver. + method: A `str`. The algorithm used for the black-box ODE solver. + See the documentation of `scipy.integrate.solve_ivp`. + eps: A `float` number. The reverse-time SDE/ODE will be integrated to `eps` for numerical stability. + device: PyTorch device. + + Returns: + A sampling function that returns samples and the number of function evaluations during sampling. + """ + + def denoise_update_fn(model, x, mask): + score_fn = get_score_fn(sde, model, train=False, continuous=True) + # Reverse diffusion predictor for denoising + predictor_obj = ReverseDiffusionPredictor(sde, score_fn, probability_flow=False) + vec_eps = torch.ones(x.shape[0], device=x.device) * eps + _, x = predictor_obj.update_fn(x, vec_eps, mask=mask) + return x + + def drift_fn(model, x, t, mask): + """Get the drift function of the reverse-time SDE.""" + score_fn = get_score_fn(sde, model, train=False, continuous=True) + rsde = sde.reverse(score_fn, probability_flow=True) + return rsde.sde(x, t, mask=mask)[0] + + def ode_sampler(model, n_nodes_pmf, z=None): + """The probability flow ODE sampler with black-box ODE solver. + + Args: + model: A score model. + n_nodes_pmf: Probability mass function of graph nodes. + z: If present, generate samples from latent code `z`. + Returns: + samples, number of function evaluations. + """ + with torch.no_grad(): + # Initial sample + if z is None: + # If not represent, sample the latent code from the prior distribution of the SDE. + x = sde.prior_sampling(shape).to(device) + else: + x = z + + # Sample the number of nodes + n_nodes = torch.multinomial(n_nodes_pmf, shape[0], replacement=True) + mask = torch.zeros((shape[0], shape[-1]), device=device) + for i in range(shape[0]): + mask[i][:n_nodes[i]] = 1. + mask = (mask[:, None, :] * mask[:, :, None]).unsqueeze(1) + + def ode_func(t, x): + x = from_flattened_numpy(x, shape).to(device).type(torch.float32) + vec_t = torch.ones(shape[0], device=x.device) * t + drift = drift_fn(model, x, vec_t, mask) + return to_flattened_numpy(drift) + + # Black-box ODE solver for the probability flow ODE + solution = integrate.solve_ivp(ode_func, (sde.T, eps), to_flattened_numpy(x), + rtol=rtol, atol=atol, method=method) + nfe = solution.nfev + x = torch.tensor(solution.y[:, -1]).reshape(shape).to(device).type(torch.float32) + + # Denoising is equivalent to running one predictor step without adding noise + if denoise: + x = denoise_update_fn(model, x, mask) + + x = inverse_scaler(x) * mask + return x, nfe, n_nodes + + return ode_sampler + + +def get_diffeq_sampler(sde, shape, inverse_scaler, denoise=False, + rtol=1e-5, atol=1e-5, step_size=0.01, method='dopri5', eps=1e-3, device='cuda'): + """ + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + shape: A sequence of integers. The expected shape of a single sample. + inverse_scaler: The inverse data normalizer. + denoise: If `True`, add one-step denoising to final samples. + rtol: A `float` number. The relative tolerance level of the ODE solver. + atol: A `float` number. The absolute tolerance level of the ODE solver. + method: A `str`. The algorithm used for the black-box ODE solver in torchdiffeq. + See the documentation of `torchdiffeq`. eg: adaptive solver('dopri5', 'bosh3', 'fehlberg2') + eps: A `float` number. The reverse-time SDE/ODE will be integrated to `eps` for numerical stability. + device: PyTorch device. + + Returns: + A sampling function that returns samples and the number of function evaluations during sampling. + """ + + def denoise_update_fn(model, x, mask): + score_fn = get_score_fn(sde, model, train=False, continuous=True) + # Reverse diffusion predictor for denoising + predictor_obj = ReverseDiffusionPredictor(sde, score_fn, probability_flow=False) + vec_eps = torch.ones(x.shape[0], device=x.device) * eps + _, x = predictor_obj.update_fn(x, vec_eps, mask=mask) + return x + + def drift_fn(model, x, t, mask): + """Get the drift function of the reverse-time SDE.""" + score_fn = get_score_fn(sde, model, train=False, continuous=True) + rsde = sde.reverse(score_fn, probability_flow=True) + return rsde.sde(x, t, mask=mask)[0] + + def diffeq_sampler(model, n_nodes_pmf, z=None): + """The probability flow ODE sampler with ODE solver from torchdiffeq. + + Args: + model: A score model. + n_nodes_pmf: Probability mass function of graph nodes. + z: If present, generate samples from latent code `z`. + Returns: + samples, number of function evaluations. + """ + with torch.no_grad(): + # initial sample + if z is None: + # If not represent, sample the latent code from the prior distribution of the SDE. + x = sde.prior_sampling(shape).to(device) + else: + x = z + + # Sample the number of nodes + n_nodes = torch.multinomial(n_nodes_pmf, shape[0], replacement=True) + mask = torch.zeros((shape[0], shape[-1]), device=device) + for i in range(shape[0]): + mask[i][:n_nodes[i]] = 1. + mask = (mask[:, None, :] * mask[:, :, None]).unsqueeze(1) + + class ODEfunc(torch.nn.Module): + def __init__(self): + super(ODEfunc, self).__init__() + self.nfe = 0 + + def forward(self, t, x): + self.nfe += 1 + x = x.reshape(shape) + vec_t = torch.ones(shape[0], device=x.device) * t + drift = drift_fn(model, x, vec_t, mask) + return drift.reshape((-1,)) + + # Black-box ODE solver for the probability flow ODE + ode_func = ODEfunc() + if method in ['dopri5', 'bosh3', 'fehlberg2']: + solution = odeint(ode_func, x.reshape((-1,)), torch.tensor([sde.T, eps], device=x.device), + rtol=rtol, atol=atol, method=method, + options={'step_t': torch.tensor([1e-3], device=x.device)}) + elif method in ['euler', 'midpoint', 'rk4', 'explicit_adams', 'implicit_adams']: + solution = odeint(ode_func, x.reshape((-1,)), torch.tensor([sde.T, eps], device=x.device), + rtol=rtol, atol=atol, method=method, + options={'step_size': step_size}) + + x = solution[-1, :].reshape(shape) + + # Denoising is equivalent to running one predictor step without adding noise + if denoise: + x = denoise_update_fn(model, x, mask) + + x = inverse_scaler(x) * mask + return x, ode_func.nfe, n_nodes + + return diffeq_sampler diff --git a/MobileNetV3/script/download_preprocessed_dataset.sh b/MobileNetV3/script/download_preprocessed_dataset.sh new file mode 100644 index 0000000..9b378af --- /dev/null +++ b/MobileNetV3/script/download_preprocessed_dataset.sh @@ -0,0 +1,3 @@ +echo '[Downloading processed]' +python main_exp/get_files/get_preprocessed_data.py +python main_exp/get_files/get_preprocessed_score_model_data.py \ No newline at end of file diff --git a/MobileNetV3/script/download_raw_dataset.sh b/MobileNetV3/script/download_raw_dataset.sh new file mode 100644 index 0000000..e95a821 --- /dev/null +++ b/MobileNetV3/script/download_raw_dataset.sh @@ -0,0 +1,13 @@ +DATANAME=$1 + +if [[ $DATANAME = 'aircraft' ]]; then + echo '[Downloading aircraft]' + python main_exp/get_files/get_aircraft.py + +elif [[ $DATANAME = 'pets' ]]; then + echo '[Downloading pets]' + python main_exp/get_files/get_pets.py + +else + echo 'Not Implemeted' +fi \ No newline at end of file diff --git a/MobileNetV3/script/tr_meta_surrogate_ofa.sh b/MobileNetV3/script/tr_meta_surrogate_ofa.sh new file mode 100644 index 0000000..79e5547 --- /dev/null +++ b/MobileNetV3/script/tr_meta_surrogate_ofa.sh @@ -0,0 +1,5 @@ +FOLDER_NAME='tr_meta_surrogate_ofa' + +CUDA_VISIBLE_DEVICES=$1 python main.py --config configs/tr_meta_surrogate_ofa.py \ + --mode train \ + --config.folder_name $FOLDER_NAME diff --git a/MobileNetV3/script/tr_scorenet_ofa.sh b/MobileNetV3/script/tr_scorenet_ofa.sh new file mode 100644 index 0000000..cc05985 --- /dev/null +++ b/MobileNetV3/script/tr_scorenet_ofa.sh @@ -0,0 +1,5 @@ +FOLDER_NAME='tr_scorenet_ofa' + +CUDA_VISIBLE_DEVICES=$1 python main.py --config configs/tr_scorenet_ofa.py \ + --mode train \ + --config.folder_name $FOLDER_NAME diff --git a/MobileNetV3/script/transfer_nag.sh b/MobileNetV3/script/transfer_nag.sh new file mode 100644 index 0000000..1112308 --- /dev/null +++ b/MobileNetV3/script/transfer_nag.sh @@ -0,0 +1,14 @@ +FOLDER_NAME='transfer_nag_ofa' + +N=30 +GENSAMPLES=5000 +GPU=$1 +DATANAME=$2 + + +CUDA_VISIBLE_DEVICES=$GPU python main_exp/run_transfer_nag.py \ + --test --data-name $DATANAME --gpu $GPU \ + --folder_name $FOLDER_NAME \ + --nvt 27 --search_space ofa --graph-data-name ofa \ + --epochs 500 --n_gen_samples $GENSAMPLES --classifier_scale 500 \ + --n_training_samples $N \ No newline at end of file diff --git a/MobileNetV3/sde_lib.py b/MobileNetV3/sde_lib.py new file mode 100644 index 0000000..2d3bb07 --- /dev/null +++ b/MobileNetV3/sde_lib.py @@ -0,0 +1,332 @@ +"""Abstract SDE classes, Reverse SDE, and VP SDEs.""" + +import abc +import torch +import numpy as np + + +class SDE(abc.ABC): + """SDE abstract class. Functions are designed for a mini-batch of inputs.""" + + def __init__(self, N): + """Construct an SDE. + + Args: + N: number of discretization time steps. + """ + super().__init__() + self.N = N + + @property + @abc.abstractmethod + def T(self): + """End time of the SDE.""" + pass + + @abc.abstractmethod + def sde(self, x, t): + pass + + @abc.abstractmethod + def marginal_prob(self, x, t): + """Parameters to determine the marginal distribution of the SDE, $p_t(x)$""" + pass + + @abc.abstractmethod + def prior_sampling(self, shape): + """Generate one sample from the prior distribution, $p_T(x)$.""" + pass + + @abc.abstractmethod + def prior_logp(self, z, mask): + """Compute log-density of the prior distribution. + + Useful for computing the log-likelihood via probability flow ODE. + + Args: + z: latent code + Returns: + log probability density + """ + pass + + def discretize(self, x, t): + """Discretize the SDE in the form: x_{i+1} = x_i + f_i(x_i) + G_i z_i. + + Useful for reverse diffusion sampling and probability flow sampling. + Defaults to Euler-Maruyama discretization. + + Args: + x: a torch tensor + t: a torch float representing the time step (from 0 to `self.T`) + + Returns: + f, G + """ + dt = 1 / self.N + drift, diffusion = self.sde(x, t) + f = drift * dt + G = diffusion * torch.sqrt(torch.tensor(dt, device=t.device)) + return f, G + + def reverse(self, score_fn, probability_flow=False): + """Create the reverse-time SDE/ODE. + + Args: + score_fn: A time-dependent score-based model that takes x and t and returns the score. + probability_flow: If `True`, create the reverse-time ODE used for probability flow sampling. + """ + + N = self.N + T = self.T + sde_fn = self.sde + discretize_fn = self.discretize + + # Build the class for reverse-time SDE. + class RSDE(self.__class__): + def __init__(self): + self.N = N + self.probability_flow = probability_flow + + @property + def T(self): + return T + + def sde(self, x, t, *args, **kwargs): + """Create the drift and diffusion functions for the reverse SDE/ODE.""" + + drift, diffusion = sde_fn(x, t) + score = score_fn(x, t, *args, **kwargs) + drift = drift - diffusion[:, None, None] ** 2 * score * (0.5 if self.probability_flow else 1.) + # Set the diffusion function to zero for ODEs. + diffusion = 0. if self.probability_flow else diffusion + return drift, diffusion + + ''' + def sde_score(self, x, t, score): + """Create the drift and diffusion functions for the reverse SDE/ODE, given score values.""" + drift, diffusion = sde_fn(x, t) + if len(score.shape) == 4: + drift = drift - diffusion[:, None, None, None] ** 2 * score * (0.5 if self.probability_flow else 1.) + elif len(score.shape) == 3: + drift = drift - diffusion[:, None, None] ** 2 * score * (0.5 if self.probability_flow else 1.) + else: + raise ValueError + diffusion = 0. if self.probability_flow else diffusion + return drift, diffusion + ''' + + def discretize(self, x, t, *args, **kwargs): + """Create discretized iteration rules for the reverse diffusion sampler.""" + f, G = discretize_fn(x, t) + rev_f = f - G[:, None, None] ** 2 * score_fn(x, t, *args, **kwargs) * \ + (0.5 if self.probability_flow else 1.) + rev_G = torch.zeros_like(G) if self.probability_flow else G + return rev_f, rev_G + + ''' + def discretize_score(self, x, t, score): + """Create discretized iteration rules for the reverse diffusion sampler, given score values.""" + f, G = discretize_fn(x, t) + if len(score.shape) == 4: + rev_f = f - G[:, None, None, None] ** 2 * score * \ + (0.5 if self.probability_flow else 1.) + elif len(score.shape) == 3: + rev_f = f - G[:, None, None] ** 2 * score * (0.5 if self.probability_flow else 1.) + else: + raise ValueError + rev_G = torch.zeros_like(G) if self.probability_flow else G + return rev_f, rev_G + ''' + + return RSDE() + + +class VPSDE(SDE): + def __init__(self, beta_min=0.1, beta_max=20, N=1000): + """Construct a Variance Preserving SDE. + + Args: + beta_min: value of beta(0) + beta_max: value of beta(1) + N: number of discretization steps + """ + super().__init__(N) + self.beta_0 = beta_min + self.beta_1 = beta_max + self.N = N + self.discrete_betas = torch.linspace(beta_min / N, beta_max / N, N) + self.alphas = 1. - self.discrete_betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_1m_alphas_cumprod = torch.sqrt(1. - self.alphas_cumprod) + + @property + def T(self): + return 1 + + def sde(self, x, t): + beta_t = self.beta_0 + t * (self.beta_1 - self.beta_0) + if len(x.shape) == 4: + drift = -0.5 * beta_t[:, None, None, None] * x + elif len(x.shape) == 3: + drift = -0.5 * beta_t[:, None, None] * x + else: + raise NotImplementedError + diffusion = torch.sqrt(beta_t) + return drift, diffusion + + def marginal_prob(self, x, t): + log_mean_coeff = -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0 + if len(x.shape) == 4: + mean = torch.exp(log_mean_coeff[:, None, None, None]) * x + elif len(x.shape) == 3: + mean = torch.exp(log_mean_coeff[:, None, None]) * x + else: + raise ValueError("The shape of x in marginal_prob is not correct.") + std = torch.sqrt(1. - torch.exp(2. * log_mean_coeff)) + return mean, std + + # def log_snr(self, t): + # log_mean_coeff = -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0 + # mean = torch.exp(log_mean_coeff) + # std = torch.sqrt(1. - torch.exp(2. * log_mean_coeff)) + # log_snr = torch.log(mean / std) + # return log_snr, mean, std + # + # def log_snr_np(self, t): + # log_mean_coeff = -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0 + # mean = np.exp(log_mean_coeff) + # std = np.sqrt(1. - np.exp(2. * log_mean_coeff)) + # log_snr = np.log(mean / std) + # return log_snr + # + # def lambda2t(self, lambda_ori): + # log_val = torch.log(torch.exp(-2. * lambda_ori) + 1.) + # t = 2. * log_val / (torch.sqrt(self.beta_0 ** 2 + 2. * (self.beta_1 - self.beta_0) * log_val) + self.beta_0) + # return t + # + # def lambda2t_np(self, lambda_ori): + # log_val = np.log(np.exp(-2. * lambda_ori) + 1.) + # t = 2. * log_val / (np.sqrt(self.beta_0 ** 2 + 2. * (self.beta_1 - self.beta_0) * log_val) + self.beta_0) + # return t + + # def prior_sampling(self, shape): + # sample = torch.randn(*shape) + # if len(shape) == 4: + # sample = torch.tril(sample, -1) + # sample = sample + sample.transpose(-1, -2) + + # return sample + + def prior_sampling(self, shape): + return torch.randn(*shape) + + def prior_logp(self, z, mask): + N = torch.sum(mask, dim=tuple(range(1, len(mask.shape)))) + logps = -N / 2. * np.log(2 * np.pi) - torch.sum((z * mask) ** 2, dim=(1, 2, 3)) / 2. + return logps + + def discretize(self, x, t): + """DDPM discretization.""" + timestep = (t * (self.N - 1) / self.T).long() + beta = self.discrete_betas.to(x.device)[timestep] + alpha = self.alphas.to(x.device)[timestep] + sqrt_beta = torch.sqrt(beta) + if len(x.shape) == 4: + f = torch.sqrt(alpha)[:, None, None, None] * x - x + elif len(x.shape) == 3: + f = torch.sqrt(alpha)[:, None, None] * x - x + else: + NotImplementedError + G = sqrt_beta + return f, G + + +class subVPSDE(SDE): + def __init__(self, beta_min=0.1, beta_max=20, N=1000): + """Construct the sub-VP SDE that excels at likelihoods. + Args: + beta_min: value of beta(0) + beta_max: value of beta(1) + N: number of discretization steps + """ + super().__init__(N) + self.beta_0 = beta_min + self.beta_1 = beta_max + self.N = N + + @property + def T(self): + return 1 + + def sde(self, x, t): + beta_t = self.beta_0 + t * (self.beta_1 - self.beta_0) + drift = -0.5 * beta_t[:, None, None] * x + discount = 1. - torch.exp(-2 * self.beta_0 * t - (self.beta_1 - self.beta_0) * t ** 2) + diffusion = torch.sqrt(beta_t * discount) + return drift, diffusion + + def marginal_prob(self, x, t): + log_mean_coeff = -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0 + mean = torch.exp(log_mean_coeff)[:, None, None] * x + std = 1 - torch.exp(2. * log_mean_coeff) + return mean, std + + def prior_sampling(self, shape): + return torch.randn(*shape) + + def prior_logp(self, z): + shape = z.shape + N = np.prod(shape[1:]) + return -N / 2. * np.log(2 * np.pi) - torch.sum(z ** 2, dim=(1, 2, 3)) / 2. + + +class VESDE(SDE): + def __init__(self, sigma_min=0.01, sigma_max=50, N=1000): + """Construct a Variance Exploding SDE. + + Args: + sigma_min: smallest sigma. + sigma_max: largest sigma. + N: number of discretization steps + """ + super().__init__(N) + self.sigma_min = sigma_min + self.sigma_max = sigma_max + self.discrete_sigmas = torch.exp(torch.linspace(np.log(self.sigma_min), np.log(self.sigma_max), N)) + self.N = N + + @property + def T(self): + return 1 + + def sde(self, x, t): + sigma = self.sigma_min * (self.sigma_max / self.sigma_min) ** t + drift = torch.zeros_like(x) + diffusion = sigma * torch.sqrt(torch.tensor(2 * (np.log(self.sigma_max) - np.log(self.sigma_min)), + device=t.device)) + return drift, diffusion + + def marginal_prob(self, x, t): + std = self.sigma_min * (self.sigma_max / self.sigma_min) ** t + mean = x + return mean, std + + def prior_sampling(self, shape): + return torch.randn(*shape) * self.sigma_max + + def prior_logp(self, z): + shape = z.shape + N = np.prod(shape[1:]) + return -N / 2. * np.log(2 * np.pi * self.sigma_max ** 2) - torch.sum(z ** 2, dim=(1, 2, 3)) / (2 * self.sigma_max ** 2) + + def discretize(self, x, t): + """SMLD(NCSN) discretization.""" + timestep = (t * (self.N - 1) / self.T).long() + sigma = self.discrete_sigmas.to(t.device)[timestep] + adjacent_sigma = torch.where(timestep == 0, torch.zeros_like(t), + self.discrete_sigmas[timestep.cpu() - 1].to(t.device)) + f = torch.zeros_like(x) + G = torch.sqrt(sigma ** 2 - adjacent_sigma ** 2) + return f, G \ No newline at end of file diff --git a/MobileNetV3/utils.py b/MobileNetV3/utils.py new file mode 100644 index 0000000..a74791d --- /dev/null +++ b/MobileNetV3/utils.py @@ -0,0 +1,270 @@ +import os +import logging +import torch +from torch_scatter import scatter +import shutil + +@torch.no_grad() +def to_dense_adj(edge_index, batch=None, edge_attr=None, max_num_nodes=None): + """Converts batched sparse adjacency matrices given by edge indices and + edge attributes to a single dense batched adjacency matrix. + + Args: + edge_index (LongTensor): The edge indices. + batch (LongTensor, optional): Batch vector + :math:`\mathbf{b} \in {\{ 0, \ldots, B-1\}}^N`, which assigns each + node to a specific example. (default: :obj:`None`) + edge_attr (Tensor, optional): Edge weights or multi-dimensional edge + features. (default: :obj:`None`) + max_num_nodes (int, optional): The size of the output node dimension. + (default: :obj:`None`) + + Returns: + adj: [batch_size, max_num_nodes, max_num_nodes] Dense adjacency matrices. + mask: Mask for dense adjacency matrices. + """ + if batch is None: + batch = edge_index.new_zeros(edge_index.max().item() + 1) + + batch_size = batch.max().item() + 1 + one = batch.new_ones(batch.size(0)) + num_nodes = scatter(one, batch, dim=0, dim_size=batch_size, reduce='add') + cum_nodes = torch.cat([batch.new_zeros(1), num_nodes.cumsum(dim=0)]) + + idx0 = batch[edge_index[0]] + idx1 = edge_index[0] - cum_nodes[batch][edge_index[0]] + idx2 = edge_index[1] - cum_nodes[batch][edge_index[1]] + + if max_num_nodes is None: + max_num_nodes = num_nodes.max().item() + + elif idx1.max() >= max_num_nodes or idx2.max() >= max_num_nodes: + mask = (idx1 < max_num_nodes) & (idx2 < max_num_nodes) + idx0 = idx0[mask] + idx1 = idx1[mask] + idx2 = idx2[mask] + edge_attr = None if edge_attr is None else edge_attr[mask] + + if edge_attr is None: + edge_attr = torch.ones(idx0.numel(), device=edge_index.device) + + size = [batch_size, max_num_nodes, max_num_nodes] + size += list(edge_attr.size())[1:] + adj = torch.zeros(size, dtype=edge_attr.dtype, device=edge_index.device) + + flattened_size = batch_size * max_num_nodes * max_num_nodes + adj = adj.view([flattened_size] + list(adj.size())[3:]) + idx = idx0 * max_num_nodes * max_num_nodes + idx1 * max_num_nodes + idx2 + scatter(edge_attr, idx, dim=0, out=adj, reduce='add') + adj = adj.view(size) + + node_idx = torch.arange(batch.size(0), dtype=torch.long, device=edge_index.device) + node_idx = (node_idx - cum_nodes[batch]) + (batch * max_num_nodes) + mask = torch.zeros(batch_size * max_num_nodes, dtype=adj.dtype, device=adj.device) + mask[node_idx] = 1 + mask = mask.view(batch_size, max_num_nodes) + + mask = mask[:, None, :] * mask[:, :, None] + + return adj, mask + + +def restore_checkpoint_partial(model, pretrained_stdict): + model_dict = model.state_dict() + # 1. filter out unnecessary keys + pretrained_dict = {k: v for k, v in pretrained_stdict.items() if k in model_dict} + # 2. overwrite entries in the existing state dict + model_dict.update(pretrained_dict) + # 3. load the new state dict + model.load_state_dict(model_dict) + return model + + +def restore_checkpoint(ckpt_dir, state, device, resume=False): + if not resume: + os.makedirs(os.path.dirname(ckpt_dir), exist_ok=True) + return state + elif not os.path.exists(ckpt_dir): + if not os.path.exists(os.path.dirname(ckpt_dir)): + os.makedirs(os.path.dirname(ckpt_dir)) + logging.warning(f"No checkpoint found at {ckpt_dir}. " + f"Returned the same state as input") + return state + else: + loaded_state = torch.load(ckpt_dir, map_location=device) + for k in state: + if k in ['optimizer', 'model', 'ema']: + state[k].load_state_dict(loaded_state[k]) + else: + state[k] = loaded_state[k] + return state + + +def save_checkpoint(ckpt_dir, state, step, save_step, is_best): + saved_state = {} + for k in state: + if k in ['optimizer', 'model', 'ema']: + saved_state.update({k: state[k].state_dict()}) + else: + saved_state.update({k: state[k]}) + + os.makedirs(ckpt_dir, exist_ok=True) + torch.save(saved_state, os.path.join(ckpt_dir, f'checkpoint_{step}_{save_step}.pth.tar')) + if is_best: + shutil.copy(os.path.join(ckpt_dir, f'checkpoint_{step}_{save_step}.pth.tar'), os.path.join(ckpt_dir, 'model_best.pth.tar')) + # remove the ckpt except is_best state + for ckpt_file in sorted(os.listdir(ckpt_dir)): + if not ckpt_file.startswith('checkpoint'): + continue + if os.path.join(ckpt_dir, ckpt_file) != os.path.join(ckpt_dir, 'model_best.pth.tar'): + os.remove(os.path.join(ckpt_dir, ckpt_file)) + + +def floyed(r): + """ + :param r: a numpy NxN matrix with float 0,1 + :return: a numpy NxN matrix with float 0,1 + """ + # r = np.array(r) + if type(r) == torch.Tensor: + r = r.cpu().numpy() + N = r.shape[0] + # import pdb; pdb.set_trace() + for k in range(N): + for i in range(N): + for j in range(N): + if r[i, k] > 0 and r[k, j] > 0: + r[i, j] = 1 + return r + + + +def aug_mask(adj, algo='long_range', data='NASBench201'): + if len(adj.shape) == 2: + adj = adj.unsqueeze(0) + + if data.lower() in ['nasbench201', 'ofa']: + assert len(adj.shape) == 3 + r = adj[0].clone().detach() + if algo == 'long_range': + mask_i = torch.from_numpy(long_range(r)).float().to(adj.device) + elif algo == 'floyed': + mask_i = torch.from_numpy(floyed(r)).float().to(adj.device) + else: + mask_i = r + masks = [mask_i] * adj.size(0) + return torch.stack(masks) + else: + masks = [] + for r in adj: + if algo == 'long_range': + mask_i = torch.from_numpy(long_range(r)).float().to(adj.device) + elif algo == 'floyed': + mask_i = torch.from_numpy(floyed(r)).float().to(adj.device) + else: + mask_i = r + masks.append(mask_i) + return torch.stack(masks) + + +def long_range(r): + """ + :param r: a numpy NxN matrix with float 0,1 + :return: a numpy NxN matrix with float 0,1 + """ + # r = np.array(r) + if type(r) == torch.Tensor: + r = r.cpu().numpy() + N = r.shape[0] + for j in range(1, N): + col_j = r[:, j][:j] + in_to_j = [i for i, val in enumerate(col_j) if val > 0] + if len(in_to_j) > 0: + for i in in_to_j: + col_i = r[:, i][:i] + in_to_i = [i for i, val in enumerate(col_i) if val > 0] + if len(in_to_i) > 0: + for k in in_to_i: + r[k, j] = 1 + return r + + +def dense_adj(graph_data, max_num_nodes, scaler=None, dequantization=False): + """Convert PyG DataBatch to dense adjacency matrices. + + Args: + graph_data: DataBatch object. + max_num_nodes: The size of the output node dimension. + scaler: Data normalizer. + dequantization: uniform dequantization. + + Returns: + adj: Dense adjacency matrices. + mask: Mask for adjacency matrices. + """ + + adj, adj_mask = to_dense_adj(graph_data.edge_index, graph_data.batch, max_num_nodes=max_num_nodes) # [B, N, N] + # adj: [32, 20, 20] / adj_mask: [32, 20, 20] + if dequantization: + noise = torch.rand_like(adj) + noise = torch.tril(noise, -1) + noise = noise + noise.transpose(1, 2) + adj = (noise + adj) / 2. + adj = scaler(adj[:, None, :, :]) # [32, 1, 20, 20] + # set diag = 0 in adj_mask + adj_mask = torch.tril(adj_mask, -1) # [32, 20, 20] + adj_mask = adj_mask + adj_mask.transpose(1, 2) + + return adj, adj_mask[:, None, :, :] + + +def adj2graph(adj, sample_nodes): + """Covert the PyTorch tensor adjacency matrices to numpy array. + + Args: + adj: [Batch_size, channel, Max_node, Max_node], assume channel=1 + sample_nodes: [Batch_size] + """ + adj_list = [] + # discretization + adj[adj >= 0.5] = 1. + adj[adj < 0.5] = 0. + for i in range(adj.shape[0]): + adj_tmp = adj[i, 0] + # symmetric + adj_tmp = torch.tril(adj_tmp, -1) + adj_tmp = adj_tmp + adj_tmp.transpose(0, 1) + # truncate + adj_tmp = adj_tmp.cpu().numpy()[:sample_nodes[i], :sample_nodes[i]] + adj_list.append(adj_tmp) + + return adj_list + + +def quantize(x, adj, alpha=0.5, qtype='threshold'): + """Covert the PyTorch tensor x, adj matrices to numpy array. + + Args: + x: [Batch_size, Max_node, N_vocab] + adj: [Batch_size, Max_node, Max_node] + """ + x_list = [] + if qtype == 'threshold': + # discretization + x[x >= alpha] = 1. + x[x < alpha] = 0. + # adj = adj[0] + for i in range(x.shape[0]): + x_tmp = x[i] + x_tmp = x_tmp.cpu().numpy() + x_list.append(x_tmp) + + elif qtype == 'argmax': + am = torch.argmax(x, dim=2, keepdim=True) # [Batch_size, Max_node] + # gather = torch.gather(x, 2, am) + x = torch.zeros_like(x).scatter(2, am, value=1) + for i in range(x.shape[0]): + x_tmp = x[i] + x_tmp = x_tmp.cpu().numpy() + x_list.append(x_tmp) + return x_list \ No newline at end of file diff --git a/NAS-Bench-201/all_path.py b/NAS-Bench-201/all_path.py new file mode 100644 index 0000000..e657618 --- /dev/null +++ b/NAS-Bench-201/all_path.py @@ -0,0 +1,8 @@ +SCORENET_CKPT_PATH="./checkpoints/scorenet/checkpoint.pth.tar" +META_SURROGATE_CKPT_PATH="./checkpoints/meta_surrogate/checkpoint.pth.tar" +META_SURROGATE_UNNOISED_CKPT_PATH = "./checkpoints/meta_surrogate/unnoised_checkpoint.pth.tar" +NASBENCH201="./data/transfer_nag/nasbench201.pt" +NASBENCH201_INFO="./data/transfer_nag/nasbench201_info.pt" +META_TEST_PATH="./data/transfer_nag/test" +RAW_DATA_PATH="./data/raw_data" +DATA_PATH = "./data/transfer_nag" diff --git a/NAS-Bench-201/analysis/arch_functions.py b/NAS-Bench-201/analysis/arch_functions.py new file mode 100644 index 0000000..bb4dcb6 --- /dev/null +++ b/NAS-Bench-201/analysis/arch_functions.py @@ -0,0 +1,347 @@ +import numpy as np +import torch +from all_path import * + + +class BasicArchMetrics(object): + def __init__(self, train_ds=None, train_arch_str_list=None): + if train_ds is None: + self.ops_decoder = ['input', 'output', 'none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3'] + else: + self.ops_decoder = train_ds.ops_decoder + self.nasbench201 = torch.load(NASBENCH201_INFO) + self.train_arch_str_list = train_arch_str_list + + + def compute_validity(self, generated): + START_TYPE = self.ops_decoder.index('input') + END_TYPE = self.ops_decoder.index('output') + + valid = [] + valid_arch_str = [] + all_arch_str = [] + for x in generated: + is_valid, error_types = is_valid_NAS201_x(x, START_TYPE, END_TYPE) + if is_valid: + valid.append(x) + arch_str = decode_x_to_NAS_BENCH_201_string(x, self.ops_decoder) + valid_arch_str.append(arch_str) + else: + arch_str = None + all_arch_str.append(arch_str) + validity = 0 if len(generated) == 0 else (len(valid)/len(generated)) + return valid, validity, valid_arch_str, all_arch_str + + + def compute_uniqueness(self, valid_arch_str): + return list(set(valid_arch_str)), len(set(valid_arch_str)) / len(valid_arch_str) + + + def compute_novelty(self, unique): + num_novel = 0 + novel = [] + if self.train_arch_str_list is None: + print("Dataset arch_str is None, novelty computation skipped") + return 1, 1 + for arch_str in unique: + if arch_str not in self.train_arch_str_list: + novel.append(arch_str) + num_novel += 1 + return novel, num_novel / len(unique) + + + def evaluate(self, generated, check_dataname='cifar10'): + valid, validity, valid_arch_str, all_arch_str = self.compute_validity(generated) + + if validity > 0: + unique, uniqueness = self.compute_uniqueness(valid_arch_str) + if self.train_arch_str_list is not None: + _, novelty = self.compute_novelty(unique) + else: + novelty = -1.0 + else: + novelty = -1.0 + uniqueness = 0.0 + unique = [] + + if uniqueness > 0.: + arch_idx_list, flops_list, params_list, latency_list = list(), list(), list(), list() + for arch in unique: + arch_index, flops, params, latency = \ + get_arch_acc_info(self.nasbench201, arch=arch, dataname=check_dataname) + arch_idx_list.append(arch_index) + flops_list.append(flops) + params_list.append(params) + latency_list.append(latency) + else: + arch_idx_list, flops_list, params_list, latency_list = [-1], [0], [0], [0] + + return ([validity, uniqueness, novelty], + unique, + dict(arch_idx_list=arch_idx_list, flops_list=flops_list, params_list=params_list, latency_list=latency_list), + all_arch_str) + + +class BasicArchMetricsMeta(object): + def __init__(self, train_ds=None, train_arch_str_list=None): + if train_ds is None: + self.ops_decoder = ['input', 'output', 'none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3'] + else: + self.ops_decoder = train_ds.ops_decoder + self.nasbench201 = torch.load(NASBENCH201_INFO) + self.train_arch_str_list = train_arch_str_list + + + def compute_validity(self, generated): + START_TYPE = self.ops_decoder.index('input') + END_TYPE = self.ops_decoder.index('output') + + valid = [] + valid_arch_str = [] + all_arch_str = [] + error_types = [] + + for x in generated: + is_valid, error_type = is_valid_NAS201_x(x, START_TYPE, END_TYPE) + if is_valid: + valid.append(x) + arch_str = decode_x_to_NAS_BENCH_201_string(x, self.ops_decoder) + valid_arch_str.append(arch_str) + else: + arch_str = None + error_types.append(error_type) + all_arch_str.append(arch_str) + + # exceptional case + validity = 0 if len(generated) == 0 else (len(valid)/len(generated)) + if len(valid) == 0: + validity = 0 + valid_arch_str = [] + + return valid, validity, valid_arch_str, all_arch_str + + + def compute_uniqueness(self, valid_arch_str): + return list(set(valid_arch_str)), len(set(valid_arch_str)) / len(valid_arch_str) + + + def compute_novelty(self, unique): + num_novel = 0 + novel = [] + if self.train_arch_str_list is None: + print("Dataset arch_str is None, novelty computation skipped") + return 1, 1 + for arch_str in unique: + if arch_str not in self.train_arch_str_list: + novel.append(arch_str) + num_novel += 1 + return novel, num_novel / len(unique) + + + def evaluate(self, generated, check_dataname='cifar10'): + valid, validity, valid_arch_str, all_arch_str = self.compute_validity(generated) + + if validity > 0: + unique, uniqueness = self.compute_uniqueness(valid_arch_str) + if self.train_arch_str_list is not None: + _, novelty = self.compute_novelty(unique) + else: + novelty = -1.0 + else: + novelty = -1.0 + uniqueness = 0.0 + unique = [] + + if uniqueness > 0.: + arch_idx_list, flops_list, params_list, latency_list = list(), list(), list(), list() + for arch in unique: + arch_index, flops, params, latency = \ + get_arch_acc_info_meta(self.nasbench201, arch=arch, dataname=check_dataname) + arch_idx_list.append(arch_index) + flops_list.append(flops) + params_list.append(params) + latency_list.append(latency) + else: + arch_idx_list, flops_list, params_list, latency_list = [-1], [0], [0], [0] + + return ([validity, uniqueness, novelty], + unique, + dict(arch_idx_list=arch_idx_list, flops_list=flops_list, params_list=params_list, latency_list=latency_list), + all_arch_str) + + +def get_arch_acc_info(nasbench201, arch, dataname='cifar10'): + arch_index = nasbench201['str'].index(arch) + flops = nasbench201['flops'][dataname][arch_index] + params = nasbench201['params'][dataname][arch_index] + latency = nasbench201['latency'][dataname][arch_index] + return arch_index, flops, params, latency + + +def get_arch_acc_info_meta(nasbench201, arch, dataname='cifar10'): + arch_index = nasbench201['str'].index(arch) + flops = nasbench201['flops'][dataname][arch_index] + params = nasbench201['params'][dataname][arch_index] + latency = nasbench201['latency'][dataname][arch_index] + return arch_index, flops, params, latency + + +def decode_igraph_to_NAS_BENCH_201_string(g): + if not is_valid_NAS201(g): + return None + m = decode_igraph_to_NAS201_matrix(g) + types = ['none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3'] + return '|{}~0|+|{}~0|{}~1|+|{}~0|{}~1|{}~2|'.\ + format(types[int(m[1][0])], + types[int(m[2][0])], types[int(m[2][1])], + types[int(m[3][0])], types[int(m[3][1])], types[int(m[3][2])]) + + +def decode_igraph_to_NAS201_matrix(g): + m = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]] + xys = [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] + for i, xy in enumerate(xys): + m[xy[0]][xy[1]] = float(g.vs[i + 1]['type']) - 2 + import numpy + return numpy.array(m) + + +def decode_x_to_NAS_BENCH_201_matrix(x): + m = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]] + xys = [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] + for i, xy in enumerate(xys): + # m[xy[0]][xy[1]] = int(torch.argmax(torch.tensor(x[i+1])).item()) - 2 + m[xy[0]][xy[1]] = int(torch.argmax(torch.tensor(x[i+1])).item()) + import numpy + return numpy.array(m) + + +def decode_x_to_NAS_BENCH_201_string(x, ops_decoder): + """_summary_ + + Args: + x (torch.Tensor): x_elem [8, 7] + + Returns: + arch_str + """ + is_valid, error_type = is_valid_NAS201_x(x) + if not is_valid: + return None + m = decode_x_to_NAS_BENCH_201_matrix(x) + types = ops_decoder + arch_str = '|{}~0|+|{}~0|{}~1|+|{}~0|{}~1|{}~2|'.\ + format(types[int(m[1][0])], + types[int(m[2][0])], types[int(m[2][1])], + types[int(m[3][0])], types[int(m[3][1])], types[int(m[3][2])]) + return arch_str + + +def decode_x_to_NAS_BENCH_201_string(x, ops_decoder): + """_summary_ + Args: + x (torch.Tensor): x_elem [8, 7] + Returns: + arch_str + """ + + if not is_valid_NAS201_x(x)[0]: + return None + m = decode_x_to_NAS_BENCH_201_matrix(x) + types = ops_decoder + arch_str = '|{}~0|+|{}~0|{}~1|+|{}~0|{}~1|{}~2|'.\ + format(types[int(m[1][0])], + types[int(m[2][0])], types[int(m[2][1])], + types[int(m[3][0])], types[int(m[3][1])], types[int(m[3][2])]) + return arch_str + + +def is_valid_DAG(g, START_TYPE=0, END_TYPE=1): + res = g.is_dag() + n_start, n_end = 0, 0 + for v in g.vs: + if v['type'] == START_TYPE: + n_start += 1 + elif v['type'] == END_TYPE: + n_end += 1 + if v.indegree() == 0 and v['type'] != START_TYPE: + return False + if v.outdegree() == 0 and v['type'] != END_TYPE: + return False + return res and n_start == 1 and n_end == 1 + + +def is_valid_NAS201(g, START_TYPE=0, END_TYPE=1): + # first need to be a valid DAG computation graph + res = is_valid_DAG(g, START_TYPE, END_TYPE) + # in addition, node i must connect to node i+1 + res = res and len(g.vs['type']) == 8 + res = res and not (START_TYPE in g.vs['type'][1:-1]) + res = res and not (END_TYPE in g.vs['type'][1:-1]) + return res + + +def check_single_node_type(x): + for x_elem in x: + if int(np.sum(x_elem)) != 1: + return False + return True + + +def check_start_end_nodes(x, START_TYPE, END_TYPE): + if x[0][START_TYPE] != 1: + return False + if x[-1][END_TYPE] != 1: + return False + return True + + +def check_interm_node_types(x, START_TYPE, END_TYPE): + for x_elem in x[1:-1]: + if x_elem[START_TYPE] == 1: + return False + if x_elem[END_TYPE] == 1: + return False + return True + + +ERORR_NB201 = { + 'MULTIPLE_NODE_TYPES': 1, + 'No_START_END': 2, + 'INTERM_START_END': 3, + 'NO_ERROR': -1 +} + + +def is_valid_NAS201_x(x, START_TYPE=0, END_TYPE=1): + # first need to be a valid DAG computation graph + assert len(x.shape) == 2 + + if not check_single_node_type(x): + return False, ERORR_NB201['MULTIPLE_NODE_TYPES'] + + if not check_start_end_nodes(x, START_TYPE, END_TYPE): + return False, ERORR_NB201['No_START_END'] + + if not check_interm_node_types(x, START_TYPE, END_TYPE): + return False, ERORR_NB201['INTERM_START_END'] + + return True, ERORR_NB201['NO_ERROR'] + + +def compute_arch_metrics(arch_list, + train_arch_str_list, + train_ds, + check_dataname='cifar10'): + metrics = BasicArchMetrics(train_ds, train_arch_str_list) + arch_metrics = metrics.evaluate(arch_list, check_dataname=check_dataname) + all_arch_str = arch_metrics[-1] + return arch_metrics, all_arch_str + +def compute_arch_metrics_meta(arch_list, + train_arch_str_list, + train_ds, + check_dataname='cifar10'): + metrics = BasicArchMetricsMeta(train_ds, train_arch_str_list) + arch_metrics = metrics.evaluate(arch_list, check_dataname=check_dataname) + return arch_metrics diff --git a/NAS-Bench-201/analysis/arch_metrics.py b/NAS-Bench-201/analysis/arch_metrics.py new file mode 100644 index 0000000..c70cce5 --- /dev/null +++ b/NAS-Bench-201/analysis/arch_metrics.py @@ -0,0 +1,77 @@ +from analysis.arch_functions import compute_arch_metrics, compute_arch_metrics_meta +import torch.nn as nn + + +class SamplingArchMetrics(nn.Module): + def __init__(self, + config, + train_ds, + exp_name,): + + super().__init__() + self.exp_name = exp_name + self.train_ds = train_ds + self.train_arch_str_list = train_ds.arch_str_list_ + + + def forward(self, + arch_list: list, + this_sample_dir, + check_dataname='cifar10'): + + arch_metrics, all_arch_str = compute_arch_metrics(arch_list=arch_list, + train_arch_str_list=self.train_arch_str_list, + train_ds=self.train_ds, + check_dataname=check_dataname) + + valid_unique_arch = arch_metrics[1] # arch_str + valid_unique_arch_prop_dict = arch_metrics[2] # flops, params, latency + textfile = open(f'{this_sample_dir}/valid_unique_archs.txt', "w") + for i in range(len(valid_unique_arch)): + textfile.write(f"Arch: {valid_unique_arch[i]} \n") + textfile.write(f"Arch Index: {valid_unique_arch_prop_dict['arch_idx_list'][i]} \n") + textfile.write(f"FLOPs: {valid_unique_arch_prop_dict['flops_list'][i]} \n") + textfile.write(f"#Params: {valid_unique_arch_prop_dict['params_list'][i]} \n") + textfile.write(f"Latency: {valid_unique_arch_prop_dict['latency_list'][i]} \n\n") + textfile.writelines(valid_unique_arch) + textfile.close() + + return arch_metrics + + +class SamplingArchMetricsMeta(nn.Module): + def __init__(self, + config, + train_ds, + exp_name): + + super().__init__() + self.exp_name = exp_name + self.train_ds = train_ds + self.search_space = config.data.name + self.train_arch_str_list = [train_ds.arch_str_list[i] for i in train_ds.idx_lst['train']] + + + def forward(self, + arch_list: list, + this_sample_dir, + check_dataname='cifar10'): + + arch_metrics = compute_arch_metrics_meta(arch_list=arch_list, + train_arch_str_list=self.train_arch_str_list, + train_ds=self.train_ds, + check_dataname=check_dataname) + + valid_unique_arch = arch_metrics[1] # arch_str + valid_unique_arch_prop_dict = arch_metrics[2] # flops, params, latency + textfile = open(f'{this_sample_dir}/valid_unique_archs.txt', "w") + for i in range(len(valid_unique_arch)): + textfile.write(f"Arch: {valid_unique_arch[i]} \n") + textfile.write(f"Arch Index: {valid_unique_arch_prop_dict['arch_idx_list'][i]} \n") + textfile.write(f"FLOPs: {valid_unique_arch_prop_dict['flops_list'][i]} \n") + textfile.write(f"#Params: {valid_unique_arch_prop_dict['params_list'][i]} \n") + textfile.write(f"Latency: {valid_unique_arch_prop_dict['latency_list'][i]} \n\n") + textfile.writelines(valid_unique_arch) + textfile.close() + + return arch_metrics \ No newline at end of file diff --git a/NAS-Bench-201/configs/eval_scorenet.py b/NAS-Bench-201/configs/eval_scorenet.py new file mode 100644 index 0000000..81a9b4f --- /dev/null +++ b/NAS-Bench-201/configs/eval_scorenet.py @@ -0,0 +1,72 @@ +"""Evaluate trained score network""" + +import ml_collections +import torch + +from all_path import SCORENET_CKPT_PATH + +def get_config(): + config = ml_collections.ConfigDict() + + # general + config.folder_name = 'test' + config.model_type = 'scorenet' + config.task = 'eval_scorenet' + config.exp_name = None + config.seed = 42 + config.device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + config.resume = False + config.scorenet_ckpt_path = SCORENET_CKPT_PATH + + # training + config.training = training = ml_collections.ConfigDict() + training.sde = 'vesde' + training.continuous = True + training.reduce_mean = True + training.noised = True + + # sampling + config.sampling = sampling = ml_collections.ConfigDict() + sampling.method = 'pc' + sampling.predictor = 'euler_maruyama' + sampling.corrector = 'langevin' + sampling.n_steps_each = 1 + sampling.noise_removal = True + sampling.probability_flow = False + sampling.snr = 0.16 + + # evaluation + config.eval = evaluate = ml_collections.ConfigDict() + evaluate.batch_size = 256 + evaluate.enable_sampling = True + evaluate.num_samples = 256 + + # data + config.data = data = ml_collections.ConfigDict() + data.centered = True + data.dequantization = False + + data.root = '../data/transfer_nag/nasbench201_info.pt' + data.name = 'NASBench201' + data.split_ratio = 1.0 + data.dataset_idx = 'random' # 'sorted' | 'random' + data.max_node = 8 + data.n_vocab = 7 # number of operations + data.START_TYPE = 0 + data.END_TYPE = 1 + data.num_graphs = 15625 + data.num_channels = 1 + data.label_list = ['test-acc'] + data.tg_dataset = 'cifar10' + # aug_mask + data.aug_mask_algo = 'floyd' # 'long_range' | 'floyd' + + # model + config.model = model = ml_collections.ConfigDict() + model.num_scales = 1000 + model.beta_min = 0.1 + model.beta_max = 5.0 + model.sigma_min = 0.1 + model.sigma_max = 5.0 + + return config diff --git a/NAS-Bench-201/configs/tr_meta_surrogate.py b/NAS-Bench-201/configs/tr_meta_surrogate.py new file mode 100644 index 0000000..543eb6a --- /dev/null +++ b/NAS-Bench-201/configs/tr_meta_surrogate.py @@ -0,0 +1,125 @@ +"""Training PGSN on Community Small Dataset with GraphGDP""" + +import ml_collections +import torch +from all_path import SCORENET_CKPT_PATH +from all_path import NASBENCH201_INFO + + +def get_config(): + config = ml_collections.ConfigDict() + + # config.search_space = None + + # general + config.folder_name = 'test' + config.model_type = 'meta_surrogate' + config.task = 'tr_meta_surrogate' + config.exp_name = None + config.seed = 42 + config.device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + config.resume = False + config.scorenet_ckpt_path = SCORENET_CKPT_PATH + + # training + config.training = training = ml_collections.ConfigDict() + training.sde = 'vesde' + training.continuous = True + training.reduce_mean = True + training.noised = True + training.batch_size = 256 + training.eval_batch_size = 100 + training.n_iters = 10000 + training.snapshot_freq = 500 + training.log_freq = 100 + training.eval_freq = 100 + training.snapshot_sampling = True + training.likelihood_weighting = False + + # sampling + config.sampling = sampling = ml_collections.ConfigDict() + sampling.method = 'pc' + sampling.predictor = 'euler_maruyama' + sampling.corrector = 'langevin' + sampling.n_steps_each = 1 + sampling.noise_removal = True + sampling.probability_flow = False + sampling.snr = 0.16 + + # for conditional sampling + sampling.classifier_scale = 10000.0 + sampling.regress = True + sampling.labels = 'max' + sampling.weight_ratio = False + sampling.weight_scheduling = False + sampling.check_dataname = 'cifar10' + + # evaluation + config.eval = evaluate = ml_collections.ConfigDict() + evaluate.batch_size = 512 + evaluate.enable_sampling = True + evaluate.num_samples = 1024 + + # data + config.data = data = ml_collections.ConfigDict() + data.centered = True + data.dequantization = False + + data.root = NASBENCH201_INFO + data.name = 'NASBench201' + data.max_node = 8 + data.n_vocab = 7 + data.START_TYPE = 0 + data.END_TYPE = 1 + data.num_channels = 1 + data.label_list = ['meta-acc'] + # aug_mask + data.aug_mask_algo = 'floyd' # 'long_range' | 'floyd' + + # model + config.model = model = ml_collections.ConfigDict() + model.name = 'MetaNeuralPredictor' + model.ema_rate = 0.9999 + model.normalization = 'GroupNorm' + model.nonlinearity = 'swish' + model.nf = 128 + model.num_gnn_layers = 4 + model.size_cond = False + model.embedding_type = 'positional' + model.rw_depth = 16 + model.graph_layer = 'PosTransLayer' + model.edge_th = -1. + model.heads = 8 + model.attn_clamp = False + + # meta-predictor + model.input_type = 'DA' + model.hs = 32 + model.nz = 56 + model.num_sample = 20 + + model.num_scales = 1000 + model.beta_min = 0.1 + model.beta_max = 5.0 + model.sigma_min = 0.1 + model.sigma_max = 5.0 + model.dropout = 0.1 + + # graph encoder + config.model.graph_encoder = graph_encoder = ml_collections.ConfigDict() + graph_encoder.initial_hidden = 7 + graph_encoder.gcn_hidden = 144 + graph_encoder.gcn_layers = 4 + graph_encoder.linear_hidden = 128 + + # optimization + config.optim = optim = ml_collections.ConfigDict() + optim.weight_decay = 0 + optim.optimizer = 'Adam' + optim.lr = 0.001 + optim.beta1 = 0.9 + optim.eps = 1e-8 + optim.warmup = 1000 + optim.grad_clip = 1. + + return config diff --git a/NAS-Bench-201/configs/tr_scorenet.py b/NAS-Bench-201/configs/tr_scorenet.py new file mode 100644 index 0000000..1ca64a1 --- /dev/null +++ b/NAS-Bench-201/configs/tr_scorenet.py @@ -0,0 +1,113 @@ +"""Training Score Network""" + +import ml_collections +import torch + + +def get_config(): + config = ml_collections.ConfigDict() + + # general + config.folder_name = 'test' + config.model_type = 'scorenet' + config.task = 'tr_scorenet' + config.exp_name = None + config.seed = 42 + config.device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + config.resume = False + config.resume_ckpt_path = '' + + # training + config.training = training = ml_collections.ConfigDict() + training.sde = 'vesde' + training.continuous = True + training.reduce_mean = True + + training.batch_size = 256 + training.eval_batch_size = 1000 + training.n_iters = 250000 + training.snapshot_freq = 10000 + training.log_freq = 200 + training.eval_freq = 10000 + training.snapshot_sampling = True + training.likelihood_weighting = False + + # sampling + config.sampling = sampling = ml_collections.ConfigDict() + sampling.method = 'pc' + sampling.predictor = 'euler_maruyama' + sampling.corrector = 'langevin' + sampling.n_steps_each = 1 + sampling.noise_removal = True + sampling.probability_flow = False + sampling.snr = 0.16 + + # evaluation + config.eval = evaluate = ml_collections.ConfigDict() + evaluate.batch_size = 1024 + evaluate.enable_sampling = True + evaluate.num_samples = 1024 + + # data + config.data = data = ml_collections.ConfigDict() + data.centered = True + data.dequantization = False + + data.root = '../data/transfer_nag/nasbench201_info.pt' + data.name = 'NASBench201' + data.split_ratio = 1.0 + data.dataset_idx = 'random' # 'sorted' | 'random' + data.max_node = 8 + data.n_vocab = 7 # number of operations + data.START_TYPE = 0 + data.END_TYPE = 1 + data.num_graphs = 15625 + data.num_channels = 1 + data.label_list = None + data.tg_dataset = None + # aug_mask + data.aug_mask_algo = 'floyd' # 'long_range' | 'floyd' + + # model + config.model = model = ml_collections.ConfigDict() + model.name = 'CATE' + model.ema_rate = 0.9999 + model.normalization = 'GroupNorm' + model.nonlinearity = 'swish' + model.nf = 128 + model.num_gnn_layers = 4 + model.size_cond = False + model.embedding_type = 'positional' + model.rw_depth = 16 + model.graph_layer = 'PosTransLayer' + model.edge_th = -1. + model.heads = 8 + model.attn_clamp = False + # for pos emb + model.pos_enc_type = 2 + + model.num_scales = 1000 + model.sigma_min = 0.1 + model.sigma_max = 5.0 + model.dropout = 0.1 + + # graph encoder + config.model.graph_encoder = graph_encoder = ml_collections.ConfigDict() + graph_encoder.n_layers = 12 + graph_encoder.d_model = 64 + graph_encoder.n_head = 8 + graph_encoder.d_ff = 128 + graph_encoder.dropout = 0.1 + graph_encoder.n_vocab = 7 + + # optimization + config.optim = optim = ml_collections.ConfigDict() + optim.weight_decay = 0 + optim.optimizer = 'Adam' + optim.lr = 2e-5 + optim.beta1 = 0.9 + optim.eps = 1e-8 + optim.warmup = 1000 + optim.grad_clip = 1. + + return config diff --git a/NAS-Bench-201/datasets_nas.py b/NAS-Bench-201/datasets_nas.py new file mode 100644 index 0000000..3f398f8 --- /dev/null +++ b/NAS-Bench-201/datasets_nas.py @@ -0,0 +1,469 @@ +from __future__ import print_function +import torch +import os +import numpy as np +from collections import defaultdict +from torch.utils.data import DataLoader, Dataset +from analysis.arch_functions import decode_x_to_NAS_BENCH_201_matrix, decode_x_to_NAS_BENCH_201_string +from all_path import * + + +def get_data_scaler(config): + """Data normalizer. Assume data are always in [0, 1].""" + + if config.data.centered: + # Rescale to [-1, 1] + return lambda x: x * 2. - 1. + else: + return lambda x: x + + +def get_data_inverse_scaler(config): + """Inverse data normalizer.""" + + if config.data.centered: + # Rescale [-1, 1] to [0, 1] + return lambda x: (x + 1.) / 2. + else: + return lambda x: x + + +def is_triu(mat): + is_triu_ = np.allclose(mat, np.triu(mat)) + return is_triu_ + + +def get_dataset(config): + train_dataset = NASBench201Dataset( + data_path=NASBENCH201_INFO, + mode='train') + + eval_dataset = NASBench201Dataset( + data_path=NASBENCH201_INFO, + mode='eval') + + test_dataset = NASBench201Dataset( + data_path=NASBENCH201_INFO, + mode='test') + + return train_dataset, eval_dataset, test_dataset + + +def get_dataloader(config, train_dataset, eval_dataset, test_dataset): + train_loader = DataLoader(dataset=train_dataset, + batch_size=config.training.batch_size, + shuffle=True, + collate_fn=None) + + eval_loader = DataLoader(dataset=eval_dataset, + batch_size=config.training.batch_size, + shuffle=False, + collate_fn=None) + + test_loader = DataLoader(dataset=test_dataset, + batch_size=config.training.batch_size, + shuffle=False, + collate_fn=None) + + return train_loader, eval_loader, test_loader + + +class NASBench201Dataset(Dataset): + def __init__( + self, + data_path, + split_ratio=1.0, + mode='train', + label_list=None, + tg_dataset=None): + + self.ops_decoder = ['input', 'output', 'none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3'] + + # ---------- entire dataset ---------- # + self.data = torch.load(data_path) + # ---------- igraph ---------- # + self.igraph_list = self.data['g'] + # ---------- x ---------- # + self.x_list = self.data['x'] + # ---------- adj ---------- # + adj = self.get_adj() + self.adj_list = [adj] * len(self.igraph_list) + # ---------- matrix ---------- # + self.matrix_list = self.data['matrix'] + # ---------- arch_str ---------- # + self.arch_str_list = self.data['str'] + # ---------- labels ---------- # + self.label_list = label_list + if self.label_list is not None: + self.val_acc_list = self.data['val-acc'][tg_dataset] + self.test_acc_list = self.data['test-acc'][tg_dataset] + self.flops_list = self.data['flops'][tg_dataset] + self.params_list = self.data['params'][tg_dataset] + self.latency_list = self.data['latency'][tg_dataset] + + # ----------- split dataset ---------- # + self.ds_idx = list(torch.load(DATA_PATH + '/ridx.pt')) + self.split_ratio = split_ratio + num_train = int(len(self.x_list) * self.split_ratio) + num_test = len(self.x_list) - num_train + + # ----------- compute mean and std w/ training dataset ---------- # + if self.label_list is not None: + self.train_idx_list = self.ds_idx[:num_train] + print('>>> Computing mean and std of the training set...') + LABEL_TO_MEAN_STD = defaultdict(dict) + assert type(self.label_list) == list, f"self.label_list is {type(self.label_list)}" + for label in self.label_list: + if label == 'val-acc': + self.val_acc_list_tr = [self.val_acc_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.val_acc_list_tr)) + elif label == 'test-acc': + self.test_acc_list_tr = [self.test_acc_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.test_acc_list_tr)) + elif label == 'flops': + self.flops_list_tr = [self.flops_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.flops_list_tr)) + elif label == 'params': + self.params_list_tr = [self.params_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.params_list_tr)) + elif label == 'latency': + self.latency_list_tr = [self.latency_list[i] for i in self.train_idx_list] + LABEL_TO_MEAN_STD[label]['std'], LABEL_TO_MEAN_STD[label]['mean'] = torch.std_mean(torch.tensor(self.latency_list_tr)) + else: + raise ValueError + + self.mode = mode + if self.mode in ['train']: + self.idx_list = self.ds_idx[:num_train] + elif self.mode in ['eval']: + if num_test == 0: + self.idx_list = self.ds_idx[:100] + else: + self.idx_list = self.ds_idx[:num_test] + elif self.mode in ['test']: + if num_test == 0: + self.idx_list = self.ds_idx[15000:] + else: + self.idx_list = self.ds_idx[num_train:] + + self.igraph_list_ = [self.igraph_list[i] for i in self.idx_list] + self.x_list_ = [self.x_list[i] for i in self.idx_list] + self.adj_list_ = [self.adj_list[i] for i in self.idx_list] + self.matrix_list_ = [self.matrix_list[i] for i in self.idx_list] + self.arch_str_list_ = [self.arch_str_list[i] for i in self.idx_list] + + if self.label_list is not None: + assert type(self.label_list) == list + for label in self.label_list: + if label == 'val-acc': + self.val_acc_list_ = [self.val_acc_list[i] for i in self.idx_list] + self.val_acc_list_ = self.normalize(self.val_acc_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + elif label == 'test-acc': + self.test_acc_list_ = [self.test_acc_list[i] for i in self.idx_list] + self.test_acc_list_ = self.normalize(self.test_acc_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + elif label == 'flops': + self.flops_list_ = [self.flops_list[i] for i in self.idx_list] + self.flops_list_ = self.normalize(self.flops_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + elif label == 'params': + self.params_list_ = [self.params_list[i] for i in self.idx_list] + self.params_list_ = self.normalize(self.params_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + elif label == 'latency': + self.latency_list_ = [self.latency_list[i] for i in self.idx_list] + self.latency_list_ = self.normalize(self.latency_list_, LABEL_TO_MEAN_STD[label]['mean'], LABEL_TO_MEAN_STD[label]['std']) + else: + raise ValueError + + + def normalize(self, original, mean, std): + return [(i-mean)/std for i in original] + + + # def get_not_connect_prev_adj(self): + def get_adj(self): + adj = np.asarray( + [[0, 1, 1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 1, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 0]] + ) + adj = torch.tensor(adj, dtype=torch.float32, device=torch.device('cpu')) + return adj + + + @property + def adj(self): + return self.adj_list_[0] + + + def mask(self, algo='floyd'): + from utils import aug_mask + return aug_mask(self.adj, algo=algo)[0] + + + def get_unnoramlized_entire_data(self, label, tg_dataset): + entire_val_acc_list = self.data['val-acc'][tg_dataset] + entire_test_acc_list = self.data['test-acc'][tg_dataset] + entire_flops_list = self.data['flops'][tg_dataset] + entire_params_list = self.data['params'][tg_dataset] + entire_latency_list = self.data['latency'][tg_dataset] + + if label == 'val-acc': + return entire_val_acc_list + elif label == 'test-acc': + return entire_test_acc_list + elif label == 'flops': + return entire_flops_list + elif label == 'params': + return entire_params_list + elif label == 'latency': + return entire_latency_list + else: + raise ValueError + + + def get_unnoramlized_data(self, label, tg_dataset): + entire_val_acc_list = self.data['val-acc'][tg_dataset] + entire_test_acc_list = self.data['test-acc'][tg_dataset] + entire_flops_list = self.data['flops'][tg_dataset] + entire_params_list = self.data['params'][tg_dataset] + entire_latency_list = self.data['latency'][tg_dataset] + + if label == 'val-acc': + return [entire_val_acc_list[i] for i in self.idx_list] + elif label == 'test-acc': + return [entire_test_acc_list[i] for i in self.idx_list] + elif label == 'flops': + return [entire_flops_list[i] for i in self.idx_list] + elif label == 'params': + return [entire_params_list[i] for i in self.idx_list] + elif label == 'latency': + return [entire_latency_list[i] for i in self.idx_list] + else: + raise ValueError + + + def __len__(self): + return len(self.x_list_) + + + def __getitem__(self, index): + label_dict = {} + if self.label_list is not None: + assert type(self.label_list) == list + for label in self.label_list: + if label == 'val-acc': + label_dict[f"{label}"] = self.val_acc_list_[index] + elif label == 'test-acc': + label_dict[f"{label}"] = self.test_acc_list_[index] + elif label == 'flops': + label_dict[f"{label}"] = self.flops_list_[index] + elif label == 'params': + label_dict[f"{label}"] = self.params_list_[index] + elif label == 'latency': + label_dict[f"{label}"] = self.latency_list_[index] + else: + raise ValueError + return self.x_list_[index], self.adj_list_[index], label_dict + + +# ---------- Meta-Dataset ---------- # +def get_meta_dataset(config): + train_dataset = MetaTrainDatabase( + data_path=DATA_PATH, + num_sample=config.model.num_sample, + label_list=config.data.label_list, + mode='train') + + eval_dataset = MetaTrainDatabase( + data_path=DATA_PATH, + num_sample=config.model.num_sample, + label_list=config.data.label_list, + mode='eval') + + test_dataset = None + + return train_dataset, eval_dataset, test_dataset + + +def get_meta_dataloader(config ,train_dataset, eval_dataset, test_dataset): + train_loader = DataLoader(dataset=train_dataset, + batch_size=config.training.batch_size, + shuffle=True) + + eval_loader = DataLoader(dataset=eval_dataset, + batch_size=config.training.batch_size, + shuffle=False) + + test_loader = None + + return train_loader, eval_loader, test_loader + + +class MetaTrainDatabase(Dataset): + def __init__( + self, + data_path, + num_sample, + label_list, + mode='train'): + + self.ops_decoder = ['input', 'output', 'none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3'] + + self.mode = mode + self.acc_norm = True + self.num_sample = num_sample + self.x = torch.load(os.path.join(data_path, 'imgnet32bylabel.pt')) + + mtr_data_path = os.path.join(data_path, 'meta_train_tasks_predictor.pt') + idx_path = os.path.join(data_path, 'meta_train_tasks_predictor_idx.pt') + data = torch.load(mtr_data_path) + + self.acc_list = data['acc'] + self.task = data['task'] + + # ---------- igraph ---------- # + self.igraph_list = data['g'] + # ---------- x ---------- # + self.x_list = data['x'] + # ---------- adj ---------- # + adj = self.get_adj() + self.adj_list = [adj] * len(self.igraph_list) + # ---------- matrix ----------- # + if 'matrix' in data: + self.matrix_list = data['matrix'] + else: + self.matrix_list = [decode_x_to_NAS_BENCH_201_matrix(i) for i in self.x_list] + # ---------- arch_str ---------- # + if 'str' in data: + self.arch_str_list = data['str'] + else: + self.arch_str_list = [decode_x_to_NAS_BENCH_201_string(i, self.ops_decoder) for i in self.x_list] + # ---------- label ---------- # + self.label_list = label_list + if self.label_list is not None: + self.flops_list = torch.tensor(data['flops']) + self.params_list = torch.tensor(data['params']) + self.latency_list = torch.tensor(data['latency']) + + random_idx_lst = torch.load(idx_path) + self.idx_lst = {} + self.idx_lst['eval'] = random_idx_lst[:400] + self.idx_lst['train'] = random_idx_lst[400:] + self.acc_list = torch.tensor(self.acc_list) + self.mean = torch.mean(self.acc_list[self.idx_lst['train']]).item() + self.std = torch.std(self.acc_list[self.idx_lst['train']]).item() + self.task_lst = torch.load(os.path.join(data_path, 'meta_train_task_lst.pt')) + + + def get_adj(self): + adj = np.asarray( + [[0, 1, 1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 1, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 0]] + ) + adj = torch.tensor(adj, dtype=torch.float32, device=torch.device('cpu')) + return adj + + + @property + def adj(self): + return self.adj_list[0] + + + def mask(self, algo='floyd'): + from utils import aug_mask + return aug_mask(self.adj, algo=algo)[0] + + + def set_mode(self, mode): + self.mode = mode + + + def __len__(self): + return len(self.idx_lst[self.mode]) + + + def __getitem__(self, index): + data = [] + ridx = self.idx_lst[self.mode] + tidx = self.task[ridx[index]] + classes = self.task_lst[tidx] + + # ---------- igraph ----------- + graph = self.igraph_list[ridx[index]] + # ---------- x ----------- + x = self.x_list[ridx[index]] + # ---------- adj ---------- + adj = self.adj_list[ridx[index]] + + acc = self.acc_list[ridx[index]] + for cls in classes: + cx = self.x[cls-1][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + task = torch.cat(data) + if self.acc_norm: + acc = ((acc- self.mean) / self.std) / 100.0 + else: + acc = acc / 100.0 + + label_dict = {} + if self.label_list is not None: + assert type(self.label_list) == list + for label in self.label_list: + if label == 'meta-acc': + label_dict[f"{label}"] = acc + elif label == 'flops': + label_dict[f"{label}"] = self.flops_list[ridx[index]] + elif label == 'params': + label_dict[f"{label}"] = self.params_list[ridx[index]] + elif label == 'latency': + label_dict[f"{label}"] = self.latency_list[ridx[index]] + else: + raise ValueError + + return x, adj, label_dict, task + + +class MetaTestDataset(Dataset): + def __init__(self, data_path, data_name, num_sample, num_class=None): + self.num_sample = num_sample + self.data_name = data_name + + num_class_dict = { + 'cifar100': 100, + 'cifar10': 10, + 'aircraft': 30, + 'pets': 37 + } + + if num_class is not None: + self.num_class = num_class + else: + self.num_class = num_class_dict[data_name] + + self.x = torch.load(os.path.join(data_path, f'{data_name}bylabel.pt')) + + + def __len__(self): + return 1000000 + + + def __getitem__(self, index): + data = [] + classes = list(range(self.num_class)) + for cls in classes: + cx = self.x[cls][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + x = torch.cat(data) + return x \ No newline at end of file diff --git a/NAS-Bench-201/logger.py b/NAS-Bench-201/logger.py new file mode 100644 index 0000000..b353efe --- /dev/null +++ b/NAS-Bench-201/logger.py @@ -0,0 +1,137 @@ +import os +import wandb +import torch +import numpy as np + + +class Logger: + def __init__( + self, + log_dir=None, + write_textfile=True + ): + + self.log_dir = log_dir + self.write_textfile = write_textfile + + self.logs_for_save = {} + self.logs = {} + + if self.write_textfile: + self.f = open(os.path.join(log_dir, 'logs.txt'), 'w') + + + def write_str(self, log_str): + self.f.write(log_str+'\n') + self.f.flush() + + + def update_config(self, v, is_args=False): + if is_args: + self.logs_for_save.update({'args': v}) + else: + self.logs_for_save.update(v) + + + def write_log(self, element, step, return_log_dict=False): + log_str = f"{step} | " + log_dict = {} + for head, keys in element.items(): + for k in keys: + if k in self.logs: + v = self.logs[k].avg + if not k in self.logs_for_save: + self.logs_for_save[k] = [] + self.logs_for_save[k].append(v) + log_str += f'{k} {v}| ' + log_dict[f'{head}/{k}'] = v + + if self.write_textfile: + self.f.write(log_str+'\n') + self.f.flush() + + if return_log_dict: + return log_dict + + + def save_log(self, name=None): + name = 'logs.pt' if name is None else name + torch.save(self.logs_for_save, os.path.join(self.log_dir, name)) + + + def update(self, key, v, n=1): + if not key in self.logs: + self.logs[key] = AverageMeter() + self.logs[key].update(v, n) + + + def reset(self, keys=None, except_keys=[]): + if keys is not None: + if isinstance(keys, list): + for key in keys: + self.logs[key] = AverageMeter() + else: + self.logs[keys] = AverageMeter() + else: + for key in self.logs.keys(): + if not key in except_keys: + self.logs[key] = AverageMeter() + + + def avg(self, keys=None, except_keys=[]): + if keys is not None: + if isinstance(keys, list): + return {key: self.logs[key].avg for key in keys if key in self.logs.keys()} + else: + return self.logs[keys].avg + else: + avg_dict = {} + for key in self.logs.keys(): + if not key in except_keys: + avg_dict[key] = self.logs[key].avg + return avg_dict + + +class AverageMeter(object): + """ + Computes and stores the average and current value + Copied from: https://github.com/pytorch/examples/blob/master/imagenet/main.py + """ + + def __init__(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def get_metrics(g_embeds, x_embeds, logit_scale, prefix='train'): + metrics = {} + logits_per_g = (logit_scale * g_embeds @ x_embeds.t()).detach().cpu() + logits_per_x = logits_per_g.t().detach().cpu() + + logits = {"g_to_x": logits_per_g, "x_to_g": logits_per_x} + ground_truth = torch.arange(len(x_embeds)).view(-1, 1) + + for name, logit in logits.items(): + ranking = torch.argsort(logit, descending=True) + preds = torch.where(ranking == ground_truth)[1] + preds = preds.detach().cpu().numpy() + metrics[f"{prefix}_{name}_mean_rank"] = preds.mean() + 1 + metrics[f"{prefix}_{name}_median_rank"] = np.floor(np.median(preds)) + 1 + for k in [1, 5, 10]: + metrics[f"{prefix}_{name}_R@{k}"] = np.mean(preds < k) + + return metrics \ No newline at end of file diff --git a/NAS-Bench-201/losses.py b/NAS-Bench-201/losses.py new file mode 100644 index 0000000..652478e --- /dev/null +++ b/NAS-Bench-201/losses.py @@ -0,0 +1,369 @@ +"""All functions related to loss computation and optimization.""" + +import torch +import torch.optim as optim +import numpy as np +from models import utils as mutils +from sde_lib import VPSDE, VESDE + + +def get_optimizer(config, params): + """Return a flax optimizer object based on `config`.""" + if config.optim.optimizer == 'Adam': + optimizer = optim.Adam(params, lr=config.optim.lr, betas=(config.optim.beta1, 0.999), eps=config.optim.eps, + weight_decay=config.optim.weight_decay) + else: + raise NotImplementedError( + f'Optimizer {config.optim.optimizer} not supported yet!' + ) + return optimizer + + +def optimization_manager(config): + """Return an optimize_fn based on `config`.""" + + def optimize_fn(optimizer, params, step, lr=config.optim.lr, + warmup=config.optim.warmup, + grad_clip=config.optim.grad_clip): + """Optimize with warmup and gradient clipping (disabled if negative).""" + if warmup > 0: + for g in optimizer.param_groups: + g['lr'] = lr * np.minimum(step / warmup, 1.0) + if grad_clip >= 0: + torch.nn.utils.clip_grad_norm_(params, max_norm=grad_clip) + optimizer.step() + + return optimize_fn + + +def get_sde_loss_fn(sde, train, reduce_mean=True, continuous=True, likelihood_weighting=True, eps=1e-5): + """Create a loss function for training with arbitrary SDEs. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + train: `True` for training loss and `False` for evaluation loss. + reduce_mean: If `True`, average the loss across data dimensions. Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + Otherwise, it requires ad-hoc interpolation to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according + to https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended in Score SDE paper. + eps: A `float` number. The smallest time step to sample from. + + Returns: + A loss function. + """ + + # reduce_op = torch.mean if reduce_mean else lambda *args, **kwargs: 0.5 * torch.sum(*args, **kwargs) + + def loss_fn(model, batch): + """Compute the loss function. + + Args: + model: A score model. + batch: A mini-batch of training data, including adjacency matrices and mask. + + Returns: + loss: A scalar that represents the average loss value across the mini-batch. + """ + adj, mask = batch + score_fn = mutils.get_score_fn(sde, model, train=train, continuous=continuous) + t = torch.rand(adj.shape[0], device=adj.device) * (sde.T - eps) + eps + + z = torch.randn_like(adj) # [B, C, N, N] + z = torch.tril(z, -1) + z = z + z.transpose(2, 3) + + mean, std = sde.marginal_prob(adj, t) + mean = torch.tril(mean, -1) + mean = mean + mean.transpose(2, 3) + + perturbed_data = mean + std[:, None, None, None] * z + score = score_fn(perturbed_data, t, mask=mask) + + mask = torch.tril(mask, -1) + mask = mask + mask.transpose(2, 3) + mask = mask.reshape(mask.shape[0], -1) # low triangular part of adj matrices + + if not likelihood_weighting: + losses = torch.square(score * std[:, None, None, None] + z) + losses = losses.reshape(losses.shape[0], -1) + if reduce_mean: + losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + else: + losses = 0.5 * torch.sum(losses * mask, dim=-1) + loss = losses.mean() + else: + g2 = sde.sde(torch.zeros_like(adj), t)[1] ** 2 + losses = torch.square(score + z / std[:, None, None, None]) + losses = losses.reshape(losses.shape[0], -1) + if reduce_mean: + losses = torch.sum(losses * mask, dim=-1) / torch.sum(mask, dim=-1) + else: + losses = 0.5 * torch.sum(losses * mask, dim=-1) + loss = (losses * g2).mean() + + return loss + + return loss_fn + + +def get_sde_loss_fn_nas(sde, train, reduce_mean=True, continuous=True, likelihood_weighting=True, eps=1e-5): + """Create a loss function for training with arbitrary SDEs. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + train: `True` for training loss and `False` for evaluation loss. + reduce_mean: If `True`, average the loss across data dimensions. Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + Otherwise, it requires ad-hoc interpolation to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according + to https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended in Score SDE paper. + eps: A `float` number. The smallest time step to sample from. + + Returns: + A loss function. + """ + + def loss_fn(model, batch): + """Compute the loss function. + + Args: + model: A score model. + batch: A mini-batch of training data, including adjacency matrices and mask. + + Returns: + loss: A scalar that represents the average loss value across the mini-batch. + """ + x, adj, mask = batch + score_fn = mutils.get_score_fn(sde, model, train=train, continuous=continuous) + t = torch.rand(x.shape[0], device=adj.device) * (sde.T - eps) + eps + + z = torch.randn_like(x) # [B, C, N, N] + + mean, std = sde.marginal_prob(x, t) + + perturbed_data = mean + std[:, None, None] * z + score = score_fn(perturbed_data, t, mask) + + if not likelihood_weighting: + losses = torch.square(score * std[:, None, None] + z) + losses = losses.reshape(losses.shape[0], -1) + if reduce_mean: + losses = torch.mean(losses, dim=-1) + else: + losses = 0.5 * torch.sum(losses, dim=-1) + loss = losses.mean() + else: + g2 = sde.sde(torch.zeros_like(x), t)[1] ** 2 + losses = torch.square(score + z / std[:, None, None]) + losses = losses.reshape(losses.shape[0], -1) + if reduce_mean: + losses = torch.mean(losses, dim=-1) + else: + losses = 0.5 * torch.sum(losses, dim=-1) + loss = (losses * g2).mean() + + return loss + + return loss_fn + + +def get_step_fn(sde, + train, + optimize_fn=None, + reduce_mean=False, + continuous=True, + likelihood_weighting=False, + data='NASBench201'): + """Create a one-step training/evaluation function. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + Tuple (`sde_lib.SDE`, `sde_lib.SDE`) that represents the forward node SDE and edge SDE. + optimize_fn: An optimization function. + reduce_mean: If `True`, average the loss across data dimensions. + Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according to + https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended by score-sde. + + Returns: + A one-step function for training or evaluation. + """ + + if continuous: + if data in ['NASBench201', 'ofa']: + loss_fn = get_sde_loss_fn_nas(sde, train, reduce_mean=reduce_mean, + continuous=True, likelihood_weighting=likelihood_weighting) + else: + raise NotImplementedError(f"Data {data} (search space) is not supported yet.") + else: + raise NotImplementedError(f"Discrete training for {sde.__class__.__name__} is not implemented.") + + + def step_fn(state, batch): + """Running one step of training or evaluation. + + For jax version: This function will undergo `jax.lax.scan` so that multiple steps can be pmapped and + jit-compiled together for faster execution. + + Args: + state: A dictionary of training information, containing the score model, optimizer, + EMA status, and number of optimization steps. + batch: A mini-batch of training/evaluation data, including min-batch adjacency matrices and mask. + + Returns: + loss: The average loss value of this state. + """ + model = state['model'] + if train: + optimizer = state['optimizer'] + optimizer.zero_grad() + loss = loss_fn(model, batch) + loss.backward() + optimize_fn(optimizer, model.parameters(), step=state['step']) + state['step'] += 1 + state['ema'].update(model.parameters()) + else: + with torch.no_grad(): + ema = state['ema'] + ema.store(model.parameters()) + ema.copy_to(model.parameters()) + loss = loss_fn(model, batch) + ema.restore(model.parameters()) + + return loss + + return step_fn + + +# ------------------- predictor ------------------- +def get_meta_predictor_loss_fn_nas(sde, + train, + reduce_mean=True, + continuous=True, + likelihood_weighting=True, + eps=1e-5, + label_list=None, + noised=True): + """Create a loss function for training with arbitrary SDEs. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + train: `True` for training loss and `False` for evaluation loss. + reduce_mean: If `True`, average the loss across data dimensions. Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + Otherwise, it requires ad-hoc interpolation to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according + to https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended in Score SDE paper. + eps: A `float` number. The smallest time step to sample from. + + Returns: + A loss function. + """ + + def loss_fn(model, batch): + """Compute the loss function. + + Args: + model: A score model. + batch: A mini-batch of training data, including adjacency matrices and mask. + + Returns: + loss: A scalar that represents the average loss value across the mini-batch. + """ + x, adj, mask, extra, task = batch + predictor_fn = mutils.get_predictor_fn(sde, model, train=train, continuous=continuous) + if noised: + t = torch.rand(x.shape[0], device=adj.device) * (sde.T - eps) + eps + z = torch.randn_like(x) # [B, C, N, N] + + mean, std = sde.marginal_prob(x, t) + + perturbed_data = mean + std[:, None, None] * z + pred = predictor_fn(perturbed_data, t, mask, task) + else: + t = eps * torch.ones(x.shape[0], device=adj.device) + pred = predictor_fn(x, t, mask, task) + + labels = extra[f"{label_list[-1]}"] + labels = labels.to(pred.device).unsqueeze(1).type(pred.dtype) + + loss = torch.nn.MSELoss()(pred, labels) + + return loss, pred, labels + + return loss_fn + + +def get_step_fn_predictor(sde, + train, + optimize_fn=None, + reduce_mean=False, + continuous=True, + likelihood_weighting=False, + data='NASBench201', + label_list=None, + noised=True): + """Create a one-step training/evaluation function. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + Tuple (`sde_lib.SDE`, `sde_lib.SDE`) that represents the forward node SDE and edge SDE. + optimize_fn: An optimization function. + reduce_mean: If `True`, average the loss across data dimensions. + Otherwise, sum the loss across data dimensions. + continuous: `True` indicates that the model is defined to take continuous time steps. + likelihood_weighting: If `True`, weight the mixture of score matching losses according to + https://arxiv.org/abs/2101.09258; otherwise, use the weighting recommended by score-sde. + + Returns: + A one-step function for training or evaluation. + """ + + if continuous: + if data in ['NASBench201', 'ofa']: + loss_fn = get_meta_predictor_loss_fn_nas(sde, + train, + reduce_mean=reduce_mean, + continuous=True, + likelihood_weighting=likelihood_weighting, + label_list=label_list, + noised=noised) + else: + raise NotImplementedError(f"Data {data} (search space) is not supported yet.") + else: + raise NotImplementedError(f"Discrete training for {sde.__class__.__name__} is not implemented.") + + + def step_fn(state, batch): + """Running one step of training or evaluation. + + For jax version: This function will undergo `jax.lax.scan` so that multiple steps can be pmapped and + jit-compiled together for faster execution. + + Args: + state: A dictionary of training information, containing the score model, optimizer, + EMA status, and number of optimization steps. + batch: A mini-batch of training/evaluation data, including min-batch adjacency matrices and mask. + + Returns: + loss: The average loss value of this state. + """ + model = state['model'] + if train: + model.train() + optimizer = state['optimizer'] + optimizer.zero_grad() + loss, pred, labels = loss_fn(model, batch) + loss.backward() + optimize_fn(optimizer, model.parameters(), step=state['step']) + state['step'] += 1 + else: + model.eval() + with torch.no_grad(): + loss, pred, labels = loss_fn(model, batch) + + return loss, pred, labels + + return step_fn \ No newline at end of file diff --git a/NAS-Bench-201/main.py b/NAS-Bench-201/main.py new file mode 100644 index 0000000..325fb35 --- /dev/null +++ b/NAS-Bench-201/main.py @@ -0,0 +1,37 @@ +"""Training and evaluation""" + +import run_lib +from absl import app, flags +from ml_collections.config_flags import config_flags +import logging + + +FLAGS = flags.FLAGS + + +config_flags.DEFINE_config_file( + 'config', None, 'Training configuration.', lock_config=True +) +config_flags.DEFINE_config_file( + 'classifier_config_nf', None, 'Training configuration.', lock_config=True +) +flags.DEFINE_enum('mode', None, ['train', 'eval'], + 'Running mode: train or eval') + + +def main(argv): + ## Set random seed + run_lib.set_random_seed(FLAGS.config) + + if FLAGS.mode == 'train': + logger = logging.getLogger() + logger.setLevel('INFO') + run_lib.train(FLAGS.config) + elif FLAGS.mode == 'eval': + run_lib.evaluate(FLAGS.config) + else: + raise ValueError(f"Mode {FLAGS.mode} not recognized.") + + +if __name__ == '__main__': + app.run(main) diff --git a/NAS-Bench-201/main_exp/diffusion/run_lib.py b/NAS-Bench-201/main_exp/diffusion/run_lib.py new file mode 100644 index 0000000..3adfab7 --- /dev/null +++ b/NAS-Bench-201/main_exp/diffusion/run_lib.py @@ -0,0 +1,286 @@ +import torch +import sys +import numpy as np +import random + +sys.path.append('.') +import sampling +import datasets_nas +from models import cate +from models import digcn +from models import digcn_meta +from models import utils as mutils +from models.ema import ExponentialMovingAverage +import sde_lib +from utils import * +from analysis.arch_functions import BasicArchMetricsMeta +from all_path import * + + +def get_sampling_fn_meta(config): + ## Set SDE + if config.training.sde.lower() == 'vpsde': + sde = sde_lib.VPSDE( + beta_min=config.model.beta_min, + beta_max=config.model.beta_max, + N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'subvpsde': + sde = sde_lib.subVPSDE( + beta_min=config.model.beta_min, + beta_max=config.model.beta_max, + N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'vesde': + sde = sde_lib.VESDE( + sigma_min=config.model.sigma_min, + sigma_max=config.model.sigma_max, + N=config.model.num_scales) + sampling_eps = 1e-5 + else: + raise NotImplementedError(f"SDE {config.training.sde} unknown.") + + ## Get data normalizer inverse + inverse_scaler = datasets_nas.get_data_inverse_scaler(config) + + ## Get sampling function + sampling_shape = (config.eval.batch_size, config.data.max_node, config.data.n_vocab) + sampling_fn = sampling.get_sampling_fn( + config=config, + sde=sde, + shape=sampling_shape, + inverse_scaler=inverse_scaler, + eps=sampling_eps, + conditional=True, + data_name=config.sampling.check_dataname, + num_sample=config.model.num_sample) + + return sampling_fn, sde + + +def get_score_model(config): + try: + score_config = torch.load(config.scorenet_ckpt_path)['config'] + ckpt_path = config.scorenet_ckpt_path + except: + config.scorenet_ckpt_path = SCORENET_CKPT_PATH + score_config = torch.load(config.scorenet_ckpt_path)['config'] + ckpt_path = config.scorenet_ckpt_path + + score_model = mutils.create_model(score_config) + score_ema = ExponentialMovingAverage( + score_model.parameters(), decay=score_config.model.ema_rate) + score_state = dict( + model=score_model, ema=score_ema, step=0, config=score_config) + score_state = restore_checkpoint( + ckpt_path, score_state, + device=config.device, resume=True) + score_ema.copy_to(score_model.parameters()) + return score_model, score_ema, score_config + + +def get_surrogate(config): + surrogate_model = mutils.create_model(config) + return surrogate_model + + +def get_adj(except_inout=False): + _adj = np.asarray( + [[0, 1, 1, 1, 0, 0, 0, 0], + [0, 0, 0, 0, 1, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 0, 0, 0]] + ) + _adj = torch.tensor(_adj, dtype=torch.float32, device=torch.device('cpu')) + if except_inout: _adj = _adj[1:-1, 1:-1] + return _adj + + +def generate_archs_meta( + config, + sampling_fn, + score_model, + score_ema, + meta_surrogate_model, + num_samples, + args=None, + task=None, + patient_factor=20, + batch_size=256,): + + metrics = BasicArchMetricsMeta() + + ## Get the adj and mask + adj_s = get_adj() + mask_s = aug_mask(adj_s)[0] + adj_c = get_adj() + mask_c = aug_mask(adj_c)[0] + assert (adj_s == adj_c).all() and (mask_s == mask_c).all() + adj_s, mask_s, adj_c, mask_c = \ + adj_s.to(config.device), mask_s.to(config.device), adj_c.to(config.device), mask_c.to(config.device) + + score_ema.copy_to(score_model.parameters()) + score_model.eval() + meta_surrogate_model.eval() + c_scale = args.classifier_scale + + num_sampling_rounds = int(np.ceil(num_samples / batch_size) * patient_factor) if num_samples > batch_size else int(patient_factor) + round = 0 + all_samples = [] + while True and round < num_sampling_rounds: + round += 1 + sample = sampling_fn(score_model, + mask_s, + meta_surrogate_model, + classifier_scale=c_scale, + task=task) + quantized_sample = quantize(sample) + _, _, valid_arch_str, _ = metrics.compute_validity(quantized_sample) + if len(valid_arch_str) > 0: all_samples += valid_arch_str + # to sample various architectures + c_scale -= args.scale_step + seed = int(random.random() * 10000) + reset_seed(seed) + # stop sampling if we have enough samples + if (len(set(all_samples)) >= num_samples): + break + + return list(set(all_samples)) + + +def save_checkpoint(ckpt_dir, state, epoch, is_best): + saved_state = {} + for k in state: + if k in ['optimizer', 'model', 'ema']: + saved_state.update({k: state[k].state_dict()}) + else: + saved_state.update({k: state[k]}) + os.makedirs(ckpt_dir, exist_ok=True) + torch.save(saved_state, os.path.join(ckpt_dir, f'checkpoint_{epoch}.pth.tar')) + if is_best: + shutil.copy(os.path.join(ckpt_dir, f'checkpoint_{epoch}.pth.tar'), os.path.join(ckpt_dir, 'model_best.pth.tar')) + + # remove the ckpt except is_best state + for ckpt_file in sorted(os.listdir(ckpt_dir)): + if not ckpt_file.startswith('checkpoint'): + continue + if os.path.join(ckpt_dir, ckpt_file) != os.path.join(ckpt_dir, 'model_best.pth.tar'): + os.remove(os.path.join(ckpt_dir, ckpt_file)) + + +def restore_checkpoint(ckpt_dir, state, device, resume=False): + if not resume: + os.makedirs(os.path.dirname(ckpt_dir), exist_ok=True) + return state + elif not os.path.exists(ckpt_dir): + if not os.path.exists(os.path.dirname(ckpt_dir)): + os.makedirs(os.path.dirname(ckpt_dir)) + logging.warning(f"No checkpoint found at {ckpt_dir}. " + f"Returned the same state as input") + return state + else: + loaded_state = torch.load(ckpt_dir, map_location=device) + for k in state: + if k in ['optimizer', 'model', 'ema']: + state[k].load_state_dict(loaded_state[k]) + else: + state[k] = loaded_state[k] + return state + + +def floyed(r): + """ + :param r: a numpy NxN matrix with float 0,1 + :return: a numpy NxN matrix with float 0,1 + """ + if type(r) == torch.Tensor: + r = r.cpu().numpy() + N = r.shape[0] + for k in range(N): + for i in range(N): + for j in range(N): + if r[i, k] > 0 and r[k, j] > 0: + r[i, j] = 1 + return r + + +def aug_mask(adj, algo='floyed', data='NASBench201'): + if len(adj.shape) == 2: + adj = adj.unsqueeze(0) + + if data.lower() in ['nasbench201', 'ofa']: + assert len(adj.shape) == 3 + r = adj[0].clone().detach() + if algo == 'long_range': + mask_i = torch.from_numpy(long_range(r)).float().to(adj.device) + elif algo == 'floyed': + mask_i = torch.from_numpy(floyed(r)).float().to(adj.device) + else: + mask_i = r + masks = [mask_i] * adj.size(0) + return torch.stack(masks) + else: + masks = [] + for r in adj: + if algo == 'long_range': + mask_i = torch.from_numpy(long_range(r)).float().to(adj.device) + elif algo == 'floyed': + mask_i = torch.from_numpy(floyed(r)).float().to(adj.device) + else: + mask_i = r + masks.append(mask_i) + return torch.stack(masks) + + +def long_range(r): + """ + :param r: a numpy NxN matrix with float 0,1 + :return: a numpy NxN matrix with float 0,1 + """ + # r = np.array(r) + if type(r) == torch.Tensor: + r = r.cpu().numpy() + N = r.shape[0] + for j in range(1, N): + col_j = r[:, j][:j] + in_to_j = [i for i, val in enumerate(col_j) if val > 0] + if len(in_to_j) > 0: + for i in in_to_j: + col_i = r[:, i][:i] + in_to_i = [i for i, val in enumerate(col_i) if val > 0] + if len(in_to_i) > 0: + for k in in_to_i: + r[k, j] = 1 + return r + + +def quantize(x): + """Covert the PyTorch tensor x, adj matrices to numpy array. + + Args: + x: [Batch_size, Max_node, N_vocab] + """ + x_list = [] + + # discretization + x[x >= 0.5] = 1. + x[x < 0.5] = 0. + + for i in range(x.shape[0]): + x_tmp = x[i] + x_tmp = x_tmp.cpu().numpy() + x_list.append(x_tmp) + + return x_list + + +def reset_seed(seed): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + random.seed(seed) + torch.backends.cudnn.deterministic = True \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/logger.py b/NAS-Bench-201/main_exp/logger.py new file mode 100644 index 0000000..b353efe --- /dev/null +++ b/NAS-Bench-201/main_exp/logger.py @@ -0,0 +1,137 @@ +import os +import wandb +import torch +import numpy as np + + +class Logger: + def __init__( + self, + log_dir=None, + write_textfile=True + ): + + self.log_dir = log_dir + self.write_textfile = write_textfile + + self.logs_for_save = {} + self.logs = {} + + if self.write_textfile: + self.f = open(os.path.join(log_dir, 'logs.txt'), 'w') + + + def write_str(self, log_str): + self.f.write(log_str+'\n') + self.f.flush() + + + def update_config(self, v, is_args=False): + if is_args: + self.logs_for_save.update({'args': v}) + else: + self.logs_for_save.update(v) + + + def write_log(self, element, step, return_log_dict=False): + log_str = f"{step} | " + log_dict = {} + for head, keys in element.items(): + for k in keys: + if k in self.logs: + v = self.logs[k].avg + if not k in self.logs_for_save: + self.logs_for_save[k] = [] + self.logs_for_save[k].append(v) + log_str += f'{k} {v}| ' + log_dict[f'{head}/{k}'] = v + + if self.write_textfile: + self.f.write(log_str+'\n') + self.f.flush() + + if return_log_dict: + return log_dict + + + def save_log(self, name=None): + name = 'logs.pt' if name is None else name + torch.save(self.logs_for_save, os.path.join(self.log_dir, name)) + + + def update(self, key, v, n=1): + if not key in self.logs: + self.logs[key] = AverageMeter() + self.logs[key].update(v, n) + + + def reset(self, keys=None, except_keys=[]): + if keys is not None: + if isinstance(keys, list): + for key in keys: + self.logs[key] = AverageMeter() + else: + self.logs[keys] = AverageMeter() + else: + for key in self.logs.keys(): + if not key in except_keys: + self.logs[key] = AverageMeter() + + + def avg(self, keys=None, except_keys=[]): + if keys is not None: + if isinstance(keys, list): + return {key: self.logs[key].avg for key in keys if key in self.logs.keys()} + else: + return self.logs[keys].avg + else: + avg_dict = {} + for key in self.logs.keys(): + if not key in except_keys: + avg_dict[key] = self.logs[key].avg + return avg_dict + + +class AverageMeter(object): + """ + Computes and stores the average and current value + Copied from: https://github.com/pytorch/examples/blob/master/imagenet/main.py + """ + + def __init__(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def get_metrics(g_embeds, x_embeds, logit_scale, prefix='train'): + metrics = {} + logits_per_g = (logit_scale * g_embeds @ x_embeds.t()).detach().cpu() + logits_per_x = logits_per_g.t().detach().cpu() + + logits = {"g_to_x": logits_per_g, "x_to_g": logits_per_x} + ground_truth = torch.arange(len(x_embeds)).view(-1, 1) + + for name, logit in logits.items(): + ranking = torch.argsort(logit, descending=True) + preds = torch.where(ranking == ground_truth)[1] + preds = preds.detach().cpu().numpy() + metrics[f"{prefix}_{name}_mean_rank"] = preds.mean() + 1 + metrics[f"{prefix}_{name}_median_rank"] = np.floor(np.median(preds)) + 1 + for k in [1, 5, 10]: + metrics[f"{prefix}_{name}_R@{k}"] = np.mean(preds < k) + + return metrics \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/get_files/get_aircraft.py b/NAS-Bench-201/main_exp/transfer_nag/get_files/get_aircraft.py new file mode 100644 index 0000000..9d02170 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/get_files/get_aircraft.py @@ -0,0 +1,63 @@ +""" +@author: Hayeon Lee +2020/02/19 +Script for downloading, and reorganizing aircraft +for few shot classification +Run this file as follows: + python get_data.py +""" + +import pickle +import os +import numpy as np +from tqdm import tqdm +import requests +import tarfile +from PIL import Image +import glob +import shutil +import pickle +import collections +import sys +sys.path.append(os.path.join(os.getcwd(), 'main_exp')) +from all_path import RAW_DATA_PATH + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + +dir_path = RAW_DATA_PATH +if not os.path.exists(dir_path): + os.makedirs(dir_path) +file_name = os.path.join(dir_path, 'fgvc-aircraft-2013b.tar.gz') + +if not os.path.exists(file_name): + print(f"Downloading {file_name}\n") + download_file( + 'http://www.robots.ox.ac.uk/~vgg/data/fgvc-aircraft/archives/fgvc-aircraft-2013b.tar.gz', + file_name) + print("\nDownloading done.\n") +else: + print("fgvc-aircraft-2013b.tar.gz has already been downloaded. Did not download twice.\n") + +untar_file_name = os.path.join(dir_path, 'aircraft') +if not os.path.exists(untar_file_name): + tarname = file_name + print("Untarring: {}".format(tarname)) + tar = tarfile.open(tarname) + tar.extractall(untar_file_name) + tar.close() +else: + print(f"{untar_file_name} folder already exists. Did not untarring twice\n") +os.remove(file_name) diff --git a/NAS-Bench-201/main_exp/transfer_nag/get_files/get_pets.py b/NAS-Bench-201/main_exp/transfer_nag/get_files/get_pets.py new file mode 100644 index 0000000..1a43e7d --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/get_files/get_pets.py @@ -0,0 +1,50 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests +import zipfile +import sys +sys.path.append(os.path.join(os.getcwd(), 'main_exp')) +from all_path import RAW_DATA_PATH + + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm(unit="B", total=int(r.headers['Content-Length'])) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update(len(chunk)) + f.write(chunk) + return filename + + +dir_path = os.path.join(RAW_DATA_PATH, 'pets') +if not os.path.exists(dir_path): + os.makedirs(dir_path) + +full_name = os.path.join(dir_path, 'test15.pth') +if not os.path.exists(full_name): + print(f"Downloading {full_name}\n") + download_file( + 'https://www.dropbox.com/s/kzmrwyyk5iaugv0/test15.pth?dl=1', full_name) + print("Downloading done.\n") +else: + print(f"{full_name} has already been downloaded. Did not download twice.\n") + +full_name = os.path.join(dir_path, 'train85.pth') +if not os.path.exists(full_name): + print(f"Downloading {full_name}\n") + download_file( + 'https://www.dropbox.com/s/w7mikpztkamnw9s/train85.pth?dl=1', full_name) + print("Downloading done.\n") +else: + print(f"{full_name} has already been downloaded. Did not download twice.\n") diff --git a/NAS-Bench-201/main_exp/transfer_nag/get_files/get_preprocessed_data.py b/NAS-Bench-201/main_exp/transfer_nag/get_files/get_preprocessed_data.py new file mode 100644 index 0000000..d8edb98 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/get_files/get_preprocessed_data.py @@ -0,0 +1,47 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import os +from tqdm import tqdm +import requests + + +DATA_PATH = "./data/transfer_nag" +dir_path = DATA_PATH +if not os.path.exists(dir_path): + os.makedirs(dir_path) + + +def download_file(url, filename): + """ + Helper method handling downloading large files from `url` + to `filename`. Returns a pointer to `filename`. + """ + chunkSize = 1024 + r = requests.get(url, stream=True) + with open(filename, 'wb') as f: + pbar = tqdm( unit="B", total=int( r.headers['Content-Length'] ) ) + for chunk in r.iter_content(chunk_size=chunkSize): + if chunk: # filter out keep-alive new chunks + pbar.update (len(chunk)) + f.write(chunk) + return filename + + +def get_preprocessed_data(file_name, url): + print(f"Downloading {file_name} datasets\n") + full_name = os.path.join(dir_path, file_name) + download_file(url, full_name) + print("Downloading done.\n") + + +for file_name, url in [ + ('aircraftbylabel.pt', 'https://www.dropbox.com/s/mb66kitv20ykctp/aircraftbylabel.pt?dl=1'), + ('cifar100bylabel.pt', 'https://www.dropbox.com/s/y0xahxgzj29kffk/cifar100bylabel.pt?dl=1'), + ('cifar10bylabel.pt', 'https://www.dropbox.com/s/wt1pcwi991xyhwr/cifar10bylabel.pt?dl=1'), + ('imgnet32bylabel.pt', 'https://www.dropbox.com/s/7r3hpugql8qgi9d/imgnet32bylabel.pt?dl=1'), + ('petsbylabel.pt', 'https://www.dropbox.com/s/mxh6qz3grhy7wcn/petsbylabel.pt?dl=1'), + ]: + + get_preprocessed_data(file_name, url) diff --git a/NAS-Bench-201/main_exp/transfer_nag/loader.py b/NAS-Bench-201/main_exp/transfer_nag/loader.py new file mode 100644 index 0000000..80f248f --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/loader.py @@ -0,0 +1,130 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import os +import torch +from torch.utils.data import Dataset +from torch.utils.data import DataLoader + + +def get_meta_train_loader(batch_size, data_path, num_sample, is_pred=True): + dataset = MetaTrainDatabase(data_path, num_sample, is_pred) + print(f'==> The number of tasks for meta-training: {len(dataset)}') + + loader = DataLoader(dataset=dataset, + batch_size=batch_size, + shuffle=True, + num_workers=0, + collate_fn=collate_fn) + return loader + + +def get_meta_test_loader(data_path, data_name, num_class=None, is_pred=False): + dataset = MetaTestDataset(data_path, data_name, num_class) + print(f'==> Meta-Test dataset {data_name}') + + loader = DataLoader(dataset=dataset, + batch_size=100, + shuffle=False, + num_workers=0) + return loader + + +class MetaTrainDatabase(Dataset): + def __init__(self, data_path, num_sample, is_pred=True): + self.mode = 'train' + self.acc_norm = True + self.num_sample = num_sample + self.x = torch.load(os.path.join(data_path, 'imgnet32bylabel.pt')) + + mtr_data_path = os.path.join( + data_path, 'meta_train_tasks_predictor.pt') + idx_path = os.path.join( + data_path, 'meta_train_tasks_predictor_idx.pt') + data = torch.load(mtr_data_path) + self.acc = data['acc'] + self.task = data['task'] + self.graph = data['g'] + + random_idx_lst = torch.load(idx_path) + self.idx_lst = {} + self.idx_lst['valid'] = random_idx_lst[:400] + self.idx_lst['train'] = random_idx_lst[400:] + self.acc = torch.tensor(self.acc) + self.mean = torch.mean(self.acc[self.idx_lst['train']]).item() + self.std = torch.std(self.acc[self.idx_lst['train']]).item() + self.task_lst = torch.load(os.path.join( + data_path, 'meta_train_task_lst.pt')) + + + def set_mode(self, mode): + self.mode = mode + + + def __len__(self): + return len(self.idx_lst[self.mode]) + + + def __getitem__(self, index): + data = [] + ridx = self.idx_lst[self.mode] + tidx = self.task[ridx[index]] + classes = self.task_lst[tidx] + graph = self.graph[ridx[index]] + acc = self.acc[ridx[index]] + for cls in classes: + cx = self.x[cls-1][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + x = torch.cat(data) + if self.acc_norm: + acc = ((acc - self.mean) / self.std) / 100.0 + else: + acc = acc / 100.0 + return x, graph, acc + + +class MetaTestDataset(Dataset): + def __init__(self, data_path, data_name, num_sample, num_class=None): + self.num_sample = num_sample + self.data_name = data_name + + num_class_dict = { + 'cifar100': 100, + 'cifar10': 10, + 'mnist': 10, + 'svhn': 10, + 'aircraft': 30, + 'pets': 37 + } + + if num_class is not None: + self.num_class = num_class + else: + self.num_class = num_class_dict[data_name] + + self.x = torch.load(os.path.join(data_path, f'{data_name}bylabel.pt')) + + + def __len__(self): + return 1000000 + + + def __getitem__(self, index): + data = [] + classes = list(range(self.num_class)) + for cls in classes: + cx = self.x[cls][0] + ridx = torch.randperm(len(cx)) + data.append(cx[ridx[:self.num_sample]]) + x = torch.cat(data) + return x + + +def collate_fn(batch): + x = torch.stack([item[0] for item in batch]) + graph = [item[1] for item in batch] + acc = torch.stack([item[2] for item in batch]) + return [x, graph, acc] diff --git a/NAS-Bench-201/main_exp/transfer_nag/main.py b/NAS-Bench-201/main_exp/transfer_nag/main.py new file mode 100644 index 0000000..3c863a4 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/main.py @@ -0,0 +1,91 @@ +import os +import sys +import random +import numpy as np +import argparse +import torch +import os +from nag import NAG +sys.path.append(os.getcwd()) +save_path = "results" +data_path = os.path.join('MetaD2A_nas_bench_201', 'data') + + +def str2bool(v): + return v.lower() in ['t', 'true', True] + + +def get_parser(): + parser = argparse.ArgumentParser() + # general settings + parser.add_argument('--seed', type=int, default=444) + parser.add_argument('--gpu', type=str, default='0', help='set visible gpus') + parser.add_argument('--save-path', type=str, default=save_path, help='the path of save directory') + parser.add_argument('--data-path', type=str, default=data_path, help='the path of save directory') + parser.add_argument('--model-load-path', type=str, default='', help='') + parser.add_argument('--save-epoch', type=int, default=20, help='how many epochs to wait each time to save model states') + parser.add_argument('--max-epoch', type=int, default=1000, help='number of epochs to train') + parser.add_argument('--batch_size', type=int, default=1024, help='batch size for generator') + parser.add_argument('--graph-data-name', default='nasbench201', help='graph dataset name') + parser.add_argument('--nvt', type=int, default=7, help='number of different node types, 7: NAS-Bench-201 including in/out node') + # set encoder + parser.add_argument('--num-sample', type=int, default=20, help='the number of images as input for set encoder') + # graph encoder + parser.add_argument('--hs', type=int, default=512, help='hidden size of GRUs') + parser.add_argument('--nz', type=int, default=56, help='the number of dimensions of latent vectors z') + # test + parser.add_argument('--test', action='store_true', default=True, help='turn on test mode') + parser.add_argument('--load-epoch', type=int, default=100, help='checkpoint epoch loaded for meta-test') + parser.add_argument('--data-name', type=str, default='pets', help='meta-test dataset name') + parser.add_argument('--trials', type=int, default=20) + parser.add_argument('--num-class', type=int, default=None, help='the number of class of dataset') + parser.add_argument('--num-gen-arch', type=int, default=500, help='the number of candidate architectures generated by the generator') + parser.add_argument('--train-arch', type=str2bool, default=True, help='whether to train the searched architecture') + parser.add_argument('--n_init', type=int, default=10) + parser.add_argument('--N', type=int, default=1) + # DiffusionNAG + parser.add_argument('--folder_name', type=str, default='debug') + parser.add_argument('--exp_name', type=str, default='') + parser.add_argument('--classifier_scale', type=float, default=10000., help='classifier scale') + parser.add_argument('--scale_step', type=float, default=300.) + parser.add_argument('--eval_batch_size', type=int, default=256) + parser.add_argument('--predictor', type=str, default='euler_maruyama', choices=['euler_maruyama', 'reverse_diffusion', 'none']) + parser.add_argument('--corrector', type=str, default='langevin', choices=['none', 'langevin']) + parser.add_argument('--patient_factor', type=int, default=20) + parser.add_argument('--n_gen_samples', type=int, default=10) + parser.add_argument('--multi_proc', type=str2bool, default=True) + + args = parser.parse_args() + return args + + +def set_exp_name(args): + exp_name = f'./results/transfer_nag/{args.folder_name}/{args.data_name}' + os.makedirs(exp_name, exist_ok=True) + args.exp_name = exp_name + + +def main(): + ## Get arguments + args = get_parser() + + ## Set gpus and seeds + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu + torch.cuda.manual_seed(args.seed) + torch.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(args.seed) + random.seed(args.seed) + + ## Set experiment name + set_exp_name(args) + + ## Run + nag = NAG(args) + nag.meta_test() + + +if __name__ == '__main__': + main() diff --git a/NAS-Bench-201/main_exp/transfer_nag/nag.py b/NAS-Bench-201/main_exp/transfer_nag/nag.py new file mode 100644 index 0000000..d9e8297 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nag.py @@ -0,0 +1,305 @@ +from __future__ import print_function +import torch +import os +import gc +import sys +import numpy as np +import os +import subprocess + +from nag_utils import mean_confidence_interval +from nag_utils import restore_checkpoint +from nag_utils import load_graph_config +from nag_utils import load_model + +sys.path.append(os.path.join(os.getcwd(), 'main_exp')) +from nas_bench_201 import train_single_model +from unnoised_model import MetaSurrogateUnnoisedModel +from diffusion.run_lib import generate_archs_meta +from diffusion.run_lib import get_sampling_fn_meta +from diffusion.run_lib import get_score_model +from diffusion.run_lib import get_surrogate +from loader import MetaTestDataset +from logger import Logger +from all_path import * + + +class NAG: + def __init__(self, args): + self.args = args + self.device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu") + + ## Target dataset information + self.raw_data_path = RAW_DATA_PATH + self.data_path = DATA_PATH + self.data_name = args.data_name + self.num_class = args.num_class + self.num_sample = args.num_sample + + graph_config = load_graph_config(args.graph_data_name, args.nvt, NASBENCH201) + self.meta_surrogate_unnoised_model = MetaSurrogateUnnoisedModel(args, graph_config) + load_model(model=self.meta_surrogate_unnoised_model, + ckpt_path=META_SURROGATE_UNNOISED_CKPT_PATH) + self.meta_surrogate_unnoised_model.to(self.device) + + ## Load pre-trained meta-surrogate model + self.meta_surrogate_ckpt_path = META_SURROGATE_CKPT_PATH + + ## Load score network model (base diffusion model) + self.load_diffusion_model(args=args) + + ## Check config + self.check_config() + + ## Set logger + self.logger = Logger( + log_dir=args.exp_name, + write_textfile=True + ) + self.logger.update_config(args, is_args=True) + self.logger.write_str(str(vars(args))) + self.logger.write_str('-' * 100) + + + def check_config(self): + """ + Check if the configuration of the pre-trained score network model matches that of the meta surrogate model. + """ + scorenet_config = torch.load(self.config.scorenet_ckpt_path)['config'] + meta_surrogate_config = torch.load(self.meta_surrogate_ckpt_path)['config'] + assert scorenet_config.model.sigma_min == meta_surrogate_config.model.sigma_min + assert scorenet_config.model.sigma_max == meta_surrogate_config.model.sigma_max + assert scorenet_config.training.sde == meta_surrogate_config.training.sde + assert scorenet_config.training.continuous == meta_surrogate_config.training.continuous + assert scorenet_config.data.centered == meta_surrogate_config.data.centered + assert scorenet_config.data.max_node == meta_surrogate_config.data.max_node + assert scorenet_config.data.n_vocab == meta_surrogate_config.data.n_vocab + + + def forward(self, x, arch): + D_mu = self.meta_surrogate_unnoised_model.set_encode(x.to(self.device)) + G_mu = self.meta_surrogate_unnoised_model.graph_encode(arch) + y_pred = self.meta_surrogate_unnoised_model.predict(D_mu, G_mu) + return y_pred + + + def meta_test(self): + if self.data_name == 'all': + for data_name in ['cifar10', 'cifar100', 'aircraft', 'pets']: + self.meta_test_per_dataset(data_name) + else: + self.meta_test_per_dataset(self.data_name) + + + def meta_test_per_dataset(self, data_name): + ## Load NASBench201 + self.nasbench201 = torch.load(NASBENCH201) + all_arch_str = np.array(self.nasbench201['arch']['str']) + + ## Load meta-test dataset + self.test_dataset = MetaTestDataset(self.data_path, data_name, self.num_sample, self.num_class) + + ## Set save path + meta_test_path = os.path.join(META_TEST_PATH, data_name) + os.makedirs(meta_test_path, exist_ok=True) + f_arch_str = open(os.path.join(self.args.exp_name, 'architecture.txt'), 'w') + f_arch_acc = open(os.path.join(self.args.exp_name, 'accuracy.txt'), 'w') + + ## Generate architectures + gen_arch_str = self.get_gen_arch_str() + gen_arch_igraph = self.get_items( + full_target=self.nasbench201['arch']['igraph'], + full_source=self.nasbench201['arch']['str'], + source=gen_arch_str) + + ## Sort with unnoised meta-surrogate model + y_pred_all = [] + self.meta_surrogate_unnoised_model.eval() + self.meta_surrogate_unnoised_model.to(self.device) + with torch.no_grad(): + for arch_igraph in gen_arch_igraph: + x, g = self.collect_data(arch_igraph) + y_pred = self.forward(x, g) + y_pred = torch.mean(y_pred) + y_pred_all.append(y_pred.cpu().detach().item()) + sorted_arch_lst = self.sort_arch(data_name, torch.tensor(y_pred_all), gen_arch_str) + + ## Record the information of the architecture generated in sorted order + for _, arch_str in enumerate(sorted_arch_lst): + f_arch_str.write(f'{arch_str}\n') + arch_idx_lst = [self.nasbench201['arch']['str'].index(i) for i in sorted_arch_lst] + arch_str_lst = [] + arch_acc_lst = [] + + ## Get the accuracy of the architecture + if 'cifar' in data_name: + sorted_acc_lst = self.get_items( + full_target=self.nasbench201['test-acc'][data_name], + full_source=self.nasbench201['arch']['str'], + source=sorted_arch_lst) + arch_str_lst += sorted_arch_lst + arch_acc_lst += sorted_acc_lst + for arch_idx, acc in zip(arch_idx_lst, sorted_acc_lst): + msg = f'Avg {acc:4f} (%)' + f_arch_acc.write(msg + '\n') + else: + if self.args.multi_proc: + ## Run multiple processes in parallel + run_file = os.path.join(os.getcwd(), 'main_exp', 'transfer_nag', 'run_multi_proc.py') + MAX_CAP = 5 # hard-coded for available GPUs + if not len(arch_idx_lst) > MAX_CAP: + arch_idx_lst_ = [arch_idx for arch_idx in arch_idx_lst if not os.path.exists(os.path.join(meta_test_path, str(arch_idx)))] + support_ = ','.join([str(i) for i in arch_idx_lst_]) + num_split = int(3 * len(arch_idx_lst_)) # why 3? => running for 3 seeds + cmd = f"python {run_file} --num_split {num_split} --arch_idx_lst {support_} --meta_test_path {meta_test_path} --data_name {data_name} --raw_data_path {self.raw_data_path}" + subprocess.run([cmd], shell=True) + else: + arch_idx_lst_ = [] + for j, arch_idx in enumerate(arch_idx_lst): + if not os.path.exists(os.path.join(meta_test_path, str(arch_idx))): + arch_idx_lst_.append(arch_idx) + if (len(arch_idx_lst_) == MAX_CAP) or (j == len(arch_idx_lst) - 1): + support_ = ','.join([str(i) for i in arch_idx_lst_]) + num_split = int(3 * len(arch_idx_lst_)) + cmd = f"python {run_file} --num_split {num_split} --arch_idx_lst {support_} --meta_test_path {meta_test_path} --data_name {data_name} --raw_data_path {self.raw_data_path}" + subprocess.run([cmd], shell=True) + arch_idx_lst_ = [] + + while True: + try: + acc_runs_lst = [] + epoch = 199 + seeds = (777, 888, 999) + for arch_idx in arch_idx_lst: + acc_runs = [] + save_path_ = os.path.join(meta_test_path, str(arch_idx)) + for seed in seeds: + result = torch.load(os.path.join(save_path_, f'seed-0{seed}.pth')) + acc_runs.append(result[data_name]['valid_acc1es'][f'x-test@{epoch}']) + acc_runs_lst.append(acc_runs) + break + except: + pass + for i in acc_runs_lst:print(np.mean(i)) + for arch_idx, acc_runs in zip(arch_idx_lst, acc_runs_lst): + for r, acc in enumerate(acc_runs): + msg = f'run {r+1} {acc:.2f} (%)' + f_arch_acc.write(msg + '\n') + m, h = mean_confidence_interval(acc_runs) + msg = f'Avg {m:.2f}+-{h.item():.2f} (%)' + f_arch_acc.write(msg + '\n') + arch_acc_lst.append(np.mean(acc_runs)) + arch_str_lst.append(all_arch_str[arch_idx]) + + else: + for arch_idx in arch_idx_lst: + acc_runs = self.train_single_arch( + data_name, self.nasbench201['str'][arch_idx], meta_test_path) + for r, acc in enumerate(acc_runs): + msg = f'run {r+1} {acc:.2f} (%)' + f_arch_acc.write(msg + '\n') + m, h = mean_confidence_interval(acc_runs) + msg = f'Avg {m:.2f}+-{h.item():.2f} (%)' + f_arch_acc.write(msg + '\n') + arch_acc_lst.append(np.mean(acc_runs)) + arch_str_lst.append(all_arch_str[arch_idx]) + + # Save results + results_path = os.path.join(self.args.exp_name, 'results.pt') + torch.save({ + 'arch_idx_lst': arch_idx_lst, + 'arch_str_lst': arch_str_lst, + 'arch_acc_lst': arch_acc_lst + }, results_path) + print(f">>> Save the results at {results_path}...") + + + def train_single_arch(self, data_name, arch_str, meta_test_path): + save_path = os.path.join(meta_test_path, arch_str) + seeds = (777, 888, 999) + train_single_model(save_dir=save_path, + workers=24, + datasets=[data_name], + xpaths=[f'{self.raw_data_path}/{data_name}'], + splits=[0], + use_less=False, + seeds=seeds, + model_str=arch_str, + arch_config={'channel': 16, 'num_cells': 5}) + epoch = 199 + test_acc_lst = [] + for seed in seeds: + result = torch.load(os.path.join(save_path, f'seed-0{seed}.pth')) + test_acc_lst.append(result[data_name]['valid_acc1es'][f'x-test@{epoch}']) + return test_acc_lst + + + def sort_arch(self, data_name, y_pred_all, gen_arch_str): + _, sorted_idx = torch.sort(y_pred_all, descending=True) + sotred_gen_arch_str = [gen_arch_str[_] for _ in sorted_idx] + return sotred_gen_arch_str + + + def collect_data_only(self): + x_batch = [] + x_batch.append(self.test_dataset[0]) + return torch.stack(x_batch).to(self.device) + + + def collect_data(self, arch_igraph): + x_batch, g_batch = [], [] + for _ in range(10): + x_batch.append(self.test_dataset[0]) + g_batch.append(arch_igraph) + return torch.stack(x_batch).to(self.device), g_batch + + + def get_items(self, full_target, full_source, source): + return [full_target[full_source.index(_)] for _ in source] + + + def load_diffusion_model(self, args): + self.config = torch.load('./configs/transfer_nag_config.pt') + self.config.device = torch.device('cuda') + self.config.data.label_list = ['meta-acc'] + self.config.scorenet_ckpt_path = SCORENET_CKPT_PATH + self.config.sampling.classifier_scale = args.classifier_scale + self.config.eval.batch_size = args.eval_batch_size + self.config.sampling.predictor = args.predictor + self.config.sampling.corrector = args.corrector + self.config.sampling.check_dataname = self.data_name + self.sampling_fn, self.sde = get_sampling_fn_meta(self.config) + self.score_model, self.score_ema, self.score_config = get_score_model(self.config) + + + def get_gen_arch_str(self): + ## Load meta-surrogate model + meta_surrogate_config = torch.load(self.meta_surrogate_ckpt_path)['config'] + meta_surrogate_model = get_surrogate(meta_surrogate_config) + meta_surrogate_state = dict(model=meta_surrogate_model, step=0, config=meta_surrogate_config) + meta_surrogate_state = restore_checkpoint( + self.meta_surrogate_ckpt_path, + meta_surrogate_state, + device=self.config.device, + resume=True) + + ## Get dataset embedding, x + with torch.no_grad(): + x = self.collect_data_only() + + ## Generate architectures + generated_arch_str = generate_archs_meta( + config=self.config, + sampling_fn=self.sampling_fn, + score_model=self.score_model, + score_ema=self.score_ema, + meta_surrogate_model=meta_surrogate_model, + num_samples=self.args.n_gen_samples, + args=self.args, + task=x) + + ## Clean up + meta_surrogate_model = None + gc.collect() + + return generated_arch_str \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nag_utils.py b/NAS-Bench-201/main_exp/transfer_nag/nag_utils.py new file mode 100644 index 0000000..8422057 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nag_utils.py @@ -0,0 +1,301 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import os +import time +import igraph +import random +import numpy as np +import scipy.stats +import torch +import logging + + +def reset_seed(seed): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + random.seed(seed) + torch.backends.cudnn.deterministic = True + + +def restore_checkpoint(ckpt_dir, state, device, resume=False): + if not resume: + os.makedirs(os.path.dirname(ckpt_dir), exist_ok=True) + return state + elif not os.path.exists(ckpt_dir): + if not os.path.exists(os.path.dirname(ckpt_dir)): + os.makedirs(os.path.dirname(ckpt_dir)) + logging.warning(f"No checkpoint found at {ckpt_dir}. " + f"Returned the same state as input") + return state + else: + loaded_state = torch.load(ckpt_dir, map_location=device) + for k in state: + if k in ['optimizer', 'model', 'ema']: + state[k].load_state_dict(loaded_state[k]) + else: + state[k] = loaded_state[k] + return state + + +def load_graph_config(graph_data_name, nvt, data_path): + if graph_data_name is not 'nasbench201': + raise NotImplementedError(graph_data_name) + g_list = [] + max_n = 0 # maximum number of nodes + ms = torch.load(data_path)['arch']['matrix'] + for i in range(len(ms)): + g, n = decode_NAS_BENCH_201_8_to_igraph(ms[i]) + max_n = max(max_n, n) + g_list.append((g, 0)) + # number of different node types including in/out node + graph_config = {} + graph_config['num_vertex_type'] = nvt # original types + start/end types + graph_config['max_n'] = max_n # maximum number of nodes + graph_config['START_TYPE'] = 0 # predefined start vertex type + graph_config['END_TYPE'] = 1 # predefined end vertex type + + return graph_config + + +def decode_NAS_BENCH_201_8_to_igraph(row): + if type(row) == str: + row = eval(row) # convert string to list of lists + n = len(row) + g = igraph.Graph(directed=True) + g.add_vertices(n) + for i, node in enumerate(row): + g.vs[i]['type'] = node[0] + if i < (n - 2) and i > 0: + g.add_edge(i, i + 1) # always connect from last node + for j, edge in enumerate(node[1:]): + if edge == 1: + g.add_edge(j, i) + return g, n + + +def is_valid_NAS201(g, START_TYPE=0, END_TYPE=1): + # first need to be a valid DAG computation graph + res = is_valid_DAG(g, START_TYPE, END_TYPE) + # in addition, node i must connect to node i+1 + res = res and len(g.vs['type']) == 8 + res = res and not (0 in g.vs['type'][1:-1]) + res = res and not (1 in g.vs['type'][1:-1]) + return res + + +def decode_igraph_to_NAS201_matrix(g): + m = [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]] + xys = [(1, 0), (2, 0), (2, 1), (3, 0), (3, 1), (3, 2)] + for i, xy in enumerate(xys): + m[xy[0]][xy[1]] = float(g.vs[i + 1]['type']) - 2 + import numpy + return numpy.array(m) + + +def decode_igraph_to_NAS_BENCH_201_string(g): + if not is_valid_NAS201(g): + return None + m = decode_igraph_to_NAS201_matrix(g) + types = ['none', 'skip_connect', 'nor_conv_1x1', + 'nor_conv_3x3', 'avg_pool_3x3'] + return '|{}~0|+|{}~0|{}~1|+|{}~0|{}~1|{}~2|'.\ + format(types[int(m[1][0])], + types[int(m[2][0])], types[int(m[2][1])], + types[int(m[3][0])], types[int(m[3][1])], types[int(m[3][2])]) + + +def is_valid_DAG(g, START_TYPE=0, END_TYPE=1): + res = g.is_dag() + n_start, n_end = 0, 0 + for v in g.vs: + if v['type'] == START_TYPE: + n_start += 1 + elif v['type'] == END_TYPE: + n_end += 1 + if v.indegree() == 0 and v['type'] != START_TYPE: + return False + if v.outdegree() == 0 and v['type'] != END_TYPE: + return False + return res and n_start == 1 and n_end == 1 + + +class Accumulator(): + def __init__(self, *args): + self.args = args + self.argdict = {} + for i, arg in enumerate(args): + self.argdict[arg] = i + self.sums = [0] * len(args) + self.cnt = 0 + + def accum(self, val): + val = [val] if type(val) is not list else val + val = [v for v in val if v is not None] + assert (len(val) == len(self.args)) + for i in range(len(val)): + if torch.is_tensor(val[i]): + val[i] = val[i].item() + self.sums[i] += val[i] + self.cnt += 1 + + def clear(self): + self.sums = [0] * len(self.args) + self.cnt = 0 + + def get(self, arg, avg=True): + i = self.argdict.get(arg, -1) + assert (i is not -1) + if avg: + return self.sums[i] / (self.cnt + 1e-8) + else: + return self.sums[i] + + def print_(self, header=None, time=None, + logfile=None, do_not_print=[], as_int=[], + avg=True): + msg = '' if header is None else header + ': ' + if time is not None: + msg += ('(%.3f secs), ' % time) + + args = [arg for arg in self.args if arg not in do_not_print] + arg = [] + for arg in args: + val = self.sums[self.argdict[arg]] + if avg: + val /= (self.cnt + 1e-8) + if arg in as_int: + msg += ('%s %d, ' % (arg, int(val))) + else: + msg += ('%s %.4f, ' % (arg, val)) + print(msg) + + if logfile is not None: + logfile.write(msg + '\n') + logfile.flush() + + def add_scalars(self, summary, header=None, tag_scalar=None, + step=None, avg=True, args=None): + for arg in self.args: + val = self.sums[self.argdict[arg]] + if avg: + val /= (self.cnt + 1e-8) + else: + val = val + tag = f'{header}/{arg}' if header is not None else arg + if tag_scalar is not None: + summary.add_scalars(main_tag=tag, + tag_scalar_dict={tag_scalar: val}, + global_step=step) + else: + summary.add_scalar(tag=tag, + scalar_value=val, + global_step=step) + + +class Log: + def __init__(self, args, logf, summary=None): + self.args = args + self.logf = logf + self.summary = summary + self.stime = time.time() + self.ep_sttime = None + + def print(self, logger, epoch, tag=None, avg=True): + if tag == 'train': + ct = time.time() - self.ep_sttime + tt = time.time() - self.stime + msg = f'[total {tt:6.2f}s (ep {ct:6.2f}s)] epoch {epoch:3d}' + print(msg) + self.logf.write(msg+'\n') + logger.print_(header=tag, logfile=self.logf, avg=avg) + + if self.summary is not None: + logger.add_scalars( + self.summary, header=tag, step=epoch, avg=avg) + logger.clear() + + def print_args(self): + argdict = vars(self.args) + print(argdict) + for k, v in argdict.items(): + self.logf.write(k + ': ' + str(v) + '\n') + self.logf.write('\n') + + def set_time(self): + self.stime = time.time() + + def save_time_log(self): + ct = time.time() - self.stime + msg = f'({ct:6.2f}s) meta-training phase done' + print(msg) + self.logf.write(msg+'\n') + + def print_pred_log(self, loss, corr, tag, epoch=None, max_corr_dict=None): + if tag == 'train': + ct = time.time() - self.ep_sttime + tt = time.time() - self.stime + msg = f'[total {tt:6.2f}s (ep {ct:6.2f}s)] epoch {epoch:3d}' + self.logf.write(msg+'\n') + print(msg) + self.logf.flush() + # msg = f'ep {epoch:3d} ep time {time.time() - ep_sttime:8.2f} ' + # msg += f'time {time.time() - sttime:6.2f} ' + if max_corr_dict is not None: + max_corr = max_corr_dict['corr'] + max_loss = max_corr_dict['loss'] + msg = f'{tag}: loss {loss:.6f} ({max_loss:.6f}) ' + msg += f'corr {corr:.4f} ({max_corr:.4f})' + else: + msg = f'{tag}: loss {loss:.6f} corr {corr:.4f}' + self.logf.write(msg+'\n') + print(msg) + self.logf.flush() + + def max_corr_log(self, max_corr_dict): + corr = max_corr_dict['corr'] + loss = max_corr_dict['loss'] + epoch = max_corr_dict['epoch'] + msg = f'[epoch {epoch}] max correlation: {corr:.4f}, loss: {loss:.6f}' + self.logf.write(msg+'\n') + print(msg) + self.logf.flush() + + +def get_log(epoch, loss, y_pred, y, acc_std, acc_mean, tag='train'): + msg = f'[{tag}] Ep {epoch} loss {loss.item()/len(y):0.4f} ' + if type(y_pred) == list: + msg += f'pacc {y_pred[0]:0.4f}' + msg += f'({y_pred[0]*100.0*acc_std+acc_mean:0.4f}) ' + else: + msg += f'pacc {y_pred:0.4f}' + msg += f'({y_pred*100.0*acc_std+acc_mean:0.4f}) ' + msg += f'acc {y[0]:0.4f}({y[0]*100*acc_std+acc_mean:0.4f})' + return msg + + +def load_model(model, ckpt_path): + model.cpu() + model.load_state_dict(torch.load(ckpt_path)) + + +def save_model(epoch, model, model_path, max_corr=None): + print("==> save current model...") + if max_corr is not None: + torch.save(model.cpu().state_dict(), + os.path.join(model_path, 'ckpt_max_corr.pt')) + else: + torch.save(model.cpu().state_dict(), + os.path.join(model_path, f'ckpt_{epoch}.pt')) + + +def mean_confidence_interval(data, confidence=0.95): + a = 1.0 * np.array(data) + n = len(a) + m, se = np.mean(a), scipy.stats.sem(a) + h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1) + return m, h diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/__init__.py new file mode 100644 index 0000000..f76c2e0 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/__init__.py @@ -0,0 +1,6 @@ +from pathlib import Path +import sys +dir_path = (Path(__file__).parent).resolve() +if str(dir_path) not in sys.path: sys.path.insert(0, str(dir_path)) + +from .architecture import train_single_model \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/architecture.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/architecture.py new file mode 100644 index 0000000..1282044 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/architecture.py @@ -0,0 +1,173 @@ +############################################################### +# NAS-Bench-201, ICLR 2020 (https://arxiv.org/abs/2001.00326) # +############################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +############################################################### +from functions import evaluate_for_seed +from nas_bench_201_models import CellStructure, CellArchitectures, get_search_spaces +from log_utils import Logger, AverageMeter, time_string, convert_secs2time +from nas_bench_201_datasets import get_datasets +from procedures import get_machine_info +from procedures import save_checkpoint, copy_checkpoint +from config_utils import load_config +from pathlib import Path +from copy import deepcopy +import os +import sys +import time +import torch +import random +import argparse +from PIL import ImageFile + +ImageFile.LOAD_TRUNCATED_IMAGES = True + + +NASBENCH201_CONFIG_PATH = os.path.join( + os.getcwd(), 'main_exp', 'transfer_nag') + + +def evaluate_all_datasets(arch, datasets, xpaths, splits, use_less, seed, + arch_config, workers, logger): + machine_info, arch_config = get_machine_info(), deepcopy(arch_config) + all_infos = {'info': machine_info} + all_dataset_keys = [] + # look all the datasets + for dataset, xpath, split in zip(datasets, xpaths, splits): + # train valid data + task = None + train_data, valid_data, xshape, class_num = get_datasets( + dataset, xpath, -1, task) + + # load the configuration + if dataset in ['mnist', 'svhn', 'aircraft', 'pets']: + if use_less: + config_path = os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/LESS.config') + else: + config_path = os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/{}.config'.format(dataset)) + + p = os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/{:}-split.txt'.format(dataset)) + if not os.path.exists(p): + import json + label_list = list(range(len(train_data))) + random.shuffle(label_list) + strlist = [str(label_list[i]) for i in range(len(label_list))] + splited = {'train': ["int", strlist[:len(train_data) // 2]], + 'valid': ["int", strlist[len(train_data) // 2:]]} + with open(p, 'w') as f: + f.write(json.dumps(splited)) + split_info = load_config(os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/{:}-split.txt'.format(dataset)), None, None) + else: + raise ValueError('invalid dataset : {:}'.format(dataset)) + + config = load_config( + config_path, {'class_num': class_num, 'xshape': xshape}, logger) + # data loader + train_loader = torch.utils.data.DataLoader(train_data, batch_size=config.batch_size, + shuffle=True, num_workers=workers, pin_memory=True) + valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=config.batch_size, + shuffle=False, num_workers=workers, pin_memory=True) + splits = load_config(os.path.join( + NASBENCH201_CONFIG_PATH, 'nas_bench_201/configs/nas-benchmark/{}-test-split.txt'.format(dataset)), None, None) + ValLoaders = {'ori-test': valid_loader, + 'x-valid': torch.utils.data.DataLoader(valid_data, batch_size=config.batch_size, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + splits.xvalid), + num_workers=workers, pin_memory=True), + 'x-test': torch.utils.data.DataLoader(valid_data, batch_size=config.batch_size, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + splits.xtest), + num_workers=workers, pin_memory=True) + } + dataset_key = '{:}'.format(dataset) + if bool(split): + dataset_key = dataset_key + '-valid' + logger.log( + 'Evaluate ||||||| {:10s} ||||||| Train-Num={:}, Valid-Num={:}, Train-Loader-Num={:}, Valid-Loader-Num={:}, batch size={:}'. + format(dataset_key, len(train_data), len(valid_data), len(train_loader), len(valid_loader), config.batch_size)) + logger.log('Evaluate ||||||| {:10s} ||||||| Config={:}'.format( + dataset_key, config)) + for key, value in ValLoaders.items(): + logger.log( + 'Evaluate ---->>>> {:10s} with {:} batchs'.format(key, len(value))) + + results = evaluate_for_seed( + arch_config, config, arch, train_loader, ValLoaders, seed, logger) + all_infos[dataset_key] = results + all_dataset_keys.append(dataset_key) + all_infos['all_dataset_keys'] = all_dataset_keys + return all_infos + + +def train_single_model(save_dir, workers, datasets, xpaths, splits, use_less, + seeds, model_str, arch_config): + assert torch.cuda.is_available(), 'CUDA is not available.' + torch.backends.cudnn.enabled = True + torch.backends.cudnn.deterministic = True + torch.set_num_threads(workers) + + save_dir = Path(save_dir) + logger = Logger(str(save_dir), 0, False) + + if model_str in CellArchitectures: + arch = CellArchitectures[model_str] + logger.log( + 'The model string is found in pre-defined architecture dict : {:}'.format(model_str)) + else: + try: + arch = CellStructure.str2structure(model_str) + except: + raise ValueError( + 'Invalid model string : {:}. It can not be found or parsed.'.format(model_str)) + + assert arch.check_valid_op(get_search_spaces( + 'cell', 'nas-bench-201')), '{:} has the invalid op.'.format(arch) + # assert arch.check_valid_op(get_search_spaces('cell', 'full')), '{:} has the invalid op.'.format(arch) + logger.log('Start train-evaluate {:}'.format(arch.tostr())) + logger.log('arch_config : {:}'.format(arch_config)) + + start_time, seed_time = time.time(), AverageMeter() + for _is, seed in enumerate(seeds): + logger.log( + '\nThe {:02d}/{:02d}-th seed is {:} ----------------------<.>----------------------'.format(_is, len(seeds), + seed)) + to_save_name = save_dir / 'seed-{:04d}.pth'.format(seed) + if to_save_name.exists(): + logger.log( + 'Find the existing file {:}, directly load!'.format(to_save_name)) + checkpoint = torch.load(to_save_name) + else: + logger.log( + 'Does not find the existing file {:}, train and evaluate!'.format(to_save_name)) + checkpoint = evaluate_all_datasets(arch, datasets, xpaths, splits, use_less, + seed, arch_config, workers, logger) + torch.save(checkpoint, to_save_name) + # log information + logger.log('{:}'.format(checkpoint['info'])) + all_dataset_keys = checkpoint['all_dataset_keys'] + for dataset_key in all_dataset_keys: + logger.log('\n{:} dataset : {:} {:}'.format( + '-' * 15, dataset_key, '-' * 15)) + dataset_info = checkpoint[dataset_key] + # logger.log('Network ==>\n{:}'.format( dataset_info['net_string'] )) + logger.log('Flops = {:} MB, Params = {:} MB'.format( + dataset_info['flop'], dataset_info['param'])) + logger.log('config : {:}'.format(dataset_info['config'])) + logger.log('Training State (finish) = {:}'.format( + dataset_info['finish-train'])) + last_epoch = dataset_info['total_epoch'] - 1 + train_acc1es, train_acc5es = dataset_info['train_acc1es'], dataset_info['train_acc5es'] + valid_acc1es, valid_acc5es = dataset_info['valid_acc1es'], dataset_info['valid_acc5es'] + # measure elapsed time + seed_time.update(time.time() - start_time) + start_time = time.time() + need_time = 'Time Left: {:}'.format(convert_secs2time( + seed_time.avg * (len(seeds) - _is - 1), True)) + logger.log( + '\n<<<***>>> The {:02d}/{:02d}-th seed is {:} other procedures need {:}'.format(_is, len(seeds), seed, + need_time)) + logger.close() diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/config_utils/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/config_utils/__init__.py new file mode 100644 index 0000000..2d57bbd --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/config_utils/__init__.py @@ -0,0 +1,13 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from .configure_utils import load_config, dict2config#, configure2str +#from .basic_args import obtain_basic_args +#from .attention_args import obtain_attention_args +#from .random_baseline import obtain_RandomSearch_args +#from .cls_kd_args import obtain_cls_kd_args +#from .cls_init_args import obtain_cls_init_args +#from .search_single_args import obtain_search_single_args +#from .search_args import obtain_search_args +# for network pruning +#from .pruning_args import obtain_pruning_args diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/config_utils/configure_utils.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/config_utils/configure_utils.py new file mode 100644 index 0000000..125e68e --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/config_utils/configure_utils.py @@ -0,0 +1,106 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# +import os, json +from os import path as osp +from pathlib import Path +from collections import namedtuple + +support_types = ('str', 'int', 'bool', 'float', 'none') + + +def convert_param(original_lists): + assert isinstance(original_lists, list), 'The type is not right : {:}'.format(original_lists) + ctype, value = original_lists[0], original_lists[1] + assert ctype in support_types, 'Ctype={:}, support={:}'.format(ctype, support_types) + is_list = isinstance(value, list) + if not is_list: value = [value] + outs = [] + for x in value: + if ctype == 'int': + x = int(x) + elif ctype == 'str': + x = str(x) + elif ctype == 'bool': + x = bool(int(x)) + elif ctype == 'float': + x = float(x) + elif ctype == 'none': + if x.lower() != 'none': + raise ValueError('For the none type, the value must be none instead of {:}'.format(x)) + x = None + else: + raise TypeError('Does not know this type : {:}'.format(ctype)) + outs.append(x) + if not is_list: outs = outs[0] + return outs + + +def load_config(path, extra, logger): + path = str(path) + if hasattr(logger, 'log'): logger.log(path) + assert os.path.exists(path), 'Can not find {:}'.format(path) + # Reading data back + with open(path, 'r') as f: + data = json.load(f) + content = { k: convert_param(v) for k,v in data.items()} + assert extra is None or isinstance(extra, dict), 'invalid type of extra : {:}'.format(extra) + if isinstance(extra, dict): content = {**content, **extra} + Arguments = namedtuple('Configure', ' '.join(content.keys())) + content = Arguments(**content) + if hasattr(logger, 'log'): logger.log('{:}'.format(content)) + return content + + +def configure2str(config, xpath=None): + if not isinstance(config, dict): + config = config._asdict() + def cstring(x): + return "\"{:}\"".format(x) + def gtype(x): + if isinstance(x, list): x = x[0] + if isinstance(x, str) : return 'str' + elif isinstance(x, bool) : return 'bool' + elif isinstance(x, int): return 'int' + elif isinstance(x, float): return 'float' + elif x is None : return 'none' + else: raise ValueError('invalid : {:}'.format(x)) + def cvalue(x, xtype): + if isinstance(x, list): is_list = True + else: + is_list, x = False, [x] + temps = [] + for temp in x: + if xtype == 'bool' : temp = cstring(int(temp)) + elif xtype == 'none': temp = cstring('None') + else : temp = cstring(temp) + temps.append( temp ) + if is_list: + return "[{:}]".format( ', '.join( temps ) ) + else: + return temps[0] + + xstrings = [] + for key, value in config.items(): + xtype = gtype(value) + string = ' {:20s} : [{:8s}, {:}]'.format(cstring(key), cstring(xtype), cvalue(value, xtype)) + xstrings.append(string) + Fstring = '{\n' + ',\n'.join(xstrings) + '\n}' + if xpath is not None: + parent = Path(xpath).resolve().parent + parent.mkdir(parents=True, exist_ok=True) + if osp.isfile(xpath): os.remove(xpath) + with open(xpath, "w") as text_file: + text_file.write('{:}'.format(Fstring)) + return Fstring + + +def dict2config(xdict, logger): + assert isinstance(xdict, dict), 'invalid type : {:}'.format( type(xdict) ) + Arguments = namedtuple('Configure', ' '.join(xdict.keys())) + content = Arguments(**xdict) + if hasattr(logger, 'log'): logger.log('{:}'.format(content)) + return content diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft-split.txt b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft-split.txt new file mode 100644 index 0000000..420ab52 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft-split.txt @@ -0,0 +1 @@ +{"train": ["int", ["353", "297", "1508", "3700", "1221", "4489", "1279", "1420", "2306", "3538", "4301", "6301", "3437", "2175", "3779", "2024", "1036", "3696", "2544", "183", "129", "2917", "5420", "3094", "448", "4018", "4037", "1639", "6070", "1308", "1385", "159", "1632", "2845", "1282", "1041", "4112", "1096", "5893", "4918", "4307", "947", "2214", "2432", "1428", "2792", "827", "3922", "163", "2545", "5992", "2226", "6196", "4349", "1959", "1287", "4743", "529", "2642", "1269", "169", "1101", "2806", "1289", "2339", "5739", "5974", "616", "641", "5863", "6401", "138", "853", "1920", "1579", "1018", "4631", "644", "6030", "6285", "4359", "787", "50", "2948", "2475", "1651", "749", "5652", "4145", "4910", "3281", "3008", "5124", "1190", "5499", "3651", "1753", "5908", "3501", "5762", "2639", "2136", "1068", "2018", "6054", "5422", "319", "6209", "2184", "4833", "4001", "848", "2454", "1625", "4028", "3309", "1780", "4041", "3275", "236", "146", "4903", "5123", "2089", "383", "1233", "6493", "4646", "1186", "282", "1706", "1586", "3968", "4450", "3217", "4731", "2983", "1871", "6332", "6528", "3680", "6323", "1459", "3173", "1846", "3916", "6001", "4197", "181", "5521", "2198", "540", "881", "2891", "4211", "5433", "502", "2627", "3264", "3427", "2274", "6116", "3608", "3761", "5052", "2686", "815", "517", "3679", "2511", "1359", "2918", "5876", "3688", "417", "4560", "3304", "1461", "1567", "6617", "1636", "725", "5125", "2495", "1343", "3792", "3631", "4105", "6313", "4084", "4474", "1141", "4070", "3739", "3117", "2446", "1763", "3092", "3382", "6056", "6282", "4959", "3808", "1734", "2912", "2543", "4870", "416", "4510", "3611", "4233", "5221", "5791", "3978", "4065", "741", "4315", "2700", "2922", "6057", "2085", "4064", "70", "5444", "5373", "4171", "2162", "2393", "793", "3570", "3328", "2381", "4430", "273", "5891", "4865", "1032", "978", "5960", "6290", "6601", "4935", "2611", "2573", "186", "2257", "4680", "556", "922", "4292", "2091", "4251", "3153", "355", "1656", "4284", "5353", "3604", "1374", "6067", "4773", "2848", "263", "3620", "3492", "786", "1777", "6291", "1611", "2582", "6328", "6300", "2844", "1000", "1996", "2603", "3389", "2646", "6106", "2488", "3440", "3592", "2326", "5136", "5849", "2414", "2205", "4906", "4297", "1779", "2609", "1881", "3012", "5393", "4864", "2663", "5379", "3154", "6366", "5180", "3388", "974", "3099", "705", "1425", "2797", "241", "5256", "5906", "6337", "3637", "3719", "4242", "477", "5912", "4883", "5979", "102", "1979", "5182", "5248", "5179", "5654", "5068", "6038", "1180", "5015", "1213", "2579", "5178", "1836", "5878", "2493", "3692", "2104", "5619", "4837", "6214", "1346", "2787", "5846", "2285", "5498", "1371", "2254", "5252", "3957", "5724", "5304", "4869", "3880", "215", "3358", "2476", "6269", "109", "1467", "3004", "6092", "5228", "3401", "5594", "4772", "3757", "5291", "4702", "729", "4893", "145", "5094", "6403", "1223", "2608", "2741", "4729", "4767", "4508", "717", "3764", "3548", "1446", "2980", "3959", "6452", "5454", "1315", "2228", "2439", "711", "1892", "4795", "132", "3227", "2408", "268", "4787", "6161", "4710", "3073", "4535", "5750", "4716", "470", "2378", "3799", "6115", "6397", "2155", "2385", "2702", "4106", "731", "1382", "4309", "5768", "85", "6251", "433", "5045", "1342", "2813", "6554", "269", "3954", "5066", "645", "1193", "5613", "158", "3614", "3736", "2139", "1578", "1761", "4180", "630", "3036", "4139", "3508", "6651", "5369", "1702", "5164", "4032", "667", "2643", "4638", "712", "2443", "5410", "1793", "4666", "1873", "1434", "2805", "618", "4920", "170", "949", "4066", "3018", "5133", "6274", "2824", "850", "1898", "1230", "4520", "3194", "4518", "2688", "4400", "120", "5501", "2245", "5735", "689", "1367", "4834", "2547", "2991", "5036", "3913", "3840", "182", "272", "3741", "4498", "3030", "3331", "5163", "2087", "4867", "3067", "3778", "184", "4790", "4281", "5797", "3949", "2769", "6469", "561", "500", "6433", "4868", "5772", "5343", "1037", "6218", "3259", "5612", "4368", "1715", "3556", "6232", "4687", "2862", "5112", "5222", "1176", "4945", "1712", "5610", "5358", "3726", "2019", "3533", "4338", "117", "728", "4998", "2357", "3587", "5671", "785", "6280", "6655", "5289", "2788", "6281", "742", "2667", "4219", "5617", "4671", "5909", "1545", "5577", "5371", "167", "991", "1004", "6024", "5894", "3393", "2153", "3576", "609", "1549", "5914", "1940", "1680", "1758", "5208", "4046", "5564", "699", "3539", "3263", "4625", "162", "2768", "6372", "1162", "2218", "61", "5811", "6358", "4609", "5493", "5645", "1500", "1302", "4569", "2206", "190", "6191", "3509", "4383", "4079", "5709", "2516", "1408", "3303", "1803", "2499", "5737", "6253", "1513", "1095", "2850", "2561", "5349", "3821", "1510", "5022", "1456", "880", "4504", "2456", "5720", "2647", "6224", "1557", "6576", "4433", "1022", "4851", "15", "3253", "6559", "1398", "3702", "1016", "990", "5098", "6414", "1654", "6564", "2574", "876", "4916", "5384", "2719", "1368", "4996", "549", "5131", "5440", "3781", "6456", "3837", "5438", "1054", "4187", "5421", "2956", "3469", "3052", "1896", "5717", "3948", "6471", "134", "4469", "3131", "3554", "3989", "1890", "5245", "4696", "5952", "1334", "2635", "3586", "5079", "6396", "1575", "6474", "6625", "4614", "5559", "3392", "463", "2617", "3877", "1159", "6027", "4823", "3262", "6222", "3879", "3288", "342", "833", "1730", "1865", "395", "5598", "3885", "6558", "6168", "2857", "954", "6249", "4794", "1478", "1143", "6388", "5403", "3462", "4546", "5188", "1745", "25", "2597", "4882", "147", "4363", "3476", "5360", "295", "3182", "5192", "6006", "426", "2133", "2503", "4708", "3488", "4624", "579", "969", "4892", "620", "5941", "1818", "2380", "2933", "3616", "2654", "1164", "6177", "2262", "3057", "6387", "6354", "4375", "4688", "2337", "4650", "4565", "2035", "6446", "953", "4423", "6480", "2324", "4779", "5780", "5110", "2340", "5159", "6357", "4013", "3813", "1447", "1533", "4086", "4470", "567", "2708", "2802", "6666", "3843", "710", "5665", "3633", "2426", "4177", "3338", "1662", "814", "3046", "2296", "6049", "2697", "1133", "4927", "4291", "5773", "3685", "5745", "3349", "4580", "692", "671", "798", "713", "1417", "3863", "3138", "2689", "4445", "5126", "908", "5799", "4038", "1108", "5731", "1307", "4373", "359", "6499", "1377", "3907", "6434", "605", "6202", "2739", "2197", "944", "5474", "6551", "3710", "2103", "1453", "2410", "4681", "5678", "6594", "886", "2086", "4311", "315", "6100", "367", "3322", "1337", "5365", "6190", "3617", "6490", "3672", "3355", "921", "933", "1633", "4044", "2520", "6662", "2669", "4228", "1250", "5286", "4352", "2204", "4481", "3325", "2297", "1084", "114", "352", "1700", "2926", "2533", "1266", "5447", "4290", "3222", "2243", "3901", "1294", "402", "5058", "4815", "1331", "5530", "6592", "3225", "2100", "1201", "1415", "1214", "4941", "4353", "334", "5861", "6255", "3215", "1433", "5806", "3662", "5176", "92", "4796", "677", "1986", "3932", "5234", "1594", "4922", "153", "960", "6364", "5989", "4119", "3861", "2990", "2776", "1551", "2429", "5943", "5888", "5653", "5088", "4250", "1197", "1457", "2468", "5023", "4357", "4673", "3870", "5885", "2195", "2773", "5507", "4384", "6533", "541", "3001", "5647", "2812", "6308", "1659", "4751", "4765", "2981", "1392", "2679", "368", "4100", "1495", "5477", "5102", "6318", "5784", "4750", "2584", "2693", "2430", "532", "2444", "1167", "3876", "5971", "3411", "370", "6521", "4806", "365", "4525", "5032", "4928", "5657", "3290", "1318", "2856", "3632", "5169", "6296", "6504", "2368", "3039", "3814", "591", "5322", "640", "2463", "394", "4737", "1637", "819", "1861", "2487", "1320", "3144", "494", "6611", "121", "2025", "1587", "530", "6311", "1484", "4253", "3640", "4402", "2477", "3795", "3177", "4122", "1189", "1869", "2765", "4499", "6152", "1254", "2709", "6110", "5316", "3838", "5378", "1825", "3090", "5007", "2309", "866", "5319", "5432", "1653", "5430", "6389", "1015", "1469", "4031", "3579", "380", "1439", "1345", "842", "2178", "1713", "3578", "2771", "2622", "2382", "1782", "5586", "1390", "4453", "3350", "3200", "3610", "2557", "2447", "2766", "2004", "5852", "358", "5582", "5736", "1182", "580", "1583", "259", "1416", "6044", "36", "5025", "4052", "1120", "3206", "3518", "4637", "1692", "1458", "4263", "6495", "1914", "3976", "5732", "5470", "5263", "6046", "721", "6131", "3463", "792", "5040", "4963", "4904", "3318", "526", "2552", "5154", "1409", "6626", "3786", "3037", "1695", "194", "6413", "511", "3911", "4958", "778", "2853", "4191", "3179", "3329", "5510", "5419", "6093", "5385", "4782", "2030", "1704", "1463", "3402", "4340", "732", "5999", "4296", "1610", "1739", "5375", "4482", "4936", "5505", "3622", "4552", "2694", "4455", "2799", "6299", "1418", "6121", "4436", "806", "4929", "1746", "6418", "3735", "1395", "2268", "2938", "3753", "411", "4861", "6546", "1370", "3314", "3744", "5427", "6599", "3782", "1379", "1234", "4006", "5718", "3058", "776", "5656", "2594", "788", "4985", "2116", "4917", "3252", "3762", "555", "5931", "4978", "5832", "5364", "6343", "1966", "6653", "4658", "161", "3909", "438", "6488", "2684", "2207", "1630", "5526", "6574", "4838", "783", "5218", "5287", "281", "951", "5324", "3845", "1882", "1930", "686", "258", "6320", "3921", "2063", "3981", "6465", "3130", "4392", "2015", "977", "6160", "3659", "4190", "1623", "5965", "4590", "1612", "1444", "3984", "1569", "1943", "5986", "759", "2389", "1145", "2143", "1481", "2179", "2361", "53", "4835", "3963", "2872", "2467", "5155", "305", "2402", "2901", "2868", "5758", "6476", "325", "586", "6518", "5867", "2836", "3363", "1121", "2231", "3009", "6284", "276", "3283", "4254", "1786", "3890", "1743", "3113", "943", "5927", "3201", "6406", "3014", "5351", "1171", "6326", "6120", "32", "4378", "2832", "3526", "505", "434", "3063", "2831", "3317", "5864", "5030", "1631", "1874", "2596", "4390", "4540", "3655", "480", "5063", "1030", "5417", "774", "4280", "1954", "2672", "4619", "939", "4797", "6245", "5335", "5242", "403", "2615", "1488", "4711", "1460", "5770", "340", "4227", "3718", "6085", "534", "2677", "3772", "870", "4460", "5865", "5260", "3552", "64", "6162", "1466", "3022", "1155", "5691", "4099", "1240", "5347", "928", "5457", "3224", "5991", "649", "2506", "1244", "5540", "2093", "2259", "3649", "3952", "1490", "4347", "3656", "3807", "5545", "4434", "2671", "2395", "498", "1634", "5877", "3207", "5284", "5550", "1231", "1941", "1241", "1783", "860", "2110", "3161", "1210", "3277", "1009", "2532", "4757", "2889", "3040", "3162", "1312", "2121", "5792", "6472", "2177", "2734", "4428", "4734", "375", "5636", "4379", "5475", "997", "726", "4500", "6264", "2288", "5630", "43", "5368", "3858", "1430", "727", "5314", "6355", "3791", "5936", "5666", "1412", "1899", "5301", "4529", "5836", "3646", "4246", "2974", "4783", "3385", "4090", "2398", "2505", "4770", "6400", "2062", "4255", "11", "5407", "3432", "6119", "2676", "3769", "2349", "1910", "3021", "4094", "5320", "3585", "107", "2989", "3118", "2601", "6086", "155", "6542", "1124", "5240", "5519", "26", "5608", "3226", "1868", "1917", "5122", "2423", "4507", "4490", "6602", "3312", "976", "1356", "16", "2106", "4769", "3313", "2920", "6565", "5686", "301", "5592", "5118", "6060", "472", "3396", "171", "6512", "5303", "1034", "6404", "1265", "4209", "6612", "2675", "2556", "1615", "1386", "542", "635", "5840", "2365", "2664", "2898", "2618", "1701", "3205", "5220", "266", "1194", "3944", "934", "4131", "5013", "1541", "3896", "3733", "767", "3448", "808", "2842", "4042", "593", "4372", "1123", "2673", "2351", "3724", "2987", "1581", "3918", "2281", "6622", "1310", "1923", "5667", "2541", "5558", "6132", "4059", "4193", "2046", "4194", "5344", "2459", "204", "2629", "4050", "1086", "5254", "6314", "3789", "2001", "4643", "2466", "225", "3924", "1698", "285", "3694", "781", "6608", "2605", "6000", "51", "4855", "3746", "5197", "2818", "6112", "2563", "3887", "3720", "622", "5600", "2911", "5078", "4008", "3849", "1813", "6464", "569", "5047", "407", "3912", "4566", "4889", "5361", "3897", "3690", "2729", "570", "5231", "5027", "2986", "839", "88", "3522", "5451", "3738", "3519", "5973", "6013", "310", "1383", "3599", "6650", "924", "3607", "6236", "22", "5402", "2904", "6419", "5010", "3681", "5541", "3788", "5398", "6429", "5689", "4639", "2353", "157", "2455", "4973", "1948", "1607", "5853", "5938", "5191", "1722", "3302", "588", "1227", "3430", "3687", "4322", "42", "2440", "4273", "5255", "1997", "1509", "3323", "4425", "1062", "5295", "4647", "3472", "5243", "443", "1005", "4938", "5148", "5902", "91", "2370", "1135", "4793", "4279", "2098", "6322", "5298", "1727", "4685", "3193", "1902", "4239", "4326", "6008", "1788", "1506", "5658", "5743", "3450", "1968", "369", "1192", "2314", "3908", "6500", "387", "6237", "2316", "4351", "4532", "4067", "757", "3446", "2327", "3269", "1565", "3671", "5818", "3120", "4113", "175", "663", "4977", "604", "21", "4257", "4839", "4404", "4813", "5275", "2129", "3032", "4715", "1975", "6640", "2366", "6283", "4801", "5334", "103", "4885", "1901", "6117", "6439", "1908", "279", "2558", "351", "6325", "1388", "6180", "5409", "2464", "2486", "2758", "4585", "1817", "5411", "2379", "2344", "562", "3005", "5659", "1964", "475", "3818", "5579", "537", "1489", "5822", "4144", "882", "3537", "3336", "3653", "6195", "4431", "6059", "1017", "445", "6302", "3089", "1151", "4181", "929", "3810", "6002", "1126", "1626", "544", "4695", "226", "3682", "1932", "4584", "6654", "5083", "2222", "3920", "4720", "4995", "4226", "1056", "6463", "2772", "4411", "1807", "3499", "1830", "2554", "2567", "2302", "4606", "4942", "1665", "2746", "1003", "476", "6614", "4661", "6124", "941", "205", "5714", "5549", "4564", "2273", "3601", "4447", "524", "3839", "2176", "4116", "2383", "4899", "293", "2017", "633", "4323", "5485", "1907", "5121", "4994", "2251", "965", "937", "608", "4803", "2187", "745", "2137", "460", "5340", "4495", "2358", "3424", "5370", "3701", "3003", "6099", "572", "5825", "2119", "333", "1251", "3024", "4825", "2973", "3156", "3078", "849", "4810", "3102", "6373", "2542", "879", "423", "2249", "3580", "6367", "4339", "4331", "3170", "1720", "701", "5363", "1911", "2113", "2691", "5556", "2994", "4205", "1682", "5193", "1759", "2341", "3790", "660", "5597", "6369", "2029", "3854", "1255", "1413", "2685", "5627", "6089", "3641", "560", "1178", "4449", "179", "4150", "1590", "5527", "1369", "1906", "4694", "4117", "5651", "2784", "3712", "3531", "1949", "2587", "3197", "5581", "321", "5381", "596", "4604", "5416", "4189", "4472", "2566", "1951", "4759", "1006", "3953", "3219", "4512", "6012", "6005", "1351", "3803", "1767", "356", "2075", "2263", "5465", "4303", "5962", "583", "230", "6052", "5804", "1260", "2301", "948", "6567", "2804", "1449", "3266", "2821", "6201", "6443", "5929", "5376", "2992", "5602", "1553", "1564", "349", "3468", "5548", "4168", "1304", "2478", "4215", "378", "6226", "4983", "5021", "2894", "1237", "2614", "4645", "2472", "5157", "3958", "1350", "6126", "5459", "1429", "5502", "1518", "6205", "5859", "1160", "4475", "617", "3449", "2144", "1960", "6609", "5640", "4847", "200", "673", "4674", "3722", "4186", "4015", "3237", "2120", "1750", "3732", "6383", "5786", "5512", "5844", "100", "2125", "4771", "1747", "5563", "3101", "4503", "4136", "1546", "2756", "752", "124", "5001", "4328", "3077", "4778", "3979", "3609", "3731", "639", "5681", "5237", "309", "5662", "2422", "6288", "2419", "4617", "274", "5236", "3243", "3639", "4578", "390", "5081", "6442", "3321", "4416", "3511", "2182", "4184", "4547", "2640", "3119", "4554", "5265", "2334", "6598", "4408", "5837", "4109", "3435", "6538", "611", "3900", "896", "4517", "2749", "2481", "6462", "5870", "6356", "5624", "1820", "2884", "2915", "4808", "5162", "487", "700", "1538", "2965", "4824", "6658", "6604", "994", "1330", "5803", "3245", "4799", "1601", "5207", "6505", "6141", "4156", "2286", "2967", "4321", "4831", "1536", "3439", "5450", "6150", "5639", "4966", "473", "3334", "4212", "3645", "2825", "219", "4034", "2660", "5244", "3083", "1181", "3787", "3709", "602", "4760", "116", "1707", "3181", "6184", "2092", "2418", "5337", "6064", "4101", "5439", "1921", "966", "5200", "6239", "6079", "4336", "2360", "5134", "1591", "4712", "105", "4817", "1474", "3816", "6553", "3564", "5469", "1148", "289", "1440", "3817", "2272", "2323", "1990", "507", "329", "3155", "5633", "4987", "5423", "3844", "246", "5202", "3628", "1831", "5585", "867", "5146", "653", "5160", "4954", "6415", "257", "558", "577", "3391", "2203", "4842", "52", "4754", "3011", "3525", "4992", "5391", "2082", "311", "1372", "5235", "4979", "3070", "5915", "6143", "1841", "6643", "135", "5135", "2460", "1840", "5262", "3016", "4925", "1644", "420", "3062", "404", "5919", "1833", "441", "5062", "4040", "964", "5881", "6458", "4654", "5328", "2742", "963", "6460", "4011", "3250", "4354", "3972", "3512", "2428", "3835", "1242", "5644", "722", "6315", "2705", "2604", "6479", "5687", "4362", "3917", "1858", "6125", "5143", "6286", "5492", "4004", "5841", "6390", "1547", "6468", "1212", "446", "2572", "2928", "37", "972", "2416", "1061", "1642", "2636", "2988", "1971", "2152", "2232", "3670", "5127", "5428", "5050", "5103", "2551", "1399", "386", "2319", "1198", "656", "4085", "6144", "5390", "877", "4376", "6014", "1122", "5589", "3056", "1468", "5429", "5542", "970", "2307", "4785", "2047", "6164", "1609", "1158", "6584", "3474", "2698", "1311", "4709", "5546", "2996", "6561", "1886", "4123", "5611", "6362", "2060", "2958", "112", "6256", "439", "1035", "73", "1903", "277", "1119", "1859", "6021", "3521", "822", "3220", "3644", "1864", "6379", "2820", "324", "1994", "5685", "1988", "1142", "1573", "4439", "1805", "1051", "4077", "3618", "3994", "5802", "3293", "4002", "995", "4295", "5779", "1483", "1741", "6424", "5302", "323", "6430", "2181", "60", "4768", "1998", "681", "58", "758", "1498", "1635", "2141", "4586", "5934", "5856", "6082", "1687", "2165", "2056", "4523", "1274", "5625", "6007", "1671", "5968", "6304", "450", "4960", "447", "4114", "916", "3775", "512", "5431", "461", "4125", "6133", "4915", "2710", "2839", "5484", "1384", "1070", "1487", "4679", "3369", "1355", "2616", "6440", "5879", "6378", "3433", "5959", "6616", "3190", "4494", "4706", "4158", "501", "6333", "3395", "5729", "863", "3456", "2083", "5216", "2054", "927", "6039", "4258", "4788", "6004", "4651", "3214", "4506", "2271", "6360", "6204", "1809", "4821", "2789", "3105", "4632", "3483", "2294", "6664", "4921", "1309", "3546", "3826", "3581", "5042", "377", "6334", "489", "3763", "3566", "5913", "2623", "4913", "3590", "4655", "1421", "6647", "4395", "5574", "5311", "4780", "1222", "533", "5307", "3752", "3376", "2208", "6203", "678", "4563", "1563", "4202", "4217", "550", "6329", "2167", "1196", "4199", "3375", "2483", "118", "4648", "3723", "6096", "6652", "1649", "242", "2910", "917", "3939", "962", "5082", "797", "5990", "2183", "2377", "6393", "3851", "5707", "4313", "2369", "4748", "3216", "5975", "3163", "4089", "2525", "6257", "2313", "149", "3658", "3630", "2527", "5046", "5487", "2630", "4763", "6545", "3629", "979", "1437", "4974", "971", "2564", "125", "6074", "2390", "1676", "520", "1195", "1375", "6321", "6155", "39", "1136", "24", "4310", "5642", "2795", "5576", "5318", "3134", "4611", "601", "6181", "4231", "3287", "3591", "3380", "3398", "4521", "3750", "1614", "5765", "3882", "4240", "110", "5824", "1984", "900", "6182", "2724", "2151", "3902", "5838", "1074", "6011", "1931", "5744", "1404", "1705", "816", "2223", "3017", "2589", "1602", "6361", "2317", "243", "5723", "2886", "2548", "912", "2641", "6523", "1503", "1718", "388", "255", "5452", "2897", "1184", "4009", "2864", "6073", "5000", "2331", "77", "3431", "3139", "1373", "2701", "1894", "3708", "4438", "4777", "857", "245", "4133", "4138", "5883", "3481", "865", "31", "4025", "1093", "1947", "203", "6170", "3192", "5935", "3172", "0", "1053", "1916", "3447", "261", "2916", "5468", "2950", "1897", "3848", "5267", "6492", "3760", "1674", "4946", "4859", "967", "2849", "1450", "2949", "5551", "2969", "2510", "5018", "1738", "5478", "304", "856", "5629", "5012", "794", "1850", "4926", "4314", "513", "3136", "891", "6477", "13", "2147", "6047", "3482", "4626", "621", "2224", "1751", "872", "4836", "4130", "1329", "5166", "4092", "1327", "3796", "3510", "195", "807", "6187", "1915", "1933", "6425", "5557", "432", "4993", "4229", "1749", "3998", "2265", "3801", "4397", "6473", "1316", "5308", "3947", "5816", "4477", "3823", "4053", "2053", "790", "6103", "3515", "4819", "5290", "5504", "6018", "5703", "1884", "832", "1026", "3589", "5699", "5464", "2867", "5697", "5626", "1118", "6455", "4132", "4509", "104", "5928", "5981", "2256", "1787", "1092", "3927", "3910", "1073", "5573", "5710", "851", "6420", "4789", "4692", "2201", "2038", "1087", "899", "6412", "3045", "4261", "1273", "1360", "4965", "3346", "2237", "6444", "72", "5387", "6385", "19", "316", "2026", "3514", "496", "392", "2823", "5089", "3812", "1792", "1023", "5074", "303", "6581", "2173", "3504", "6094", "4581", "173", "1648", "3996", "75", "3426", "2628", "6457", "1270", "3343", "836", "5185", "4330", "5855", "4621", "2826", "1853", "6588", "6259", "2735", "4952", "1405", "483", "3140", "5238", "4159", "1683", "4493", "1319", "332", "993", "248", "1248", "6470", "4271", "5119", "1130", "3872", "3484", "1226", "4394", "1576", "3020", "5388", "4830", "918", "603", "4317", "2852", "1972", "4023", "2067", "3026", "3588", "3320", "3465", "4391", "2238", "30", "5269", "1519", "3221", "3444", "223", "6016", "1091", "1872", "2755", "495", "3211", "371", "6072", "4223", "4690", "4930", "1021", "6183", "1768", "5833", "5061", "4162", "3028", "4689", "2420", "1725", "3315", "662", "5704", "2896", "5857", "3946", "2645", "6475", "5090", "6003", "3970", "789", "2321", "6381", "33", "1011", "1239", "3353", "2895", "4047", "5967", "4890", "4360", "6208", "574", "6154", "2877", "4324", "6268", "5049", "4413", "4401", "2959", "5056", "6509", "5253", "427", "1977", "3559", "210", "1505", "4218", "5884", "4288", "6037", "521", "3044", "3495", "89", "141", "2138", "1993", "4299", "3714", "3168", "345", "6638", "191", "5174", "5455", "3928", "306", "18", "3409", "4809", "2753", "5145", "5109", "4901", "919", "478", "2593", "5722", "2059", "6656", "3051", "685", "1078", "2032", "5944", "5467", "1641", "3667", "2944", "6417", "3487", "3856", "4318", "575", "3384", "6303", "6287", "1109", "647", "5734", "4435", "4956", "318", "1785", "3704", "322", "1772", "3689", "2751", "559", "106", "1521", "212", "1530", "6055", "4853", "1795", "6615", "5543", "6197", "4691", "2997", "684", "3171", "421", "1476", "1071", "904", "5448", "936", "3091", "3084", "5842", "4541", "874", "1353", "5141", "3379", "2780", "4370", "4152", "3169", "3234", "2128", "894", "3112", "267", "897", "753", "5547", "4599", "2524", "2930", "539", "3904", "2438", "409", "3148", "142", "2261", "2480", "2145", "2666", "284", "2051", "1592", "4487", "6293", "986", "1526", "5655", "2117", "2376", "449", "2346", "2571", "6502", "2652", "5591", "1887", "1014", "5412", "5807", "2935", "6156", "3149", "2637", "6549", "6363", "327", "4111", "424", "357", "6105", "5823", "6530", "2131", "6646", "607", "3425", "2725", "4668", "3265", "2984", "1462", "2491", "140", "3133", "286", "4267", "4539", "4022", "3663", "2109", "1114", "2124", "4786", "1929", "2375", "3955", "4026", "4459", "1952", "6603", "840", "5019", "5462", "2731", "4024", "1103", "2107", "907", "1835", "3648", "3473", "2906", "4080", "1655", "691", "6015", "5752", "2170", "1485", "4335", "1364", "4592", "4305", "4677", "889", "4342", "791", "1152", "2727", "2171", "3598", "1451", "2000", "2461", "626", "5442", "5875", "4545", "518", "5520", "6010", "3819", "1937", "5024", "2266", "69", "299", "5142", "4074", "3804", "5698", "3419", "4722", "3975", "746", "2407", "5246", "6292", "4948", "5170", "1407", "4873", "3347", "1839", "6573", "4260", "2819", "4762", "2391", "2993", "2908", "152", "4900", "1079", "668", "768", "6587", "1283", "1391", "5250", "4511", "2227", "5531", "5637", "3811", "82", "3420", "2633", "4371", "4818", "3883", "4221", "4912", "1721", "1924", "1378", "2384", "4484", "5650", "5227", "5966", "4247", "2796", "280", "2239", "4192", "654", "4420", "2325", "5426", "3408", "456", "6663", "2626", "1380", "1955", "469", "5186", "5815", "59", "5932", "6459", "6569", "4344", "3734", "3486", "4660", "1271", "2570", "3485", "3319", "5108", "2458", "2172", "3678", "3378", "4143", "2250", "5138", "4791", "796", "3903", "1789", "2132", "4451", "4682", "3311", "6556", "3703", "3544", "3241", "582", "5529", "5819", "1925", "2537", "3159", "5866", "1048", "1001", "3373", "4488", "1710", "2080", "1347", "5144", "1691", "453", "1043", "2581", "2764", "1900", "2748", "4659", "2304", "2546", "2442", "1574", "2336", "3255", "3475", "4049", "744", "4237", "2210", "6359", "6382", "5436", "1410", "3926", "4107", "5198", "3866", "160", "4061", "2900", "3218", "5288", "3661", "5201", "5620", "6140", "213", "6478", "1689", "5506", "2388", "3239", "821", "3657", "6618", "4355", "3405", "5998", "1726", "3188", "1580", "1740", "1082", "3551", "772", "4515", "4852", "4531", "4406", "3033", "1494", "5362", "1111", "3852", "3244", "6529", "4259", "4943", "2057", "4276", "2392", "6122", "4419", "2540", "3977", "4633", "565", "5461", "2105", "5677", "4798", "650", "6266", "4588", "2962", "760", "6336", "5528", "3106", "5392", "1826", "4804", "2625", "983", "3942", "5562", "1535", "3164", "2202", "1523", "1796", "1677", "5518", "3410", "5892", "2785", "4456", "6431", "2043", "5325", "320", "2049", "6621", "6138", "838", "4333", "4693", "3971", "3060", "2159", "240", "3434", "1888", "6540", "3478", "5020", "6063", "2058", "6175", "5137", "1638", "5443", "3088", "5900", "3931", "6416", "5778", "330", "6649", "2740", "2851", "1228", "3535", "1522", "1599", "909", "835", "566", "3698", "6327", "4802", "4826", "739", "547", "4350", "878", "2522", "2186", "852", "2513", "5359"]], "valid": ["int", ["2761", "3988", "4157", "1477", "6501", "2846", "2971", "6068", "3561", "4670", "1139", "4312", "6607", "3516", "6025", "4327", "2123", "1187", "594", "1464", "3529", "3634", "1303", "1202", "3524", "5513", "5670", "354", "2621", "4021", "3496", "6294", "6535", "1040", "96", "1340", "4735", "2225", "2767", "5672", "5005", "1183", "292", "5424", "3160", "1987", "1039", "5041", "5149", "3674", "777", "4662", "5071", "740", "5315", "5064", "3842", "6438", "1365", "4811", "6589", "3572", "3438", "6244", "2292", "4587", "1471", "1556", "4805", "17", "1863", "5175", "5437", "66", "4243", "20", "4704", "2371", "144", "3351", "1207", "587", "961", "2424", "4516", "1660", "1115", "3573", "5730", "6305", "3868", "398", "2759", "4991", "5129", "6146", "708", "5596", "901", "6178", "6527", "6338", "5817", "1985", "38", "1694", "1699", "2800", "5839", "6555", "6531", "1020", "4230", "2838", "3754", "4699", "1188", "3523", "3567", "384", "3352", "1291", "4850", "6270", "1829", "65", "4862", "4730", "5215", "2479", "2722", "361", "1280", "130", "1646", "3805", "6563", "4422", "6398", "1161", "5009", "5184", "4567", "2096", "6026", "6606", "4462", "3270", "5898", "3208", "497", "2523", "3143", "3513", "3983", "4471", "6375", "5232", "3297", "6423", "3366", "844", "2600", "5700", "4463", "4820", "2977", "720", "5516", "3991", "4605", "5623", "5400", "3428", "1889", "4879", "4583", "4480", "5233", "1790", "1232", "366", "5224", "3271", "4537", "4911", "3240", "3081", "1652", "6104", "6077", "661", "4068", "3743", "1801", "6258", "709", "4598", "6516", "1153", "2887", "3124", "1719", "1678", "1582", "2150", "5017", "1703", "5869", "2189", "4304", "1507", "4361", "3356", "6075", "4726", "3129", "2624", "1811", "1895", "4964", "914", "6352", "1257", "391", "2448", "5350", "2276", "1658", "535", "3857", "302", "859", "3199", "264", "5794", "372", "4003", "2979", "4293", "6629", "5688", "4062", "4454", "9", "4278", "1616", "4491", "2649", "5406", "6335", "1204", "3452", "440", "216", "5196", "2359", "3406", "3962", "6402", "2598", "5051", "2174", "3574", "3348", "233", "5266", "3251", "5552", "1033", "2157", "1414", "1081", "5093", "3550", "4877", "2526", "211", "6242", "3443", "6295", "2421", "3461", "2023", "2757", "3285", "2330", "239", "2794", "4519", "5632", "4103", "4377", "290", "3086", "6272", "464", "5956", "6254", "980", "6297", "6076", "3292", "1755", "410", "1672", "1776", "682", "3278", "3695", "2576", "5854", "4717", "5945", "6221", "2320", "5769", "3691", "2899", "4738", "4249", "4522", "406", "3774", "3636", "76", "300", "5923", "5616", "5767", "2754", "5719", "6087", "3254", "2931", "2913", "6310", "5926", "4723", "761", "1335", "1154", "6351", "4206", "1879", "2312", "1566", "4947", "762", "63", "5177", "2657", "3098", "887", "2779", "5016", "492", "1597", "5705", "2863", "6578", "4036", "6053", "6137", "5495", "670", "4294", "6593", "4306", "1808", "6525", "5922", "1475", "3455", "5356", "3919", "5065", "1537", "6229", "4248", "6048", "2947", "360", "136", "723", "3103", "1199", "2462", "47", "6628", "5575", "3686", "911", "6118", "5332", "610", "1387", "2995", "4238", "4446", "3986", "3421", "3095", "3507", "735", "1769", "6421", "3833", "373", "4388", "139", "1336", "2322", "4744", "2040", "5297", "5835", "5721", "1441", "5580", "6220", "5777", "3853", "6550", "3123", "3846", "2005", "3974", "6347", "3203", "87", "5105", "799", "6467", "4083", "5848", "5831", "4029", "6506", "4551", "337", "595", "275", "6142", "3906", "1619", "4275", "3684", "6623", "331", "3204", "83", "4623", "2403", "959", "1299", "527", "5987", "592", "2876", "5310", "2003", "4845", "6395", "564", "2373", "5389", "3047", "1515", "14", "3423", "1532", "5886", "4151", "4283", "177", "5708", "328", "4745", "3480", "3491", "2084", "5257", "2078", "250", "841", "3274", "1203", "6083", "2827", "6324", "5982", "1465", "5895", "2354", "5279", "6200", "5039", "3235", "481", "4127", "989", "1013", "3142", "3128", "3937", "6575", "6627", "4172", "296", "2270", "4544", "6199", "1963", "3471", "1400", "563", "3399", "5329", "3498", "5957", "3184", "1268", "3992", "4232", "3666", "1735", "5323", "5241", "5489", "4102", "6485", "430", "6630", "646", "1482", "98", "2801", "2909", "2387", "5910", "973", "4485", "2127", "5401", "3638", "484", "5486", "6645", "6496", "885", "3500", "3990", "3956", "4170", "4909", "4137", "335", "2166", "5404", "5590", "1965", "6234", "4937", "1128", "3135", "341", "4538", "1854", "1137", "2457", "5276", "3737", "1647", "6031", "6227", "898", "1217", "2869", "4286", "5583", "5746", "3097", "2591", "3697", "5114", "49", "5760", "4822", "313", "4269", "115", "992", "5566", "1491", "2115", "1138", "1981", "1473", "2242", "317", "824", "431", "4962", "6547", "1389", "5712", "1668", "3560", "3257", "1185", "1845", "1918", "1402", "2501", "1419", "5272", "1613", "5456", "1499", "2190", "4414", "6078", "4010", "2102", "3534", "224", "4556", "1756", "2295", "1643", "6017", "1571", "4501", "736", "1496", "5048", "6128", "5533", "5209", "40", "3100", "4196", "3654", "2496", "5115", "3441", "4019", "4282", "1348", "648", "545", "1814", "2562", "5539", "6306", "6562", "2332", "4060", "782", "6449", "6", "1554", "4204", "4676", "743", "6298", "643", "5603", "5903", "1125", "5958", "4486", "5950", "4951", "1028", "6033", "6198", "5525", "4270", "3555", "5003", "5259", "5496", "1502", "3569", "1117", "557", "5116", "4713", "3041", "923", "2723", "2275", "1775", "4128", "2299", "454", "4988", "2810", "3413", "3023", "6447", "1577", "3993", "1857", "5858", "956", "1666", "166", "1224", "5414", "6583", "3337", "2101", "4898", "5282", "5199", "672", "2942", "2114", "4142", "150", "4081", "1798", "1089", "228", "2363", "5535", "2180", "3665", "5331", "6631", "3126", "482", "3517", "688", "1325", "4595", "1256", "4950", "2033", "6350", "6243", "831", "2752", "3802", "5983", "4872", "1628", "2298", "2404", "3371", "4533", "28", "6041", "3527", "2193", "2837", "6040", "2308", "1002", "4203", "5044", "4045", "5537", "6661", "869", "2396", "2946", "1298", "3829", "488", "5055", "2498", "6157", "3025", "4126", "619", "3127", "2362", "6644", "3943", "2333", "5173", "1066", "3360", "1366", "4562", "3414", "1525", "3707", "5445", "2620", "6632", "1834", "4774", "2482", "766", "5692", "548", "1149", "2400", "5029", "5277", "2854", "6410", "5206", "3836", "2471", "6071", "4174", "3489", "2347", "2531", "2791", "5808", "4496", "6579", "4667", "2998", "950", "5213", "2834", "1822", "3326", "3342", "3841", "3247", "1531", "4386", "5397", "6307", "2474", "737", "5357", "350", "5247", "4561", "5132", "6062", "6605", "719", "6665", "4173", "2399", "4905", "6508", "4665", "3176", "1501", "4369", "1332", "5251", "1134", "1992", "3174", "6216", "5453", "5588", "3664", "4756", "5172", "5925", "2580", "5595", "4088", "2777", "486", "4268", "5212", "2763", "2955", "3238", "4016", "5405", "6036", "3422", "2728", "5156", "764", "2858", "820", "3711", "389", "467", "5268", "4348", "2163", "3381", "4440", "2079", "3082", "6503", "2278", "2592", "1970", "260", "2122", "2394", "1696", "4752", "4135", "6101", "2502", "795", "2881", "1448", "4207", "1973", "6597", "5638", "3929", "6312", "5139", "6186", "1852", "2606", "4980", "5342", "625", "168", "5503", "5377", "6319", "6532", "2213", "3593", "6231", "338", "1427", "2405", "3950", "2452", "4982", "2585", "4418", "86", "6212", "6586", "95", "5565", "4367", "5845", "3699", "6481", "1055", "6211", "5035", "5664", "3675", "4091", "952", "5418", "4272", "1891", "847", "811", "185", "2880", "206", "581", "1909", "3114", "2061", "3248", "1065", "4129", "4146", "5810", "2586", "5511", "3693", "642", "4784", "4989", "3894", "1555", "2303", "174", "5151", "2744", "599", "6498", "892", "2036", "6148", "6377", "3945", "3066", "414", "3562", "3627", "3256", "80", "2515", "694", "6526", "3260", "4502", "3505", "938", "5813", "3186", "3213", "674", "5171", "3493", "5299", "3074", "5554", "2234", "451", "1146", "6108", "5479", "5584", "4678", "1019", "3824", "154", "1824", "5230", "4841", "6613", "1252", "1235", "3715", "4285", "837", "4134", "6185", "4753", "4616", "3855", "2662", "1288", "4740", "1175", "623", "3683", "3605", "363", "2097", "3279", "4319", "773", "2068", "5570", "6136", "344", "231", "1452", "3467", "1560", "6277", "10", "6028", "5482", "5043", "3368", "2658", "5508", "3300", "4907", "1220", "5748", "5104", "553", "81", "5948", "4166", "1967", "5158", "2726", "3464", "955", "5037", "6577", "108", "945", "45", "1225", "4437", "4185", "5396", "6097", "381", "3166", "2284", "4033", "2536", "412", "5820", "5978", "5961", "2191", "5890", "1757", "192", "4", "3650", "207", "1760", "2052", "6610", "3884", "4235", "5782", "1261", "3571", "884", "6129", "1936", "5993", "931", "4858", "6639", "2865", "984", "5771", "5793", "2720", "5872", "4443", "2010", "5994", "262", "5294", "493", "2866", "3416", "5646", "4005", "1572", "3964", "2730", "465", "5996", "3310", "2968", "2786", "6189", "68", "1935", "1396", "5727", "4577", "636", "861", "5181", "5985", "94", "3951", "3272", "4627", "2560", "606", "3340", "2549", "5701", "750", "1669", "1667", "6341", "5458", "4407", "2012", "1177", "5725", "1305", "5002", "405", "4596", "4683", "1794", "4536", "1247", "613", "2951", "2492", "5386", "6483", "6497", "854", "1711", "2497", "4827", "998", "4630", "2683", "800", "5140", "3387", "522", "3365", "90", "2441", "5694", "2775", "4570", "4387", "3565", "5946", "4589", "4986", "6022", "1165", "3284", "2164", "244", "6519", "41", "5214", "657", "2318", "5194", "5933", "6235", "1823", "1604", "1559", "3280", "3982", "4208", "5605", "519", "4076", "999", "1962", "576", "1219", "3059", "4277", "2300", "1593", "2168", "2583", "4051", "3895", "3542", "1296", "2632", "6380", "202", "1622", "5330", "3647", "3335", "2963", "5776", "2200", "1455", "830", "4399", "3860", "4078", "6368", "5631", "3158", "5270", "1716", "3985", "3995", "3859", "4955", "3065", "2269", "2619", "3705", "3930", "1107", "1116", "4886", "1341", "1904", "543", "2924", "4473", "5634", "1173", "6636", "62", "2717", "4896", "5026", "536", "5481", "6090", "2610", "218", "5273", "55", "3506", "3327", "4999", "716", "1693", "1497", "4875", "6374", "2372", "4183", "5695", "199", "2013", "3966", "4098", "5728", "5601", "6051", "1640", "5435", "5084", "6107", "5352", "1934", "637", "5904", "397", "5413", "2199", "590", "703", "1529", "1276", "2305", "3436", "2489", "6123", "2064", "4332", "5607", "6524", "4356", "2260", "2934", "193", "5380", "1424", "5028", "4664", "6489", "6571", "2130", "6641", "5072", "382", "4576", "3575", "78", "2807", "4724", "271", "5969", "2230", "1913", "4409", "197", "122", "6346", "3768", "817", "5787", "2185", "4188", "1661", "1838", "2449", "4967", "890", "3888", "459", "5790", "2927", "4075", "5555", "283", "4163", "1804", "6165", "2737", "509", "2830", "4458", "2682", "664", "1511", "3602", "3372", "1045", "3362", "4953", "4844", "6517", "6240", "3776", "6279", "2328", "5281", "5684", "2699", "1744", "6035", "1961", "6029", "3236", "5911", "1357", "2450", "5604", "3806", "2790", "2921", "5471", "5514", "6514", "3626", "48", "5190", "462", "4919", "3151", "3961", "2975", "4396", "3759", "4747", "4874", "1939", "127", "222", "1527", "4736", "312", "396", "1029", "624", "4733", "5106", "5372", "2077", "2578", "6376", "5789", "4214", "3043", "4057", "4104", "362", "5834", "5382", "1435", "4884", "2569", "733", "5761", "1542", "3568", "5970", "2879", "3915", "1050", "3289", "4728", "1791", "5317", "5150", "3923", "5572", "379", "1411", "5764", "571", "1275", "6088", "4828", "6487", "5229", "1300", "5809", "738", "5814", "4364", "6042", "2007", "23", "1200", "176", "2638", "5187", "2530", "3583", "3677", "1765", "2048", "4505", "3122", "3454", "2519", "3797", "883", "5751", "3935", "6111", "3668", "6466", "1063", "4860", "4972", "2142", "5920", "1976", "2529", "528", "2070", "2050", "6169", "3676", "6548", "4634", "629", "2002", "4412", "5466", "2885", "5977", "6316", "4476", "1098", "6436", "2445", "3822", "2733", "3809", "5713", "4213", "503", "6173", "308", "3010", "925", "942", "2412", "8", "627", "5008", "4175", "4976", "2409", "1290", "294", "1262", "3530", "1922", "5441", "5649", "3898", "2291", "5092", "802", "1847", "1077", "2721", "1174", "525", "2960", "1208", "3189", "1621", "6163", "5", "3196", "5850", "4087", "3547", "6262", "4981", "2335", "2732", "4289", "2229", "4072", "1728", "5754", "3178", "2954", "3013", "1543", "178", "123", "2809", "2612", "1285", "227", "982", "4649", "5067", "2156", "3758", "4035", "1272", "5939", "2348", "6151", "1540", "3249", "2154", "3766", "1982", "3210", "1862", "4225", "3273", "2940", "376", "2258", "3121", "4393", "1843", "4701", "2835", "2855", "4198", "2814", "3460", "5346", "1215", "2874", "2066", "5955", "2374", "1259", "3370", "810", "4902", "56", "2484", "5749", "3079", "3034", "1688", "2235", "2248", "2678", "1049", "1163", "452", "3780", "422", "4939", "6570", "6552", "3050", "5614", "1097", "1338", "1528", "1588", "6642", "5781", "5567", "4382", "1827", "1094", "5963", "2453", "3390", "2602", "2535", "3110", "5497", "1422", "1627", "4672", "2656", "1956", "5491", "573", "2882", "237", "1596", "3282", "3109", "3457", "958", "4641", "6331", "2507", "552", "6241", "3232", "3878", "1875", "198", "5682", "3069", "5167", "652", "2212", "6147", "3258", "2937", "5759", "6515", "3532", "5690", "3965", "1754", "1883", "2859", "3299", "4714", "5383", "6176", "5211", "1534", "5168", "775", "2665", "1969", "2293", "2782", "2607", "4432", "1403", "4082", "1245", "4154", "6409", "3417", "4417", "34", "3941", "6034", "2188", "4607", "5449", "2192", "3625", "4148", "5099", "5271", "1733", "2736", "6624", "578", "4017", "2952", "3494", "5733", "458", "5847", "2822", "2843", "1216", "2500", "968", "3451", "6544", "3869", "35", "5117", "5086", "3798", "4876", "217", "4115", "3147", "3673", "2099", "631", "4358", "4365", "1844", "1246", "508", "1815", "2674", "3111", "4636", "3019", "2277", "2028", "1106", "2888", "4816", "3899", "2508", "119", "4176", "6510", "5726", "2433", "5128", "2413", "2661", "6659", "846", "1060", "4684", "6066", "3276", "5766", "5293", "2431", "2287", "1172", "6450", "2815", "5693", "3545", "1358", "3577", "748", "429", "5096", "5560", "172", "3404", "2634", "3006", "1752", "5742", "2069", "1057", "3407", "4478", "920", "6134", "4971", "5930", "2706", "3157", "3553", "903", "1595", "1010", "1842", "3107", "3793", "3967", "3889", "126", "6513", "2355", "2350", "4897", "554", "6408", "1263", "2841", "4256", "1731", "5683", "5918", "4334", "1295", "996", "2923", "2465", "251", "5544", "278", "2148", "2209", "981", "3048", "3643", "1293", "506", "4832", "2711", "2762", "6344", "1267", "5524", "2718", "2436", "6271", "3053", "3660", "3031", "1629", "1090", "1292", "5336", "1729", "873", "5828", "2401", "845", "1816", "2945", "4698", "5292", "5483", "665", "5073", "4934", "2650", "1806", "5339", "4568", "5183", "4337", "2976", "1486", "5374", "3333", "5643", "3185", "4341", "3751", "6261", "4640", "5901", "5473", "5147", "4742", "2695", "4932", "4514", "5415", "148", "5621", "1608", "4071", "5606", "5500", "5569", "1169", "2220", "4866", "4719", "6411", "5680", "754", "1363", "1800", "5434", "2668", "6461", "4096", "3767", "1880", "6345", "4766", "1600", "4571", "4557", "680", "1218", "3747", "385", "5204", "4579", "913", "2112", "5571", "4201", "1052", "988", "3027", "3470", "4161", "5054", "1709", "2936", "3202", "1771", "2135", "5006", "2081", "2042", "1598", "6427", "5882", "1008", "2707", "2264", "6069", "1393", "1585", "2902", "2470", "3815", "1322", "3261", "4574", "4167", "4467", "6520", "3296", "3015", "4325", "6192", "4725", "1558", "3383", "718", "1905", "5321", "4236", "3594", "6167", "1673", "2252", "1083", "3477", "3748", "3582", "3132", "232", "3339", "5980", "600", "6210", "326", "3386", "5783", "910", "6238", "3429", "3831", "3233", "4840", "4969", "5394", "6486", "4300", "6339", "5821", "3442", "4054", "6633", "2828", "6171", "1104", "1442", "1301", "4466", "1147", "6590", "209", "2829", "6252", "747", "3490", "6009", "2553", "3847", "1492", "6127", "2703", "3865", "598", "1044", "5532", "6230", "2041", "2870", "2982", "2961", "2469", "6207", "4843", "131", "5711", "3558", "1031", "2939", "3061", "5561", "2941", "5014", "5219", "4241", "2415", "1837", "5296", "3619", "3400", "2241", "2555", "6453", "5798", "1436", "2712", "249", "687", "1870", "499", "3771", "5702", "5460", "3191", "4656", "666", "5280", "4613", "1072", "704", "2217", "3543", "1946", "3341", "1168", "1284", "5333", "287", "1127", "5873", "3936", "1110", "6267", "2781", "1166", "479", "1799", "6371", "805", "5153", "2343", "4739", "2194", "5964", "1926", "4069", "1438", "2538", "2883", "515", "468", "2108", "3007", "4642", "1983", "1112", "6135", "5225", "5663", "2774", "5152", "298", "1957", "5100", "4030", "4302", "2367", "3459", "5300", "444", "1075", "5628", "1802", "1764", "418", "4746", "4329", "4961", "4628", "3364", "769", "235", "2", "4461", "4265", "6065", "3115", "1099", "4141", "4700", "2893", "428", "4548", "1950", "6619", "2435", "1974", "5217", "2490", "437", "675", "926", "5305", "5203", "4707", "6386", "3934", "128", "3344", "4732", "1849", "1685", "669", "336", "3783", "4881", "6392", "3354", "756", "706", "164", "1774", "2517", "2750", "1562", "113", "5354", "4618", "4718", "5924", "5988", "4316", "3875", "3377", "1209", "1603", "4891", "5669", "3093", "5899", "5517", "1670", "5679", "5327", "3324", "4888", "4758", "1206", "5889", "597", "3596", "4346", "4058", "2565", "676", "291", "3756", "1645", "638", "6166", "2932", "3886", "2681", "1999", "6113", "4908", "4308", "2919", "485", "2966", "1663", "1742", "3613", "895", "2512", "2282", "3064", "531", "4857", "829", "2094", "4427", "6582", "1810", "695", "3717", "5755", "2280", "2021", "3301", "1570", "6536", "1361", "5070", "1179", "2770", "1038", "2247", "5676", "6494", "3730", "3332", "801", "4675", "6637", "2985", "2907", "5897", "2833", "6091", "1454", "1828", "6591", "3850", "1860", "4653", "6342", "1443", "2045", "6491", "5887", "1406", "1708", "2890", "1724", "4800", "4120", "6289", "1314", "702", "1426", "6061", "6275", "307", "5949", "3359", "1323", "6620", "4266", "930", "1618", "6534", "612", "5953", "4097", "1", "905", "201", "5326", "5210", "3394", "6159", "6217", "3669", "4014", "4615", "5355", "4741", "6043", "3071", "2311", "2356", "4924", "4856", "3549", "6432", "6391", "3230", "2073", "2134", "4559", "4764", "4374", "3862", "5097", "2860", "2233", "2760", "2427", "4686", "4441", "2219", "3635", "1942", "466", "585", "2970", "3728", "4234", "2039", "6441", "4421", "4448", "2118", "1995", "4721", "5091", "2111", "4492", "3940", "2680", "2494", "252", "2577", "4385", "2716", "987", "3765", "1144", "5463", "6585", "1058", "734", "3049", "54", "2840", "4582", "79", "2595", "510", "4697", "165", "5312", "4121", "1617", "5261", "3000", "679", "3502", "214", "5615", "3367", "3183", "6246", "932", "5077", "6153", "4483", "3777", "2599", "823", "5851", "4020", "1324", "2651", "6384", "3773", "3871", "3223", "4244", "4043", "1024", "4871", "5480", "3874", "2169", "4140", "690", "813", "3729", "5747", "3749", "2037", "4997", "1085", "4398", "4949", "1989", "2687", "779", "3584", "1867", "2009", "2425", "1945", "1690", "4846", "6370", "1397", "868", "3867", "1504", "658", "4410", "3167", "6580", "3075", "4274", "1253", "1445", "5345", "3345", "6566", "1512", "5753", "3563", "568", "5536", "2216", "3029", "4550", "1520", "1131", "3716", "1317", "855", "4612", "5408", "4573", "3642", "67", "4095", "589", "1953", "765", "516", "4601", "6102", "1249", "1762", "4608", "4497", "6543", "2575", "5706", "5107", "4894", "5741", "1781", "3597", "2504", "809", "435", "4457", "1278", "1848", "3721", "1297", "1686", "5599", "5075", "6276", "5995", "634", "6596", "288", "4933", "715", "6095", "3615", "1238", "2873", "6179", "1514", "4262", "6019", "413", "5161", "714", "2146", "5034", "940", "1697", "1381", "902", "455", "3187", "1980", "4165", "2011", "6250", "3212", "6648", "4940", "1376", "1025", "5812", "4878", "3612", "4622", "628", "5534", "6139", "84", "2518", "2957", "2071", "1748", "4389", "2692", "615", "1100", "6188", "3745", "6445", "2847", "3096", "3864", "2386", "6265", "2670", "3770", "1229", "2417", "4591", "693", "3116", "1878", "5057", "4093", "5553", "5367", "229", "2055", "1958", "5472", "1067", "6399", "3054", "2020", "1944", "5674", "6353", "1561", "6330", "4530", "6278", "1129", "5976", "6020", "5101", "1281", "2905", "6228", "1339", "6223", "784", "3072", "6511", "6174", "238", "2485", "3466", "4669", "2008", "5366", "2095", "3825", "5205", "523", "3146", "1431", "4124", "44", "6248", "1258", "1080", "314", "6435", "5843", "2747", "5278", "3755", "2539", "4153", "4705", "5947", "1927", "2031", "6206", "2437", "2006", "2246", "6045", "4534", "5756", "1832", "2925", "3137", "347", "5757", "5130", "1277", "3891", "4468", "3246", "4381", "3195", "1877", "4895", "2160", "6145", "2648", "3740", "1313", "3784", "4380", "1321", "1732", "6233", "5917", "1589", "1354", "2875", "5716", "2253", "97", "3624", "1042", "1978", "3960", "4776", "1236", "4426", "491", "6482", "2738", "683", "1352", "780", "5004", "3987", "3291", "3520", "2473", "1650", "3938", "1885", "6098", "6273", "5827", "3080", "137", "3652", "2655", "2653", "5715", "3832", "1723", "4012", "1102", "5285", "2798", "5258", "3800", "5587", "3", "5896", "471", "975", "1912", "4792", "4155", "1548", "5648", "3165", "2929", "2715", "101", "4957", "6348", "5011", "3830", "2196", "2808", "4245", "3374", "6172", "4442", "5538", "1606", "3600", "2644", "858", "3412", "4975", "6557", "3528", "3445", "538", "4880", "4048", "1851", "5522", "2745", "4527", "6365", "5829", "985", "6260", "6213", "1856", "2509", "3706", "364", "5871", "425", "5223", "1919", "4849", "3038", "208", "3242", "4549", "2434", "6539", "3725", "5826", "6194", "4703", "6149", "133", "2022", "3294", "2215", "6522", "5618", "4178", "5905", "4000", "256", "2074", "3268", "3152", "4224", "4593", "4970", "5274", "915", "6247", "957", "5673", "2690", "5880", "1012", "3606", "2016", "4657", "1778", "1328", "348", "6428", "4775", "651", "3595", "697", "5940", "3905", "3175", "3361", "5795", "632", "343", "5069", "3827", "818", "5801", "6451", "5830", "3820", "2783", "6050", "4968", "4118", "1493", "2076", "346", "5661", "751", "2550", "4452", "2149", "474", "1770", "4220", "730", "1876", "6158", "906", "3087", "1306", "2778", "1552", "4553", "5738", "419", "3231", "871", "6219", "4558", "5490", "6660", "2568", "4424", "546", "4644", "4343", "5165", "4345", "2044", "6484", "4761", "1479", "834", "4629", "4610", "1657", "1516", "888", "5111", "401", "5523", "5060", "4603", "4405", "2364", "2559", "5399", "6657", "5076", "5341", "4149", "3479", "1855", "2090", "400", "2743", "5984", "1191", "4366", "5874", "1046", "4526", "399", "3125", "1264", "1344", "99", "698", "2065", "6225", "4055", "4147", "3330", "2290", "6568", "4575", "2345", "270", "1243", "5053", "5568", "5951", "5031", "6560", "5494", "5309", "4572", "5264", "6595", "2793", "504", "5785", "3892", "2236", "5578", "1812", "6572", "151", "1286", "5446", "3150", "2528", "4524", "3306", "1544", "5059", "2878", "3497", "3881", "7", "812", "5593", "5476", "6317", "189", "3541", "2803", "4403", "2158", "3742", "1326", "4063", "46", "4110", "724", "770", "5868", "3180", "5641", "5788", "234", "6600", "4848", "6405", "5338", "2817", "490", "6081", "3828", "2240", "221", "862", "1150", "875", "1991", "1423", "3999", "4812", "457", "2978", "1211", "2659", "5937", "6193", "5226", "3914", "2811", "1170", "4195", "864", "1938", "6507", "1893", "3228", "6541", "1736", "4108", "220", "6537", "2315", "755", "4465", "3557", "3308", "2714", "1069", "415", "2338", "3068", "696", "436", "1432", "3973", "1717", "1675", "893", "2871", "2914", "3893", "6634", "2861", "2140", "5622", "2713", "3458", "2411", "5954", "3002", "1007", "4222", "1047", "2289", "4200", "5774", "3403", "5425", "1684", "4513", "93", "6263", "247", "5862", "2161", "4755", "1105", "1157", "2072", "6635", "4056", "2451", "5038", "6454", "5085", "1064", "5997", "74", "2892", "6080", "4415", "2534", "4829", "1027", "2590", "2283", "3536", "803", "187", "3415", "1333", "2903", "2613", "5921", "5775", "826", "29", "1088", "6114", "614", "2088", "4179", "5668", "1866", "1714", "5488", "5120", "3209", "3104", "3397", "1737", "3316", "2221", "5348", "2126", "6084", "1584", "4444", "5033", "2342", "6394", "12", "1059", "4464", "2279", "3229", "1076", "5313", "1113", "2816", "2953", "4944", "4073", "6407", "4931", "3145", "1679", "1539", "4543", "5763", "2034", "5515", "442", "3453", "1928", "843", "3727", "5805", "3286", "5800", "4216", "5696", "3141", "2310", "4479", "374", "1766", "156", "5972", "1524", "4620", "4210", "2943", "6023", "3035", "6215", "6109", "935", "4652", "804", "3055", "4781", "1664", "180", "6032", "4600", "111", "4252", "763", "3267", "1568", "3085", "707", "4182", "3540", "4814", "2211", "4923", "1681", "3969", "3933", "4320", "1605", "4914", "514", "3503", "3785", "1470", "3357", "2267", "1773", "5509", "3603", "5675", "4027", "2972", "2514", "659", "6437", "5113", "655", "6130", "1620", "3295", "408", "5249", "2352", "3997", "2014", "4863", "4160", "5609", "4287", "1401", "4555", "27", "57", "4594", "2631", "2027", "4749", "1517", "5283", "5087", "946", "4007", "6349", "1472", "4597", "1205", "6448", "3042", "188", "4887", "1394", "5796", "2244", "1156", "2999", "551", "2588", "4169", "196", "5195", "4663", "3713", "3305", "4635", "4602", "265", "4298", "4429", "825", "3873", "6422", "3925", "5189", "1819", "1349", "4854", "1140", "5306", "5660", "1480", "3621", "4807", "1784", "5395", "5080", "1550", "5740", "339", "3623", "5860", "1797", "2521", "393", "3794", "143", "4984", "4039", "1362", "3298", "3307", "584", "1132", "4164", "3108", "2964", "2329", "4727", "2704", "4264", "3834", "5942", "6309", "2696", "5916", "1821", "2397", "5239", "4990", "828", "254", "4542", "5095", "3198", "5907", "6058", "4528", "2255", "6426", "253", "5635", "3980", "3418", "3076", "6340", "1624", "71", "771", "2406"]]} \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft-test-split.txt b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft-test-split.txt new file mode 100644 index 0000000..d2d9a0e --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft-test-split.txt @@ -0,0 +1 @@ +{"xvalid": ["int", ["187", "699", "1190", "586", "1102", "1457", "2151", "1560", "1326", "1204", "1493", "2146", "436", "2196", "2314", "492", "765", "2743", "3254", "2563", "789", "1855", "2831", "911", "2350", "198", "1020", "1931", "1018", "257", "3176", "1164", "139", "3142", "1433", "236", "380", "441", "1441", "1054", "1274", "1928", "152", "2703", "294", "2300", "1565", "350", "3277", "2077", "742", "2933", "1026", "1275", "1133", "2765", "1926", "170", "1938", "1352", "2592", "914", "2287", "385", "2494", "2202", "1902", "2371", "202", "444", "1953", "1620", "1650", "2587", "3242", "3132", "2470", "1836", "2571", "1272", "1717", "2032", "968", "1675", "2716", "2481", "1778", "1236", "2505", "1588", "1544", "2557", "870", "3172", "1619", "2473", "2058", "610", "926", "830", "1414", "1227", "754", "1944", "1205", "493", "647", "2278", "1764", "1247", "2135", "1061", "593", "2810", "2123", "828", "727", "356", "842", "1862", "2789", "2600", "1219", "2472", "958", "1676", "2246", "976", "3215", "723", "461", "807", "1662", "578", "360", "3259", "401", "1613", "1438", "330", "2119", "2628", "2569", "1770", "2461", "1253", "1107", "2219", "30", "1538", "1952", "2764", "2280", "623", "1791", "3191", "668", "1858", "1327", "1660", "2180", "2936", "140", "2468", "1372", "1682", "1989", "1271", "979", "1044", "1502", "485", "642", "2378", "363", "1988", "2550", "2871", "2034", "3038", "3291", "59", "1835", "690", "515", "881", "2804", "2931", "1244", "701", "1376", "1890", "1324", "357", "2904", "427", "305", "740", "487", "2006", "871", "573", "3219", "703", "1104", "1088", "2160", "1285", "954", "1130", "197", "3028", "1394", "562", "3253", "2850", "3207", "878", "2293", "1689", "2945", "1170", "2026", "3153", "2103", "199", "661", "358", "2545", "1805", "584", "1771", "2247", "2236", "1740", "1622", "862", "1543", "1004", "3318", "2869", "1332", "377", "1569", "1762", "653", "3025", "3124", "3167", "349", "758", "669", "636", "505", "1156", "1062", "2564", "1254", "2948", "1353", "471", "2780", "2649", "1319", "1413", "1445", "1723", "117", "973", "3310", "2093", "33", "372", "2647", "2319", "2190", "3326", "2374", "1111", "295", "599", "322", "3092", "2747", "3007", "1708", "2626", "2445", "2060", "2705", "2140", "1651", "2239", "3112", "2195", "480", "3123", "2594", "2412", "861", "1551", "3329", "1473", "2456", "821", "2892", "115", "1339", "2339", "1663", "2", "2718", "868", "683", "1895", "3101", "2847", "1399", "1305", "2506", "1630", "804", "2538", "2420", "1564", "1516", "1496", "863", "997", "1366", "119", "1687", "1962", "1943", "183", "1742", "707", "3224", "3054", "1669", "2048", "2234", "2429", "1615", "424", "241", "1914", "1179", "213", "3286", "706", "84", "1267", "2197", "889", "3000", "2851", "2916", "2108", "2157", "3208", "341", "2259", "2465", "2453", "3057", "260", "292", "875", "825", "1861", "169", "2029", "229", "2706", "912", "28", "1000", "490", "2223", "2828", "694", "1908", "752", "3022", "3311", "3147", "621", "2109", "510", "1900", "144", "3013", "2059", "961", "101", "35", "3218", "1896", "43", "2188", "3190", "672", "1474", "370", "2393", "1539", "277", "1760", "652", "270", "3236", "2664", "622", "1064", "408", "909", "1035", "1220", "3117", "748", "3056", "3233", "1442", "1873", "1734", "802", "1617", "3111", "2785", "231", "1747", "2713", "1155", "2690", "1163", "2290", "1405", "1718", "1961", "1513", "3141", "3179", "2312", "1387", "2279", "2459", "3135", "560", "2091", "406", "342", "1290", "2409", "2968", "1409", "402", "1213", "1114", "1719", "1498", "2349", "978", "2240", "1800", "143", "1359", "691", "66", "2821", "2591", "2992", "1273", "888", "250", "2774", "3029", "3185", "452", "297", "2845", "1909", "1983", "38", "39", "2844", "908", "932", "1116", "1098", "2369", "1056", "1079", "79", "3325", "2766", "1768", "1099", "2714", "1580", "2508", "2072", "684", "1724", "670", "715", "2642", "625", "299", "1587", "2346", "1610", "720", "244", "572", "2761", "1958", "3041", "2053", "2977", "398", "2853", "1579", "1567", "1057", "2763", "1154", "1840", "125", "1798", "2900", "1024", "1106", "17", "2276", "656", "1011", "3174", "1769", "2340", "1093", "3069", "756", "2709", "3275", "1069", "2903", "1269", "2167", "847", "3047", "2079", "2269", "1489", "2243", "287", "1804", "2673", "1257", "2942", "221", "3290", "1846", "1162", "1025", "2734", "15", "1110", "2375", "1999", "200", "3300", "133", "1256", "335", "1978", "134", "1527", "1970", "156", "1383", "2524", "829", "1594", "1221", "2520", "692", "1834", "456", "2281", "2999", "2746", "2533", "3004", "3059", "648", "1832", "1497", "1032", "2469", "3014", "1808", "3284", "2395", "3145", "2163", "619", "3182", "1385", "2779", "2615", "2542", "2352", "849", "2262", "3020", "1794", "394", "2687", "1535", "0", "1081", "2680", "44", "3071", "1866", "2516", "331", "3211", "1086", "2593", "2439", "848", "2116", "1187", "759", "2570", "2446", "174", "3053", "1492", "167", "271", "2786", "784", "72", "1436", "1100", "2848", "2324", "2723", "1224", "2898", "1599", "1710", "274", "2362", "3040", "2370", "646", "1223", "1796", "162", "1477", "1055", "1686", "2913", "1019", "1439", "785", "2065", "882", "3046", "3180", "2726", "942", "1506", "491", "595", "1670", "443", "3079", "592", "662", "2584", "1434", "814", "1974", "2133", "5", "2401", "1310", "2490", "1829", "901", "1553", "1134", "3100", "1888", "1029", "2113", "2771", "2444", "1673", "1139", "2296", "3252", "2450", "657", "1680", "1681", "2479", "1298", "3272", "2422", "2567", "1874", "1514", "2137", "1002", "2301", "2532", "3301", "982", "466", "216", "2341", "2924", "1259", "113", "1519", "3067", "1317", "769", "2106", "688", "1609", "2322", "289", "381", "1452", "1775", "1706", "453", "2517", "2448", "193", "1685", "728", "282", "1279", "733", "3109", "413", "142", "92", "1297", "527", "2298", "2039", "1049", "1869", "2221", "2178", "70", "2040", "577", "2294", "518", "1820", "2895", "3321", "3314", "164", "1967", "2304", "1657", "1341", "1469", "1076", "2523", "3195", "1128", "2998", "904", "3133", "1407", "189", "2738", "2497", "1554", "1666", "2783", "2896", "2795", "2604", "2226", "3026", "1484", "1766", "1491", "1972", "3076", "1432", "2403", "990", "2722", "509", "1030", "2819", "2363", "1264", "2250", "1147", "1654", "1051", "514", "2306", "3006", "1919", "102", "1591", "3156", "160", "630", "1758", "2249", "1214", "2654", "1965", "1316", "910", "2672", "3245", "3154", "2865", "1384", "2585", "2432", "3240", "178", "291", "1042", "2820", "2961", "3015", "1982", "1508", "185", "995", "365", "3289", "2695", "2022", "386", "867", "2054", "608", "2891", "1801", "2066", "1979", "2754", "1980", "2073", "2554", "1417", "311", "426", "3187", "2912", "317", "31", "463", "1841", "2084", "687", "126", "1023", "1080", "1447", "583", "1582", "2177", "1600", "1825", "1918", "1113", "2671", "2775", "524", "3089", "242", "1863", "3328", "2937", "3166", "2019", "2396", "3220", "2792", "3169", "2777", "1103", "1893", "362", "1280", "2983", "2366", "1488", "2978", "986", "569", "2463", "2547", "1174", "2201", "1328", "227", "3216", "1849", "1229", "2455", "2704", "2071", "1509", "2495", "2224", "312", "531", "2956", "1228", "948", "1215", "834", "883", "1541", "196", "345", "2946", "1022", "1137", "2796", "2577", "2273", "2885", "1186", "2960", "927", "2030", "535", "591", "3150", "1759", "2104", "3082", "977", "1821", "2809", "2144", "2883", "962", "2272", "2864", "3110", "1052", "2868", "2629", "223", "2458", "2794", "1903", "1653", "836", "13", "2229", "2372", "590", "1779", "2007", "521", "1124", "2595", "2938", "238", "1635", "1021", "268", "2773", "1292", "1209", "2974", "281", "1006", "1589", "697", "654", "2361", "2271", "1361", "974", "664", "966", "3231", "460", "201", "1501", "3239", "1714", "1391", "839", "1410", "2118", "1185", "1132", "766", "1001", "3261", "3250", "1031", "708", "2087", "379", "718", "303", "1360", "1123", "2358", "1059", "940", "3323", "1074", "190", "2323", "1652", "2637", "1471", "1012", "3149", "2142", "348", "1082", "624", "2607", "2172", "1249", "2408", "613", "2252", "512", "97", "2311", "1306", "1520", "2781", "1540", "1322", "2359", "1641", "1633", "151", "127", "3159", "2684", "1455", "2222", "2745", "1621", "957", "118", "1606", "2964", "2136", "3051", "1857", "822", "1806", "20", "1592", "32", "1146", "1007", "617", "994", "99", "793", "732", "3021", "2437", "234", "2299", "175", "2337", "1997", "2089", "529", "3234", "1728", "2833", "2527", "2799", "3165", "628", "337", "790", "2559", "3267", "1827", "2131", "3203", "2330", "2047", "3094", "495", "3012", "1843", "3330", "2907", "517", "1364", "606", "220", "243", "2728", "1345", "389", "2558", "60", "913", "1287", "165", "1235", "1138", "2758", "3074", "1072", "1819", "171", "2070", "2860", "2662", "1136", "315", "96", "2165", "1730", "934", "405", "400", "1788", "2121", "2105", "2263", "2206", "1369", "2181", "2086", "2541", "2191", "2405", "2110", "1278", "2255", "2753", "719", "1168", "1429", "714", "451", "1149", "2677", "685", "2193", "3087", "2996", "633", "533", "923", "3146", "2043", "1440", "86", "479", "53", "1172", "2926", "2011", "93", "3027", "1222", "3122", "601", "640", "2669", "388", "508", "1741", "3288", "1143", "2854", "2724", "3188", "3036", "1161", "2521", "447", "3221", "884", "1304", "548", "2935", "2611", "2750", "1756", "2824", "1898", "3322", "247", "2164", "3175", "393", "1424", "638", "2153", "872", "1067", "458", "1954", "1810", "2291", "252", "667", "1165", "319", "2742", "87", "1584", "1977", "3152", "2575", "1738", "1337", "561", "2166", "81", "2989", "1612", "1388", "3324", "2417", "3052", "919", "1581", "853", "520", "2608", "588", "2343", "2134", "306", "761", "2957", "522", "2025", "1395", "2530", "2757", "320", "513", "895", "19", "1268", "1921", "1321", "1709", "3035", "1203", "2320", "799", "1623", "2225", "264", "991", "3113", "2620", "1356", "3260", "1015", "154", "1443", "3032", "2120", "1188", "1812", "1150", "1255", "2451", "173", "1141", "2127", "1604", "2347", "2016", "1294", "2509", "2292", "2426", "2149", "2100", "2128", "1823", "3128", "1960", "2812", "1182", "58", "1334", "2175", "2872", "2389", "525", "1705", "2855", "1802", "2125", "2215", "1495", "2815", "2995", "2549", "3151", "1811", "2870", "1435", "2749", "2565", "681", "239", "496", "2331", "929", "2367", "1749", "222", "2217", "2474", "286", "1425", "361", "1344", "261", "16", "359", "1562", "462", "1777", "2431", "2625", "2971", "739", "2702", "1301", "580", "2858", "2811", "418", "1879", "450", "254", "2627", "2947", "3157", "108", "516", "3313", "810", "835", "2973", "1524", "3183", "1178", "975", "2840", "26", "1336", "1906", "182", "2327", "1125", "921", "1158", "336", "2965", "421", "217", "2009", "2184", "1891", "2099", "2665", "1643", "1700", "1934", "1428", "2076", "9", "1830", "1555", "953", "433", "2471", "430", "382", "2825", "2930", "435", "671", "2051", "568", "832", "3001", "1971", "1852", "3199", "893", "55", "440", "972", "1289", "1423", "1202", "1071", "1463", "332", "650", "3280", "431", "805", "2275", "2260", "837", "1389", "920", "1404", "110", "1191", "3181", "2797", "1586", "1153", "1118", "1722", "45", "2744", "2659", "2394", "2908", "576", "3164", "73", "3192", "1750", "1075", "2391", "1552", "3064", "787", "2631", "2364", "992", "2102", "90", "3170", "3279", "547", "3086", "1248", "383", "3327", "3223", "1693", "615", "111", "741", "2496", "886", "1743", "581", "941", "985", "2576", "1323", "1945", "2257", "798", "2038", "1212", "2407", "773", "1948", "1715", "67", "698", "204", "11", "1787", "1872", "2055", "1815", "1973", "301", "423", "2634", "1237", "1266", "1561", "860", "1206", "858", "1744", "507", "1363", "1939", "1731", "1036", "3120", "746", "246", "2816", "172", "2126", "2037", "2598", "2834", "933", "147", "2132", "3088", "1860", "1453", "563", "184", "2814", "2013", "2150", "3050", "543", "1871", "2802", "155", "2650", "753", "1822", "788", "1546", "1464", "1583", "677", "2653", "2485", "1963", "1780", "150", "1536", "3158", "474", "1547", "1720", "711", "1419", "2663", "3121", "846", "645", "2877", "1251", "62", "146", "967", "1904", "2923", "3114", "1331", "494", "3210", "2207", "141", "2183", "2791", "2148", "3140", "1503", "1183", "3269", "750", "1640", "2082", "3193", "1260", "2635", "304", "943", "717", "232", "2238", "2633", "1745", "1528", "1782", "2313", "812", "2701", "3134", "907", "2187", "1034", "420", "1446", "290", "2580", "1097", "104", "475", "2214", "1702", "1039", "604", "65", "3213", "2512", "2360", "3037", "2668", "399", "1699", "1790", "2837", "253", "611", "1231", "1665", "559", "233", "3039", "1729", "2617", "2932", "2943", "1602", "2866", "2543", "416", "48", "792", "2838", "2969", "1883", "641", "285", "2784", "1101", "136", "1627", "857", "368", "609"]], "xtest": ["int", ["2068", "2697", "3258", "544", "3294", "651", "637", "1751", "2826", "1929", "1634", "1704", "2101", "3144", "2953", "2717", "255", "3249", "2425", "2539", "40", "94", "2605", "826", "2739", "1240", "3130", "122", "550", "1483", "2601", "1017", "757", "1847", "309", "2067", "1129", "1374", "2274", "1016", "755", "23", "866", "457", "3033", "818", "1993", "123", "1342", "2000", "2919", "469", "1713", "1180", "225", "22", "2876", "1505", "2619", "2988", "1171", "1595", "3248", "696", "874", "1784", "2546", "478", "1772", "3197", "1115", "915", "411", "2158", "2027", "2526", "946", "851", "2171", "2681", "69", "2406", "2098", "2398", "869", "675", "890", "2760", "1996", "1120", "2582", "2035", "532", "432", "1448", "459", "1739", "2303", "3005", "2152", "2556", "2478", "2210", "1195", "1347", "34", "2688", "1964", "2124", "2111", "1854", "1711", "1207", "2428", "2581", "629", "776", "1028", "391", "3189", "343", "2036", "439", "3102", "2380", "2602", "2020", "951", "2921", "1598", "1043", "3062", "930", "1907", "1703", "384", "3273", "2315", "1692", "24", "1574", "896", "2489", "107", "1089", "2980", "1173", "1732", "1412", "2230", "632", "3", "387", "1941", "71", "1813", "2731", "1095", "2265", "1951", "1696", "649", "3043", "2130", "1577", "983", "1916", "1456", "2477", "2529", "283", "2237", "551", "1949", "464", "3107", "3209", "1803", "2675", "2955", "1522", "14", "2666", "2379", "655", "2502", "3034", "2282", "738", "1605", "2316", "2397", "3091", "88", "2046", "3295", "1335", "2641", "2519", "3178", "582", "1870", "422", "1401", "780", "56", "2920", "2768", "1349", "2325", "3103", "3296", "1911", "2590", "844", "700", "2354", "1302", "224", "2863", "3309", "415", "211", "85", "2922", "3104", "2856", "2176", "511", "902", "1245", "749", "1677", "1402", "1281", "2873", "325", "205", "2694", "2525", "3292", "1511", "827", "1923", "1258", "1917", "2997", "366", "2616", "2057", "964", "1218", "959", "2348", "1485", "2440", "2390", "3160", "987", "1842", "3097", "1628", "1053", "1614", "2511", "3081", "100", "1066", "1340", "1818", "502", "2987", "2679", "1526", "772", "1754", "98", "1765", "903", "1649", "355", "1008", "1763", "1073", "2457", "984", "2852", "2231", "536", "725", "2884", "248", "1601", "1166", "1239", "2934", "1085", "840", "795", "2823", "3320", "2962", "2958", "1672", "2984", "503", "1853", "541", "2534", "2902", "192", "2415", "2651", "791", "855", "2667", "2024", "1109", "676", "2867", "2772", "2886", "1572", "1578", "2447", "1045", "499", "2241", "542", "2033", "2056", "1955", "2373", "2063", "2888", "1091", "2596", "504", "54", "302", "1548", "2522", "2638", "2555", "1459", "3139", "340", "3241", "1912", "2357", "2487", "618", "2491", "2335", "1683", "8", "2381", "713", "1193", "3278", "454", "3303", "27", "2710", "82", "486", "1885", "42", "3048", "1288", "2685", "2941", "2890", "2967", "2092", "763", "988", "3030", "2205", "3018", "1530", "1105", "259", "3085", "2887", "1225", "3243", "600", "1200", "3235", "3229", "428", "1311", "2245", "1558", "876", "2698", "859", "2881", "63", "2897", "1817", "334", "538", "2139", "2507", "1894", "2233", "392", "1038", "103", "730", "1241", "3200", "2652", "2069", "2769", "2334", "1727", "2535", "1998", "2411", "534", "2400", "1753", "3072", "918", "64", "1985", "2915", "1159", "2344", "659", "6", "1421", "482", "1295", "1875", "2251", "2332", "1479", "293", "1350", "3009", "2384", "781", "1664", "455", "284", "3201", "501", "130", "1406", "1785", "1243", "163", "2493", "2503", "1504", "1690", "1532", "37", "1721", "352", "3205", "2661", "567", "1773", "2985", "1355", "191", "704", "3095", "1976", "2645", "3078", "2261", "468", "2484", "1468", "1381", "965", "1618", "378", "2803", "3268", "922", "3126", "49", "2235", "2700", "2383", "1833", "993", "2213", "1986", "489", "2021", "2751", "1648", "1181", "180", "1905", "1809", "1478", "483", "1068", "1636", "230", "307", "3265", "157", "308", "2017", "2208", "824", "326", "7", "179", "1678", "1216", "1152", "2646", "660", "2696", "944", "3217", "263", "3264", "1348", "1573", "1625", "2720", "2686", "1379", "2767", "3198", "1490", "1816", "1211", "77", "159", "3137", "1933", "2078", "2940", "1995", "952", "296", "2711", "1362", "564", "2875", "2894", "2644", "897", "906", "1679", "3214", "2266", "570", "445", "612", "1246", "3316", "21", "673", "1737", "747", "166", "813", "1824", "996", "2993", "885", "523", "1882", "1987", "2168", "2699", "1371", "1482", "2418", "310", "2159", "2441", "3255", "1925", "2442", "137", "354", "2001", "373", "3011", "2917", "1735", "1060", "1126", "721", "407", "1994", "477", "2950", "1647", "129", "2005", "2606", "2589", "3105", "2404", "3010", "2842", "1593", "2424", "1403", "2500", "936", "208", "266", "797", "2790", "3061", "556", "1411", "1194", "2566", "1033", "768", "2182", "2267", "1774", "1487", "1119", "472", "3266", "1671", "168", "1430", "2518", "2540", "1500", "10", "2353", "219", "2356", "2643", "658", "1427", "2014", "678", "2966", "2880", "528", "616", "771", "1397", "89", "2064", "2003", "1127", "686", "135", "3080", "51", "1358", "955", "2096", "775", "1346", "1697", "3317", "1307", "2818", "1848", "2618", "1947", "865", "1545", "800", "2218", "777", "2878", "371", "1420", "553", "3060", "275", "938", "838", "425", "2421", "644", "2216", "2658", "2042", "497", "1047", "2462", "3093", "3161", "1521", "442", "3304", "1990", "2510", "3148", "3202", "2297", "2285", "1382", "1624", "864", "2793", "2835", "1992", "1177", "1396", "1486", "2859", "579", "2562", "3262", "1659", "1157", "2586", "1169", "207", "3016", "2012", "2660", "1418", "327", "2338", "2480", "300", "2994", "1748", "682", "786", "1668", "2499", "2514", "1367", "438", "2173", "314", "2544", "596", "2736", "2074", "770", "2402", "2083", "803", "1176", "1005", "2382", "2925", "214", "2154", "329", "1542", "1494", "364", "1930", "745", "333", "565", "1608", "1507", "2385", "1046", "689", "3083", "2318", "2454", "2735", "1856", "1370", "2599", "1009", "1886", "3017", "3171", "806", "1144", "2578", "2808", "3285", "3058", "3075", "212", "1308", "1283", "57", "778", "414", "1531", "1167", "1426", "2438", "3119", "2258", "1529", "2807", "1795", "2552", "587", "1646", "2944", "3237", "3131", "1867", "2579", "1184", "1449", "1390", "3084", "1230", "2268", "1701", "2336", "2254", "626", "1112", "900", "3049", "1897", "1959", "1924", "2928", "3031", "1291", "635", "1991", "3263", "210", "2986", "925", "91", "3230", "2198", "1462", "729", "1932", "256", "1922", "1887", "3283", "249", "1776", "2377", "2486", "2122", "2410", "429", "2573", "620", "2692", "203", "2002", "3125", "1232", "819", "1481", "3106", "1571", "1050", "2081", "186", "1537", "3096", "3238", "1616", "1070", "313", "2049", "245", "121", "905", "195", "1901", "369", "679", "403", "2248", "1881", "3136", "3186", "95", "2676", "1950", "1284", "1559", "2289", "1575", "3108", "1320", "1299", "555", "2433", "1415", "1626", "2211", "1631", "1845", "1499", "1969", "783", "1799", "2423", "734", "404", "1844", "1083", "3307", "1712", "1523", "2186", "2929", "1691", "251", "540", "2610", "2199", "1199", "2693", "894", "2062", "3196", "947", "2018", "639", "2624", "2488", "2949", "2712", "1585", "3003", "161", "852", "2879", "3116", "634", "396", "557", "917", "3168", "981", "75", "2901", "2939", "3063", "2253", "2179", "2830", "78", "2914", "434", "558", "397", "318", "3118", "3270", "2832", "2430", "181", "2874", "2918", "500", "1330", "1458", "1937", "2800", "194", "3066", "188", "1655", "1040", "1160", "1864", "722", "2145", "1252", "823", "2553", "1234", "1315", "1767", "709", "726", "25", "666", "2008", "258", "2639", "209", "1392", "705", "928", "1077", "1716", "1300", "2528", "603", "1142", "145", "2114", "1140", "237", "963", "112", "3282", "2991", "2212", "1915", "899", "1090", "3212", "956", "2326", "2307", "2621", "1807", "1357", "1467", "2138", "960", "1695", "710", "2862", "3008", "124", "2857", "1450", "449", "2551", "605", "2052", "1108", "3287", "2333", "3305", "2861", "12", "2376", "1838", "3163", "375", "1688", "2954", "1868", "1131", "602", "3099", "1726", "3225", "1078", "1380", "1645", "549", "526", "83", "215", "998", "1314", "2387", "2416", "2910", "2277", "736", "2483", "153", "2756", "2959", "1603", "1400", "47", "1639", "2689", "898", "395", "2302", "3019", "3162", "873", "74", "1510", "298", "2097", "316", "2475", "1196", "1789", "2640", "2467", "1638", "1309", "2648", "2015", "481", "2155", "2435", "2839", "1884", "3315", "1755", "1262", "2741", "737", "665", "1910", "2317", "2905", "267", "999", "2004", "537", "412", "1707", "1010", "1596", "2392", "3065", "674", "2951", "2443", "3204", "3068", "2970", "1461", "3055", "1889", "1913", "854", "1557", "1831", "1368", "4", "2143", "353", "206", "1797", "114", "2107", "1752", "3184", "2622", "2295", "762", "3023", "2156", "2574", "3299", "1837", "1333", "1037", "1533", "2778", "2141", "506", "2044", "18", "1590", "2090", "1570", "3155", "1733", "2612", "1629", "1014", "2345", "2737", "2708", "971", "2656", "2031", "2927", "106", "589", "1981", "2911", "887", "3077", "158", "3319", "566", "1597", "3298", "2762", "1151", "50", "2788", "2162", "46", "643", "1013", "631", "109", "1058", "1899", "2050", "2719", "3251", "1968", "347", "465", "2727", "2161", "1048", "1238", "2204", "3226", "1084", "321", "2309", "2492", "36", "351", "545", "949", "2623", "1422", "2630", "2715", "724", "2597", "1027", "751", "552", "879", "1122", "1839", "1644", "1525", "2452", "131", "1226", "1480", "2730", "2963", "735", "1210", "1343", "1935", "2981", "2531", "939", "1148", "2537", "2094", "1534", "530", "1444", "176", "2365", "1975", "1563", "2755", "29", "2822", "1792", "2427", "3173", "3244", "2683", "2244", "1217", "2682", "273", "1465", "2270", "2657", "575", "2805", "1263", "1828", "892", "2770", "1277", "2504", "2232", "2286", "2801", "877", "1197", "731", "2414", "2023", "1956", "1793", "1092", "2843", "2283", "1957", "937", "344", "2889", "809", "1121", "1927", "744", "1393", "1684", "2733", "240", "2583", "2080", "607", "269", "850", "3138", "116", "2075", "2561", "663", "1878", "374", "1377", "437", "278", "148", "924", "76", "2174", "2482", "3256", "3312", "2386", "41", "760", "409", "3276", "1145", "1265", "2476", "1198", "969", "2560", "2636", "2829", "3129", "1063", "1698", "2434", "2169", "2691", "1667", "767", "120", "3042", "1296", "970", "2841", "279", "1814", "1276", "1451", "2906", "1378", "177", "1312", "1786", "3247", "2827", "2284", "1242", "2674", "702", "3227", "815", "2836", "695", "2464", "574", "1517", "782", "743", "2220", "2782", "1386", "1338", "61", "950", "2721", "1313", "1041", "2112", "1865", "2501", "1375", "3308", "3302", "272", "149", "2194", "2613", "2449", "779", "856", "2732", "2614", "2045", "2972", "417", "488", "2203", "794", "2979", "3271", "2436", "2147", "2952", "3044", "138", "3331", "1270", "614", "2678", "546", "2899", "1783", "1851", "80", "2351", "367", "235", "390", "3332", "2095", "1192", "1607", "280", "1642", "843", "2707", "831", "2603", "2975", "1282", "3206", "2355", "2670", "716", "2256", "2846", "2588", "3115", "2909", "1329", "680", "1549", "1656", "1877", "2759", "2189", "276", "1781", "2466", "1096", "1550", "3232", "265", "2227", "811", "2813", "228", "2498", "498", "1351", "346", "1408", "1876", "1725", "1880", "105", "817", "2632", "2882", "1065", "338", "1515", "473", "1850", "1746", "2655", "3194", "410", "3098", "980", "808", "3090", "1984", "2413", "989", "52", "1189", "1512", "712", "1576", "2748", "627", "571", "1293", "2787", "3228", "1094", "1942", "2185", "2729", "891", "2228", "2568", "2990", "3257", "597", "3293", "3070", "1946", "328", "1566", "833", "1460", "2085", "796", "2028", "2115", "2041", "1568", "519", "3274", "1318", "2893", "935", "2982", "2242", "2310", "1087", "1936", "3306", "1556", "585", "1476", "2288", "845", "484", "1761", "2460", "2419", "2849", "2209", "1303", "3073", "2117", "470", "2010", "2976", "3297", "2798", "1470", "2328", "1920", "1661", "68", "539", "2776", "764", "2170", "323", "2368", "2388", "2806", "1658", "376", "2740", "2536", "945", "1611", "1325", "1757", "1003", "1175", "1250", "1201", "1365", "3246", "2399", "226", "3222", "774", "1466", "1261", "467", "693", "1416", "594", "2515", "288", "3127", "419", "1694", "2548", "1674", "1940", "1475", "132", "1632", "1518", "2329", "1454", "339", "1117", "3024", "554", "1431", "2609", "816", "2321", "446", "2308", "820", "1398", "3002", "916", "3143", "1", "1966", "2752", "2088", "3281", "1826", "476", "880", "448", "2513", "2192", "3045", "2200", "1208", "1892", "1637", "598", "2264", "1437", "3177", "1373", "1233", "931", "1354", "801", "2129", "1286", "2572", "1135", "841", "128", "1736", "2305", "2817", "218", "2725", "1472", "2342", "1859", "262", "2061", "324"]]} \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft.config b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft.config new file mode 100644 index 0000000..b681784 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/aircraft.config @@ -0,0 +1,13 @@ +{ + "scheduler": ["str", "cos"], + "eta_min" : ["float", "0.0"], + "epochs" : ["int", "200"], + "warmup" : ["int", "0"], + "optim" : ["str", "SGD"], + "LR" : ["float", "0.1"], + "decay" : ["float", "0.0005"], + "momentum" : ["float", "0.9"], + "nesterov" : ["bool", "1"], + "criterion": ["str", "Softmax"], + "batch_size": ["int", "256"] +} diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist-split.txt b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist-split.txt new file mode 100644 index 0000000..e14d7fb --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist-split.txt @@ -0,0 +1 @@ +{"train": ["int", ["20720", "26055", "29223", "48857", "37477", "34672", "54127", "2593", "27737", "46788", "42365", "13355", "48024", "9155", "49041", "42818", "27198", "18826", "36083", "50082", "11602", "43786", "19218", "37843", "22180", "8338", "46325", "10100", "14453", "24589", "31499", "27887", "15496", "51194", "35832", "10101", "34203", "5547", "43446", "41844", "6908", "52372", "12756", "19", "46737", "14940", "43644", "13331", "51910", "577", "41828", "48207", "14434", "11359", "24400", "5667", "8638", "8409", "36632", "52386", "16105", "26276", "8482", "52674", "17990", "34303", "32787", "38220", "41504", "20286", "35468", "44178", "24570", "4025", "55210", "22957", "10120", "54222", "3049", "23142", "47585", "48012", "30999", "33277", "7669", "53416", "24168", "59780", "16350", "53190", "46841", "41579", "28853", "42482", "4667", "5133", "28140", "6955", "40149", "43882", "32105", "20797", "10130", "25981", "36222", "49390", "55991", "46123", "14091", "4619", "20805", "6057", "40552", "16003", "28716", "59825", "23947", "249", "19039", "3635", "43942", "27106", "49612", "54294", "28366", "32832", "49044", "41875", "25118", "24114", "34931", "28912", "44868", "4873", "1556", "18541", "48184", "49061", "46251", "36215", "45371", "13893", "38877", "50769", "42617", "44345", "14449", "25452", "23023", "9855", "19052", "57468", "824", "40794", "36816", "30515", "19232", "58761", "31993", "48554", "41717", "28557", "34776", "1385", "54955", "46414", "23821", "44230", "19298", "169", "20787", "34282", "45183", "36907", "30068", "45770", "15956", "992", "9742", "19211", "20155", "5074", "12370", "37818", "34344", "5615", "44577", "36600", "57033", "54460", "285", "41749", "21427", "32225", "2419", "46293", "54010", "21674", "50481", "33469", "55822", "37926", "11229", "16181", "44736", "8217", "4993", "41733", "8549", "22723", "55732", "8825", "44875", "10141", "18416", "38350", "40826", "54108", "50020", "17278", "36402", "58941", "11410", "33723", "52104", "53807", "15604", "53773", "28376", "49374", "44777", "8629", "3153", "20014", "12982", "56615", "48648", "31502", "45378", "378", "54748", "47436", "2486", "49339", "52119", "44732", "31270", "31268", "4893", "37781", "10754", "59499", "6079", "19279", "13875", "19193", "55854", "49939", "10173", "9511", "14299", "59952", "29510", "52698", "32274", "5099", "44021", "58291", "2387", "51700", "3758", "5425", "45445", "21272", "51871", "38481", "43412", "14959", "51969", "1042", "14013", "21460", "22494", "36786", "6564", "11732", "43458", "53153", "50890", "44858", "57339", "40215", "56726", "4172", "35520", "53648", "50788", "34745", "11571", "30477", "59380", "8624", "59165", "28672", "59925", "1529", "50698", "40671", "6635", "59564", "44804", "26683", "11523", "35447", "2661", "5476", "17814", "9199", "38425", "32306", "39290", "33627", "24234", "28878", "1013", "27412", "49741", "44744", "53137", "26570", "22352", "9930", "44910", "50135", "48032", "55866", "28187", "25291", "31151", "13677", "28402", "14963", "55177", "45523", "11879", "7687", "43219", "17978", "34480", "35221", "39970", "25663", "46989", "10361", "33148", "38543", "53120", "1230", "22328", "47355", "40272", "52440", "54122", "40204", "10324", "1241", "55681", "55502", "3552", "53480", "36852", "27567", "34393", "29204", "50771", "4476", "6715", "24618", "47939", "31742", "36716", "31064", "7606", "6068", "39086", "9535", "7806", "36486", "41883", "42666", "38100", "39431", "37899", "20833", "30287", "5828", "28567", "2956", "1118", "48599", "37521", "37039", "42930", "21056", "51696", "9203", "49359", "21211", "12626", "16366", "3178", "21508", "32429", "52012", "28232", "49645", "14394", "26558", "29920", "39052", "32095", "25655", "52158", "47793", "28907", "59333", "3122", "17057", "31136", "3534", "6182", "52568", "17536", "59953", "6505", "9313", "47047", "1169", "48706", "21428", "25438", "598", "11738", "15418", "12078", "6628", "36463", "29111", "55078", "30563", "2378", "28027", "3056", "32658", "21454", "41084", "39257", "7572", "3191", "37233", "13579", "46891", "22543", "5887", "49767", "54702", "41334", "50841", "55745", "28998", "18063", "30714", "9400", "54301", "39922", "1834", "31080", "22444", "23312", "26544", "44682", "59966", "56757", "13069", "9926", "49919", "42593", "34484", "54623", "49947", "57441", "293", "6695", "7497", "33047", "30823", "53430", "34852", "3400", "47804", "3929", "22361", "41497", "12608", "38953", "55670", "7718", "58335", "24360", "24933", "57696", "11680", "11178", "34068", "34160", "38332", "5847", "9978", "28561", "44645", "59963", "44905", "46094", "30000", "19213", "51804", "15866", "40454", "42396", "40962", "56555", "11426", "9062", "51262", "11319", "18706", "26356", "17661", "43717", "13630", "43750", "47810", "33650", "56790", "35079", "55526", "8766", "13223", "56539", "41403", "14243", "30126", "1804", "9791", "31779", "42382", "30589", "16027", "29164", "39477", "27776", "36144", "20301", "48503", "55631", "18709", "35050", "51419", "1702", "40479", "9918", "8730", "18962", "5970", "43663", "13917", "9267", "18436", "51190", "15433", "17722", "5895", "55099", "7312", "20879", "23577", "685", "23184", "7087", "48654", "31894", "27692", "32544", "34761", "28092", "42672", "1797", "36893", "27763", "39739", "32010", "17137", "42677", "19578", "26574", "57260", "6074", "15238", "26305", "42692", "24026", "28207", "2831", "59109", "44464", "35528", "15541", "27294", "35686", "30765", "11245", "41047", "36964", "7102", "13725", "54276", "17810", "58173", "33405", "990", "27242", "31969", "13601", "50257", "52126", "11982", "59488", "54340", "11107", "45762", "6006", "53363", "30712", "47423", "48089", "25548", "28772", "45033", "32998", "14972", "21716", "44580", "28015", "10406", "43186", "227", "16317", "42182", "55936", "42779", "33079", "42568", "2934", "10018", "16109", "23380", "32062", "27820", "53429", "55927", "8961", "30405", "3130", "1740", "26204", "56712", "616", "18450", "53123", "20680", "50169", "46342", "39046", "8823", "35288", "25628", "213", "45729", "172", "23084", "39201", "25356", "8748", "36004", "4967", "12547", "8360", "52501", "27676", "12367", "4964", "48703", "21594", "38880", "27751", "17981", "47534", "12950", "2492", "9393", "10966", "11230", "58861", "27054", "7897", "58101", "48324", "20894", "19862", "47390", "55935", "35090", "52379", "26822", "52303", "55581", "21918", "21742", "16121", "53245", "39798", "25376", "56758", "2698", "42739", "12681", "38894", "22961", "19513", "23209", "54149", "28066", "26751", "4387", "47097", "39554", "25671", "46959", "59468", "35423", "44037", "33827", "46309", "24227", "17265", "17961", "17282", "17122", "41004", "45743", "53259", "14407", "21565", "35035", "45317", "38835", "25905", "15818", "19397", "58153", "1878", "13487", "20363", "58739", "33380", "35232", "43654", "29275", "58288", "26481", "24268", "29723", "19257", "55780", "20541", "8778", "12430", "19522", "18664", "10537", "24997", "30971", "26884", "49959", "29021", "18158", "20962", "38454", "3175", "10969", "2004", "18333", "30201", "39709", "23333", "9476", "17199", "36590", "15076", "21088", "38270", "5818", "23102", "6470", "58576", "3342", "12786", "25629", "19509", "23226", "59028", "51170", "15705", "34690", "34552", "19423", "44212", "41102", "36724", "34036", "16205", "28569", "21766", "30743", "31982", "59467", "41745", "20133", "18360", "1251", "47966", "29991", "54225", "50214", "55827", "46557", "10596", "10802", "50836", "51586", "27423", "42801", "52001", "29503", "58395", "3847", "42336", "32031", "22243", "19234", "42776", "7258", "54491", "47472", "37909", "57043", "11726", "48595", "36797", "42167", "53740", "23434", "54129", "35012", "53665", "20484", "46772", "37948", "31850", "22946", "58332", "6536", "14716", "48422", "18568", "58236", "26571", "40214", "44833", "34187", "794", "22763", "38909", "45022", "59006", "8018", "52844", "4804", "53774", "36672", "10443", "34919", "12423", "24895", "10235", "7154", "13556", "26614", "21084", "32496", "16946", "28625", "42001", "42085", "33955", "11412", "38108", "57057", "50982", "46894", "18510", "12952", "37758", "42686", "49507", "18182", "55704", "10816", "59389", "11266", "11322", "55727", "26468", "39424", "58423", "17614", "29183", "3908", "13580", "41352", "10169", "16701", "40951", "16305", "20076", "14207", "23233", "55947", "41536", "37460", "57180", "294", "20493", "45711", "27032", "54030", "21274", "8697", "15331", "20207", "36390", "9615", "36003", "56314", "23815", "26732", "35566", "11064", "40426", "36995", "11641", "24118", "2188", "2195", "26795", "34421", "51653", "14594", "6578", "31676", "24389", "43303", "34628", "54652", "38255", "35396", "48813", "30093", "706", "56954", "29154", "6845", "46971", "26885", "20185", "49885", "42005", "59999", "50992", "12621", "11355", "27154", "49827", "5604", "21570", "33324", "40843", "40798", "35580", "22816", "46651", "2452", "32757", "33211", "22075", "2147", "55167", "42125", "2226", "57427", "26826", "4098", "9670", "3652", "34367", "53916", "32926", "24665", "18993", "39697", "29025", "23405", "37525", "25667", "40443", "37727", "1733", "8076", "1433", "37265", "12692", "14838", "48442", "12745", "30693", "34094", "44728", "28018", "40709", "35411", "28489", "34089", "16647", "2234", "7269", "30274", "28118", "30276", "9875", "10690", "20762", "45740", "50792", "33860", "20162", "18965", "49487", "44572", "22396", "48045", "25341", "20834", "21162", "39804", "678", "46969", "41180", "7167", "52328", "27372", "31446", "42081", "9073", "43120", "28148", "44999", "3850", "1402", "9501", "59071", "4128", "11763", "41342", "26805", "45482", "34948", "10777", "56875", "57181", "31498", "2912", "32741", "28917", "20577", "15600", "56724", "55543", "54367", "8687", "47141", "43773", "6328", "49371", "21306", "38268", "18232", "45035", "25681", "50270", "38896", "47357", "45805", "25448", "1391", "24710", "59367", "44348", "40489", "40508", "7078", "17668", "23724", "35722", "5209", "41346", "42333", "52763", "482", "24202", "1448", "17414", "13160", "27286", "11104", "6352", "56286", "58110", "6530", "41784", "35728", "58639", "34904", "8387", "24391", "56366", "5410", "20288", "9291", "41037", "51775", "19387", "38068", "52370", "14696", "57903", "21293", "52883", "10199", "13517", "20995", "56500", "29345", "19034", "47719", "37205", "43025", "57844", "20489", "18803", "50007", "23361", "37059", "45576", "11626", "53683", "35587", "4500", "34432", "4164", "29989", "17620", "21740", "39989", "53920", "33367", "25152", "4663", "19669", "33171", "24371", "58995", "54854", "43734", "42176", "44915", "37415", "49584", "46750", "31083", "33253", "17390", "33244", "35417", "55144", "11933", "58092", "55463", "49331", "4989", "24425", "45038", "7553", "21818", "12596", "18947", "22339", "36070", "5616", "44550", "37151", "15978", "38827", "25080", "57557", "3763", "17190", "38696", "29798", "4699", "29568", "14681", "1722", "508", "23079", "3475", "24353", "27639", "43286", "39449", "41775", "7361", "49897", "20648", "58751", "35379", "8440", "30910", "24999", "38902", "23694", "57089", "44379", "21582", "3666", "50849", "36636", "59528", "45077", "46514", "38417", "30690", "20256", "47136", "9263", "54223", "20048", "41204", "29180", "57647", "5831", "52902", "5325", "53812", "14115", "22713", "53962", "33746", "53167", "42949", "56928", "49178", "12387", "31214", "1043", "26566", "7514", "8659", "2049", "45437", "41938", "24352", "37747", "9069", "5054", "41129", "37391", "57428", "23159", "14860", "51796", "9604", "51717", "22732", "37071", "9076", "30523", "35917", "33585", "51405", "59661", "32789", "59648", "26686", "2704", "48254", "50", "50178", "17304", "46570", "8235", "42231", "42285", "2094", "6171", "2075", "49301", "43578", "6084", "48437", "54256", "55654", "5437", "55367", "58732", "2703", "6386", "56093", "45436", "27655", "39284", "13329", "44578", "40291", "12226", "11222", "22621", "17430", "17453", "10309", "54666", "55300", "58247", "39792", "14011", "12971", "16049", "8353", "8555", "8709", "39699", "24379", "27826", "18686", "51402", "39019", "55298", "13040", "40765", "24546", "57617", "23771", "11562", "50935", "16167", "45574", "11679", "49735", "36686", "12021", "58036", "37376", "8701", "5064", "54318", "29646", "26740", "37912", "8867", "46016", "48493", "19712", "56051", "29978", "33185", "6515", "17362", "55547", "35513", "9905", "26185", "40652", "33857", "6998", "29586", "29309", "11016", "52232", "29770", "9678", "19440", "48729", "29419", "44546", "23247", "14468", "24158", "26519", "32226", "45946", "9640", "14049", "31207", "22671", "53506", "51782", "20549", "17894", "45238", "23105", "24795", "47582", "56637", "8911", "59413", "56280", "16401", "4151", "9323", "50149", "561", "9697", "16526", "27747", "35029", "52414", "17326", "19344", "8368", "8015", "3894", "23786", "44841", "8055", "30445", "43466", "52090", "14697", "2450", "52601", "30471", "58447", "526", "16607", "20195", "42850", "9280", "18934", "38536", "39409", "53704", "26764", "47948", "39304", "18500", "47859", "4402", "34470", "32853", "977", "284", "29495", "3926", "21966", "41972", "4033", "19436", "8816", "55910", "28915", "13768", "52376", "40159", "52483", "36619", "38198", "11791", "30921", "28748", "15499", "626", "51128", "28256", "9493", "57176", "21731", "18779", "36674", "3008", "14551", "26272", "6275", "30822", "36108", "59020", "42297", "20352", "1014", "18835", "22596", "24707", "22570", "1408", "42217", "34839", "49504", "35602", "19478", "58477", "19848", "56596", "34833", "19472", "15916", "52505", "1026", "14648", "8599", "3255", "25326", "8407", "32119", "51432", "15317", "13528", "2491", "33463", "14397", "12669", "45207", "58122", "46276", "5121", "3573", "34583", "40805", "44741", "22836", "27983", "58991", "26487", "40140", "24608", "37735", "30849", "17317", "29340", "16652", "30614", "34200", "55432", "10586", "44355", "9376", "13882", "27238", "10927", "58760", "20967", "24240", "14490", "33974", "50630", "32207", "27405", "16246", "10806", "44303", "3710", "53298", "15059", "50255", "5051", "402", "23720", "12103", "35752", "27477", "3944", "53859", "21639", "12645", "18202", "54094", "10613", "39190", "51157", "37089", "59215", "43294", "22794", "54969", "31280", "13728", "14682", "48165", "24509", "26395", "41721", "26022", "59342", "4044", "25533", "8857", "29955", "27290", "34182", "57405", "56952", "39026", "35761", "26074", "32289", "55206", "43307", "37754", "39688", "45361", "11681", "42901", "53246", "15485", "21101", "49388", "39255", "2385", "28529", "45567", "57017", "27587", "16179", "43008", "41960", "13300", "25771", "13145", "46108", "15477", "43664", "29960", "26030", "47006", "16527", "1020", "28295", "39379", "35989", "51214", "14841", "47232", "17862", "25787", "15556", "15137", "55628", "17096", "47631", "18792", "51085", "10401", "12205", "52848", "48647", "46009", "50828", "58716", "51765", "3137", "50292", "54097", "4952", "6760", "21233", "19209", "31191", "32179", "59176", "31528", "28448", "39874", "2834", "18707", "28927", "38736", "8411", "44961", "39867", "52510", "7049", "4182", "11624", "39783", "37536", "30703", "51891", "960", "39590", "39685", "21870", "1090", "28319", "50103", "31463", "3505", "57475", "14563", "40757", "53001", "14076", "52467", "31670", "33456", "37183", "21155", "39632", "31353", "44799", "24337", "3341", "15630", "27430", "55624", "29406", "29889", "4522", "21414", "39545", "1688", "29961", "50884", "8435", "38830", "58209", "2862", "10433", "50921", "48237", "33145", "28039", "45906", "45293", "12017", "59964", "41057", "2310", "11420", "5786", "19058", "4276", "36291", "16449", "20302", "15598", "18995", "3138", "1531", "41965", "13947", "47963", "37568", "29791", "6992", "48117", "15301", "1585", "19514", "47608", "12873", "20898", "39596", "51010", "14310", "33920", "3239", "416", "49609", "32755", "33196", "30779", "18193", "583", "13101", "19598", "5845", "53717", "37958", "42580", "18290", "7325", "28382", "20523", "32122", "51931", "11842", "50542", "48781", "1267", "4877", "24566", "26179", "16704", "48205", "33425", "43626", "53458", "11382", "42639", "29232", "25284", "28012", "1728", "20563", "45875", "20775", "50188", "48868", "32564", "43722", "35161", "6535", "24780", "38710", "6375", "32703", "14892", "3528", "821", "58115", "7421", "45827", "34096", "45514", "34568", "59665", "55775", "26477", "22546", "25695", "48635", "58878", "24473", "36982", "7041", "23392", "57252", "49296", "40302", "43643", "24358", "26342", "3429", "607", "34180", "44065", "34837", "20405", "59074", "43641", "52632", "8955", "51340", "28833", "59340", "27556", "1605", "3123", "49378", "117", "28896", "24335", "54019", "4669", "54758", "15682", "55992", "57895", "16345", "35821", "36603", "27912", "51614", "4959", "14667", "50987", "40091", "36542", "50764", "12953", "9895", "39271", "11854", "14685", "18839", "33232", "47212", "43474", "38343", "23810", "44857", "18452", "40266", "33229", "49084", "55731", "47971", "34136", "46306", "54881", "14471", "59693", "42283", "23959", "59344", "37866", "35163", "38651", "4267", "58929", "53623", "17499", "14849", "459", "3522", "44316", "38618", "35224", "38577", "45970", "52047", "50983", "21791", "5890", "2447", "30979", "34633", "50291", "27008", "6934", "8325", "21889", "44319", "1771", "19097", "56914", "31762", "4717", "16655", "21762", "58799", "25991", "5737", "7686", "34395", "43521", "16782", "24116", "13477", "44808", "49367", "2015", "50124", "6495", "43624", "16127", "15756", "5998", "59328", "1144", "47997", "36998", "10743", "23276", "42806", "47026", "13162", "36317", "40436", "5870", "53138", "15035", "42318", "14186", "27493", "57621", "49706", "57285", "21531", "7962", "31404", "30984", "6582", "15103", "42947", "30263", "29731", "2523", "4244", "16481", "36821", "48338", "53609", "34249", "6252", "30791", "44441", "59659", "40558", "41734", "6364", "3389", "47639", "8813", "22830", "19308", "54298", "29438", "46152", "1283", "42960", "34240", "39294", "9674", "33565", "22341", "54707", "27760", "35563", "29743", "18864", "42703", "4379", "325", "52", "37861", "27082", "7830", "55781", "12473", "16319", "49689", "46463", "36105", "50623", "10711", "50484", "48578", "46138", "40428", "6949", "26296", "23657", "35342", "45419", "52251", "50536", "3751", "10181", "30771", "41996", "11973", "25227", "31579", "46709", "49431", "3197", "43189", "23098", "40321", "672", "2532", "48356", "59244", "37446", "22849", "19781", "6449", "4442", "44398", "53547", "22468", "51009", "9084", "54837", "54781", "20096", "18005", "50416", "41471", "14584", "25428", "34194", "42357", "12560", "46798", "17401", "6011", "52543", "44847", "17853", "41158", "36420", "25490", "40726", "25147", "16892", "28520", "10340", "38587", "29797", "38237", "54015", "33750", "47258", "55067", "45468", "18951", "44980", "45255", "56488", "53263", "56364", "13467", "51117", "23022", "2466", "51865", "20773", "20135", "1520", "10493", "53296", "15880", "56654", "4788", "58933", "8674", "25601", "47268", "34580", "36451", "51544", "8355", "43810", "11162", "32080", "31934", "19950", "17478", "12019", "48017", "27577", "46520", "19054", "37221", "52233", "3728", "49646", "6960", "5863", "25007", "26223", "1115", "35667", "12353", "31717", "35242", "23330", "20020", "9594", "43122", "30957", "40923", "52910", "52064", "35228", "31666", "10413", "42046", "15274", "31456", "8938", "54339", "30148", "47922", "25946", "27830", "389", "42549", "20672", "43913", "32536", "31114", "26643", "22260", "5707", "36913", "8975", "21346", "2822", "5455", "58775", "31027", "23921", "37956", "14925", "15987", "27012", "29258", "47500", "53730", "30117", "174", "5065", "19603", "45144", "6811", "1780", "20922", "53569", "52744", "28131", "19114", "43295", "24972", "37121", "31937", "3866", "33894", "45886", "8494", "22093", "19294", "25931", "55163", "32868", "53785", "15551", "3785", "30088", "54146", "5946", "45498", "13836", "54209", "37792", "4514", "34615", "534", "15735", "49942", "32420", "57488", "48086", "33628", "18151", "49466", "31533", "30478", "699", "12043", "57005", "15632", "56711", "30273", "6851", "32667", "31777", "20657", "39779", "47959", "51359", "14969", "44978", "33298", "33525", "52122", "53845", "20368", "38023", "13906", "11000", "42830", "11566", "40988", "1534", "36379", "59134", "18689", "18632", "44136", "52429", "18199", "20066", "34506", "7451", "36261", "41214", "14641", "16253", "33695", "2240", "21774", "5222", "55377", "55908", "58428", "18282", "50094", "4822", "23336", "48211", "6228", "4744", "46536", "46789", "42787", "58828", "15060", "48460", "26352", "51021", "8553", "43590", "53842", "1769", "16557", "57551", "34377", "22070", "45818", "29069", "36588", "21562", "37745", "6953", "59537", "4721", "42214", "36642", "28710", "13807", "6098", "51281", "54730", "31554", "54820", "12584", "23804", "47184", "18503", "28543", "28848", "936", "20362", "44492", "38414", "44336", "22776", "7220", "31837", "54667", "5984", "48758", "49478", "51643", "2382", "11437", "51711", "4597", "32879", "24515", "56048", "51933", "28534", "20872", "17755", "23508", "25495", "4776", "7113", "37269", "4444", "33176", "22871", "46081", "35169", "4790", "13870", "59914", "27897", "17898", "18773", "57071", "5872", "59010", "25869", "24531", "9125", "6911", "37196", "40352", "41838", "16171", "27989", "4400", "15627", "48632", "22063", "4225", "10172", "58779", "52525", "32313", "10222", "12688", "48167", "44221", "1507", "16387", "15837", "7367", "57362", "24848", "41837", "27295", "3980", "43589", "42165", "59681", "58856", "49089", "32284", "24094", "33636", "1835", "16881", "36751", "36788", "37186", "11338", "27016", "34424", "46386", "19864", "2656", "41280", "9072", "8710", "38084", "59486", "48380", "35977", "1986", "26710", "18733", "50389", "25776", "50972", "41153", "16740", "31331", "6991", "1185", "48660", "57747", "43974", "19464", "6721", "29760", "26165", "32843", "33685", "11648", "35093", "9047", "1500", "32264", "17993", "25584", "55877", "20832", "11297", "7517", "55472", "25027", "26865", "17775", "43693", "32534", "45737", "52795", "59706", "26665", "7381", "57821", "25223", "20421", "1290", "15455", "55211", "23008", "181", "27217", "31053", "46627", "5431", "54890", "32163", "55458", "56595", "45941", "47083", "22115", "42800", "35866", "29208", "48791", "11886", "40568", "5139", "11015", "46879", "31071", "43871", "20348", "37526", "59742", "37681", "25265", "56838", "41130", "44005", "48797", "11184", "51041", "49964", "35698", "21850", "54784", "45180", "10090", "41402", "40702", "40334", "9897", "57585", "58790", "34650", "43728", "57830", "26164", "37969", "26105", "25912", "26852", "38193", "41558", "32935", "30034", "23048", "6326", "44917", "59872", "39383", "24285", "31996", "48423", "15864", "37663", "12677", "32449", "8362", "39514", "41581", "50086", "43701", "25487", "28857", "20246", "21585", "18012", "21117", "19577", "52998", "52686", "24231", "18758", "4229", "17951", "19296", "26205", "54138", "51728", "14560", "31907", "13423", "56035", "10543", "33194", "29867", "31541", "5354", "22588", "53063", "38524", "40707", "59336", "35795", "9732", "31156", "53515", "52511", "25827", "34683", "9089", "37057", "31697", "21603", "34417", "35462", "24926", "17750", "39535", "21896", "18946", "48821", "42195", "20025", "21139", "8546", "20674", "3067", "38194", "3784", "12153", "51035", "8491", "32763", "43373", "22846", "9701", "43783", "46584", "27841", "56763", "27182", "10186", "38151", "24601", "39404", "9117", "39910", "3370", "39584", "55598", "4510", "12176", "21079", "56730", "45853", "11561", "36823", "19525", "12658", "14092", "8577", "54701", "51096", "28769", "44006", "12442", "41930", "54472", "4181", "27671", "22886", "51549", "33230", "11035", "7898", "48709", "53284", "21138", "56728", "9213", "58675", "21240", "9636", "17434", "33884", "43099", "25564", "14431", "50742", "26201", "37207", "32813", "50576", "55477", "27733", "13025", "3169", "57991", "54913", "18646", "54972", "33590", "24757", "19473", "43262", "16103", "25051", "28491", "57175", "20733", "38629", "2010", "35045", "5015", "51146", "56682", "35077", "25437", "33046", "51715", "5326", "5205", "23372", "19938", "6040", "39738", "3810", "26575", "16136", "41934", "2583", "47203", "50078", "54591", "29820", "46226", "16865", "16443", "20739", "37254", "35251", "26959", "385", "38833", "11691", "24679", "20283", "46447", "25134", "20434", "22582", "28708", "32353", "40915", "21174", "33060", "46907", "21200", "20825", "34292", "45198", "43986", "12675", "25095", "52867", "29660", "6425", "5079", "16891", "3786", "57877", "19063", "39822", "20953", "8008", "58116", "33634", "26484", "31847", "42337", "28178", "59330", "24657", "44958", "7396", "4773", "48517", "34452", "45905", "22464", "52077", "31279", "30607", "18781", "4871", "53018", "27871", "55703", "16471", "41981", "500", "36969", "10217", "38811", "903", "14801", "2578", "15845", "43402", "54610", "28077", "54382", "27843", "58664", "2596", "48560", "55951", "25534", "17228", "28151", "23761", "47111", "54348", "18207", "55730", "17719", "7670", "42350", "56476", "29825", "54783", "15370", "16447", "49642", "11715", "5533", "39753", "58486", "59962", "55566", "37307", "5200", "3005", "27387", "22037", "2241", "20850", "59715", "39237", "13002", "13599", "18370", "42768", "54039", "16862", "47301", "13241", "11903", "11354", "10369", "1173", "11418", "27959", "9346", "51246", "47707", "42423", "39226", "23493", "2538", "44052", "27785", "6884", "43109", "34117", "19852", "28766", "46923", "37513", "12028", "29845", "59816", "17791", "47323", "32267", "51062", "43800", "40556", "46518", "6488", "36990", "21641", "9940", "32881", "46595", "30390", "43802", "52141", "23566", "29915", "35705", "58683", "24299", "9643", "56931", "39674", "47876", "30374", "59097", "50770", "11236", "55922", "27686", "25403", "47329", "46274", "49969", "23115", "29320", "51763", "40186", "16547", "5542", "22806", "41474", "56736", "13098", "22039", "73", "35638", "23863", "28502", "32818", "54578", "2533", "16959", "13863", "1141", "47982", "10674", "126", "28311", "58942", "42664", "16247", "711", "59764", "59439", "11353", "2", "23202", "59062", "44310", "59469", "8427", "33553", "55092", "5802", "57590", "36942", "5983", "15535", "20345", "32402", "37585", "58782", "15058", "11569", "3421", "24647", "8426", "14387", "7743", "51120", "3427", "36351", "9341", "8138", "39055", "33393", "57211", "44039", "40761", "30301", "55773", "19891", "6388", "31683", "14949", "22013", "40080", "12871", "52636", "7914", "14879", "10315", "57812", "22144", "17021", "29103", "31638", "27937", "7186", "35290", "15364", "54471", "26239", "11026", "58692", "25245", "28392", "43679", "43542", "59036", "54197", "28206", "25978", "14451", "54620", "19942", "30802", "7554", "32432", "29290", "45608", "20009", "16996", "41914", "27537", "12057", "7847", "54213", "21553", "28203", "13319", "55870", "38241", "20406", "4461", "19476", "44451", "9907", "50081", "41542", "9752", "58863", "11475", "14534", "42024", "11999", "48382", "45480", "27063", "47568", "23584", "14814", "6672", "30657", "49051", "48104", "5645", "43248", "53843", "34791", "52654", "21566", "21129", "59268", "3595", "16751", "32033", "18348", "4786", "50064", "1455", "24514", "40825", "14215", "22758", "35863", "51055", "10885", "46938", "16712", "3253", "37804", "39238", "11588", "44135", "34410", "34649", "58297", "47486", "53714", "47955", "11240", "47577", "20336", "34050", "40483", "10839", "56191", "56439", "16771", "3098", "28134", "29361", "15228", "16364", "53527", "2337", "28314", "32302", "35488", "19991", "17220", "1537", "45407", "41444", "44935", "47648", "32203", "5493", "24099", "14636", "1620", "53635", "44299", "3995", "25976", "29543", "514", "15736", "49025", "23462", "11873", "25605", "45899", "32003", "43398", "38702", "1387", "44012", "31439", "16467", "25945", "32742", "16383", "13447", "16150", "22505", "3608", "12761", "10851", "8087", "47817", "38319", "23213", "33358", "24199", "40090", "49663", "22607", "50817", "49557", "30067", "20704", "30170", "16007", "40780", "35011", "44493", "34882", "52087", "47992", "33056", "49282", "51642", "28302", "41885", "33069", "11349", "1915", "3449", "13285", "14173", "14151", "54583", "46072", "24905", "37740", "37973", "6220", "33068", "50060", "57642", "2289", "19145", "35421", "48269", "27818", "21411", "36834", "52872", "46925", "13257", "38703", "48402", "20482", "36887", "22695", "43047", "8626", "37135", "39771", "48498", "49302", "183", "15763", "14029", "50058", "484", "28130", "14044", "18622", "56746", "7194", "59029", "29079", "25560", "37252", "10004", "288", "55523", "58840", "41240", "44771", "1671", "2281", "35346", "43980", "55440", "5324", "50552", "47371", "13250", "21001", "3022", "31829", "23126", "10108", "44727", "54068", "37219", "16035", "38784", "43431", "50343", "40242", "27925", "24196", "12574", "16441", "45434", "41028", "8516", "209", "16114", "34634", "27075", "39733", "58380", "17905", "14048", "14501", "7852", "50835", "50818", "27201", "14622", "46820", "5271", "42431", "3997", "47256", "19882", "5640", "58120", "49893", "28615", "41699", "6492", "51148", "1163", "7801", "49127", "55492", "20309", "50099", "26690", "26699", "23342", "41604", "5624", "55322", "45962", "6346", "46846", "44436", "28687", "30893", "53203", "45823", "2889", "33133", "12557", "5777", "2009", "45636", "662", "48467", "53793", "7175", "38975", "34708", "47644", "52507", "4368", "3089", "30318", "34978", "39476", "50216", "19877", "24211", "35309", "42200", "42075", "27001", "26704", "48429", "27033", "13810", "19137", "43506", "36477", "32646", "22572", "33778", "41625", "23619", "52384", "51114", "17425", "6809", "39233", "8988", "27148", "31659", "35295", "44045", "31987", "55019", "8848", "51044", "10007", "58009", "19199", "24504", "35959", "22288", "47662", "14251", "35769", "19696", "54828", "49101", "48225", "32193", "38942", "6661", "20393", "16713", "24996", "34834", "4519", "57490", "35793", "27884", "16961", "35071", "34582", "30078", "12413", "51553", "15613", "26618", "29640", "26092", "41087", "38612", "59487", "52365", "32541", "14848", "18045", "12933", "23944", "49491", "18678", "37195", "933", "49135", "41693", "11023", "19331", "24673", "28265", "49679", "17842", "4951", "8611", "9081", "28852", "13748", "13006", "45738", "37914", "19493", "50847", "15954", "16307", "18330", "35848", "3343", "42965", "57118", "42555", "32088", "18260", "36369", "35806", "56337", "23069", "11849", "40148", "40781", "58573", "20365", "10965", "8309", "25425", "19821", "18425", "40883", "1094", "14210", "6165", "58207", "45280", "20594", "39324", "36320", "14418", "25565", "46574", "11321", "57641", "2472", "49712", "45477", "4535", "2501", "51894", "46516", "56270", "54769", "1758", "4111", "3551", "58590", "41674", "56427", "26769", "9349", "18975", "14723", "28042", "14666", "17848", "14231", "34911", "57730", "26447", "22317", "872", "31260", "47424", "52972", "29353", "39486", "57085", "17859", "5508", "27869", "30838", "37065", "30378", "47088", "46454", "44102", "23850", "6817", "777", "17557", "54343", "25287", "46312", "18421", "41839", "50729", "27614", "30121", "10956", "2187", "6942", "365", "24343", "7809", "6865", "31478", "21503", "11979", "30241", "50379", "46573", "4192", "7166", "16585", "38067", "56030", "21618", "39509", "23714", "16524", "8096", "34063", "32844", "43239", "45703", "15211", "40896", "56479", "12018", "55911", "37509", "887", "7952", "10534", "23285", "41114", "7570", "19998", "2182", "30891", "11698", "51205", "38437", "31448", "54566", "6408", "18984", "20622", "20198", "59899", "2266", "25716", "54393", "40547", "9536", "53697", "57501", "29217", "44272", "25261", "15825", "44872", "44794", "34681", "17651", "34355", "39444", "53318", "32476", "38389", "1117", "14402", "40028", "16580", "40921", "38069", "13296", "10811", "49907", "1754", "33023", "22647", "41271", "3128", "37951", "43289", "42613", "20370", "15710", "59270", "42823", "13551", "57404", "55387", "11139", "54526", "45541", "20171", "55459", "48349", "27927", "54196", "27062", "53519", "52481", "53833", "47482", "20371", "24102", "36288", "31644", "20606", "30479", "7083", "323", "7050", "30286", "43170", "51279", "16716", "22533", "57772", "46503", "11721", "727", "38748", "36813", "59160", "52790", "29086", "55384", "50577", "19558", "57143", "4299", "2626", "2629", "45125", "1343", "40997", "24551", "2326", "46255", "7356", "15924", "25867", "46200", "37497", "57883", "19906", "6771", "23293", "47017", "34137", "27643", "32373", "15268", "35258", "46269", "14200", "24089", "13274", "44880", "48645", "59691", "15868", "2216", "29026", "51404", "5442", "1911", "43833", "4659", "4808", "27229", "29102", "50608", "26317", "2540", "49892", "18486", "15830", "19939", "41920", "4540", "22428", "39829", "56525", "12809", "6480", "53565", "9302", "55673", "12258", "5596", "17849", "38768", "1051", "38773", "43678", "10601", "26421", "28829", "4935", "54171", "9886", "49708", "56699", "54195", "57094", "32574", "32025", "1977", "9821", "59300", "46218", "8292", "59512", "53984", "37679", "15953", "48606", "52825", "20153", "32904", "14858", "48741", "20702", "4610", "34407", "37543", "44705", "37870", "28580", "42111", "42705", "8340", "8764", "55659", "55929", "21901", "59283", "34441", "40375", "13546", "14420", "34884", "49286", "33611", "23461", "18050", "42688", "14620", "16074", "32578", "49904", "18419", "9975", "8071", "9524", "42162", "59606", "30575", "45982", "27162", "47640", "56520", "47730", "33488", "11260", "50795", "14064", "6445", "3508", "20961", "17880", "26525", "2589", "36855", "49923", "26141", "50656", "35657", "58030", "33946", "36536", "46922", "23624", "3888", "47820", "23780", "54918", "8791", "21994", "43525", "44146", "52498", "30460", "20891", "44022", "27593", "8128", "24913", "11723", "36962", "5764", "32531", "55844", "21151", "51048", "11820", "35762", "35015", "36872", "58475", "20638", "17440", "38765", "59070", "58710", "48283", "28335", "54706", "40010", "20580", "54928", "35558", "57462", "19178", "21989", "1304", "6482", "39491", "41635", "34381", "6947", "59718", "14405", "5993", "37213", "51701", "29939", "48967", "2400", "37890", "37272", "19191", "32394", "52023", "25206", "25094", "49999", "42689", "23414", "47556", "8951", "45865", "6653", "28735", "49073", "3366", "50696", "49102", "21954", "30339", "53495", "13432", "48820", "19046", "59813", "14938", "29031", "20542", "50532", "18488", "13177", "29233", "59219", "1904", "12147", "49579", "10351", "41215", "10859", "51145", "30764", "9648", "9665", "19221", "35673", "16491", "32985", "17119", "56777", "9992", "51732", "31406", "13193", "30446", "2106", "37837", "45626", "11060", "43766", "35109", "8852", "15713", "35708", "3420", "23670", "18528", "50186", "3286", "5812", "36482", "51322", "53014", "8294", "50525", "5876", "9340", "3492", "31529", "32859", "12803", "51987", "55861", "16843", "32462", "17240", "9494", "5329", "23764", "17866", "10570", "24169", "26742", "9672", "6291", "55287", "24441", "1359", "28393", "7676", "16627", "33424", "46837", "58179", "43330", "21108", "26956", "23550", "2170", "46663", "20439", "25167", "33472", "47404", "4795", "6214", "52168", "31108", "23110", "47510", "11605", "4892", "19460", "17500", "33610", "9431", "44474", "20758", "37734", "38043", "13479", "53988", "59235", "8630", "24765", "29238", "26780", "943", "25920", "22157", "38346", "16993", "26854", "22460", "28711", "52669", "42581", "49800", "37649", "15896", "13699", "19672", "59951", "51802", "2428", "37798", "24427", "6350", "15490", "40281", "12331", "418", "13632", "58307", "6592", "38018", "865", "36863", "11340", "29105", "20084", "19589", "2138", "39844", "52022", "33100", "37129", "48477", "3252", "32743", "35830", "31491", "57251", "22766", "5942", "5525", "52537", "10330", "59169", "49420", "17261", "53756", "30758", "39039", "22899", "13682", "10150", "49205", "55591", "1330", "7057", "48544", "17973", "19804", "23318", "3582", "6090", "48141", "58824", "26913", "536", "45511", "53690", "40178", "4531", "3544", "44927", "27464", "24461", "48881", "24935", "2262", "2366", "6354", "21609", "53494", "21098", "58872", "19985", "50157", "16815", "35168", "24960", "702", "49443", "26490", "823", "15288", "55639", "48814", "5898", "56796", "28678", "9411", "49023", "5693", "3476", "19823", "59604", "36184", "49640", "44126", "15709", "21822", "17595", "16571", "17578", "45965", "51159", "7564", "34616", "45625", "6770", "57795", "47706", "15010", "56037", "14417", "33375", "56953", "18627", "38904", "50862", "2695", "30808", "47616", "45499", "1317", "57688", "41167", "45967", "47035", "45951", "29389", "27987", "55280", "19910", "51739", "50489", "32573", "7386", "7403", "8729", "32057", "39287", "3445", "52192", "2438", "23543", "37433", "51888", "59724", "45616", "3299", "42265", "15862", "24592", "51389", "52927", "25381", "14698", "26080", "13010", "11144", "6117", "23835", "45851", "37429", "38546", "14904", "31025", "27855", "54084", "52202", "17311", "19290", "4725", "6134", "44284", "23169", "56810", "45921", "18721", "33454", "40008", "38168", "57009", "57755", "7376", "24329", "1339", "28477", "39266", "20480", "51025", "42598", "35337", "58218", "46242", "9225", "27885", "36328", "8800", "31175", "23569", "44694", "54425", "41063", "39932", "50797", "15236", "19391", "3984", "38763", "8655", "28490", "45607", "5568", "40051", "52281", "20734", "49247", "27485", "30769", "51178", "56057", "28359", "53126", "10529", "28008", "2960", "43257", "20170", "23141", "30777", "49204", "44409", "38473", "13656", "21587", "41337", "39672", "56381", "50584", "684", "42896", "29625", "11801", "20851", "31912", "10545", "3563", "45650", "59213", "25017", "20703", "24600", "3882", "3952", "51095", "6855", "12088", "42534", "59024", "14882", "24298", "33543", "18092", "22842", "39960", "25209", "8428", "5012", "24000", "19201", "5908", "36461", "9357", "45248", "15848", "46613", "53831", "41349", "35387", "7392", "46566", "26587", "50423", "57357", "45845", "6334", "39299", "105", "25940", "29335", "24375", "46799", "692", "9782", "41519", "51940", "55175", "57377", "33701", "14659", "27007", "24072", "34827", "42049", "56406", "20180", "14499", "46064", "29819", "35648", "4104", "31274", "31822", "12425", "4226", "5867", "49702", "41803", "53612", "55268", "33124", "17492", "16578", "7368", "2829", "6522", "54371", "1838", "17003", "33759", "47865", "33713", "22399", "46604", "50896", "36442", "41162", "11623", "52161", "30578", "39269", "44943", "33654", "55849", "46124", "32277", "22567", "25870", "58023", "49960", "7027", "23516", "50498", "20102", "33793", "2384", "18314", "40548", "13706", "51916", "7035", "43816", "31923", "1210", "11871", "16650", "1471", "29348", "21805", "3827", "36133", "2449", "43900", "32210", "14039", "11092", "48974", "56186", "1701", "49405", "48359", "17050", "47800", "45599", "17684", "27255", "22293", "3940", "44996", "22980", "47351", "38596", "18302", "44076", "27449", "27697", "57930", "16077", "581", "30868", "57311", "16408", "44625", "51748", "36223", "35839", "31769", "14533", "7425", "38960", "37633", "19816", "10854", "36726", "54609", "12346", "43526", "17078", "55339", "16140", "10684", "56782", "17777", "28468", "5902", "17352", "44093", "51431", "33936", "39821", "40789", "35819", "21571", "11361", "51539", "6570", "19312", "32650", "3541", "22084", "46137", "37666", "34909", "23658", "59301", "44643", "17613", "6295", "55715", "58835", "59839", "6148", "24615", "50201", "56803", "2214", "8920", "50397", "56895", "21067", "1506", "30586", "9939", "2543", "19749", "18999", "21532", "10638", "14205", "30864", "50362", "38279", "21866", "13924", "1766", "59806", "6163", "41915", "11565", "25189", "35930", "7882", "17743", "1502", "20631", "55444", "48190", "54798", "56584", "41198", "57777", "42467", "36345", "55607", "35971", "52426", "12598", "6838", "26872", "9419", "44693", "51702", "13765", "9498", "20575", "33944", "33510", "42405", "31416", "45925", "5100", "58063", "33985", "5228", "37726", "4976", "16242", "28839", "19600", "25829", "58439", "20741", "29945", "40180", "15202", "40818", "53895", "28438", "53212", "57653", "17160", "18161", "42908", "10722", "29263", "55818", "10166", "34163", "41958", "9660", "55764", "37197", "48370", "45074", "40135", "20963", "55307", "59031", "28850", "24890", "31956", "50146", "45154", "38226", "38993", "9456", "16002", "26085", "21647", "3464", "2778", "48916", "41850", "39806", "47110", "6687", "47731", "11415", "15760", "10569", "8610", "8027", "43553", "20870", "58341", "502", "15420", "40697", "20324", "44402", "52712", "15342", "13457", "45696", "13185", "16659", "42147", "32745", "7056", "11814", "43044", "25024", "56946", "26509", "27660", "43481", "40523", "28046", "55575", "55595", "59436", "43994", "59917", "57666", "21598", "17705", "24664", "19860", "32457", "1750", "59658", "49021", "52273", "22641", "44609", "49636", "32652", "43557", "8779", "25270", "55283", "33104", "11568", "25508", "13163", "24687", "36487", "1195", "41082", "12986", "56744", "47294", "4554", "2397", "5397", "31640", "39009", "28355", "42978", "30303", "393", "43704", "23075", "46582", "29822", "9696", "32431", "20094", "19404", "49934", "33479", "30647", "46119", "9585", "37156", "39223", "17310", "32369", "16151", "14775", "6135", "20558", "17135", "47552", "10374", "1083", "54050", "8861", "55671", "16061", "13980", "33496", "115", "54265", "40404", "52179", "925", "17387", "25726", "16868", "14750", "7370", "59172", "42123", "31809", "13418", "36130", "25470", "55664", "5791", "15820", "2560", "50066", "30569", "9471", "17891", "53904", "5067", "16595", "18687", "45351", "47185", "15232", "49226", "4614", "2404", "55610", "19445", "42540", "1575", "3383", "3601", "27108", "58325", "26763", "26695", "21081", "38639", "27204", "30031", "56115", "19870", "21682", "59173", "30763", "4520", "35649", "13670", "13685", "21777", "40493", "1082", "56344", "2985", "40309", "58284", "28681", "46238", "50543", "15747", "7419", "56029", "16632", "45671", "37722", "45715", "29812", "48639", "25230", "24692", "27341", "57751", "19358", "51635", "25786", "28415", "49294", "55549", "14981", "17056", "33703", "46715", "23163", "3788", "28603", "20977", "47050", "52360", "28333", "11327", "46455", "9372", "39862", "35626", "47068", "27507", "24107", "54884", "25738", "25801", "37583", "59352", "38026", "9562", "43947", "8651", "6219", "32123", "23668", "40534", "57274", "43929", "26671", "30828", "18429", "34670", "22717", "24542", "43183", "45680", "13271", "4543", "15121", "33073", "43194", "12746", "9421", "59518", "29092", "55370", "3183", "41956", "45108", "23002", "41318", "58556", "13390", "22327", "17952", "3450", "21075", "11766", "36039", "33395", "57635", "35049", "15054", "20114", "17954", "33344", "59602", "46143", "14276", "31540", "53729", "39001", "56196", "21358", "46635", "33961", "34888", "52007", "30336", "40122", "36922", "13308", "26358", "51343", "2616", "16939", "3126", "53829", "24022", "51704", "51692", "805", "11072", "54695", "8547", "11313", "29677", "25520", "31059", "48345", "3294", "15113", "30208", "20810", "48305", "47575", "3936", "10021", "30044", "10935", "20516", "9779", "54935", "10337", "54669", "52888", "55115", "37030", "24214", "21282", "54038", "32886", "2164", "17553", "20039", "29963", "14922", "2966", "466", "41339", "48341", "5592", "32356", "23288", "18593", "47598", "1860", "7055", "55380", "6893", "24818", "10713", "30316", "7262", "25998", "40539", "17673", "31077", "59736", "15300", "47322", "2749", "4258", "52417", "3566", "44882", "6974", "32698", "1329", "48361", "3006", "10388", "29857", "296", "25102", "29262", "23430", "52496", "4116", "53651", "5244", "41036", "33326", "3934", "33999", "23688", "45357", "31036", "38979", "11851", "56519", "25311", "19043", "48566", "40764", "55517", "4228", "14415", "34041", "16280", "11303", "11878", "45402", "36659", "24428", "30695", "14811", "26634", "55603", "44965", "13799", "42433", "41632", "14400", "4357", "19345", "23826", "32680", "27350", "56628", "15990", "20658", "35353", "55306", "7986", "11014", "7463", "11859", "56949", "16279", "39386", "17266", "35136", "3234", "43363", "56776", "56955", "27723", "11785", "19023", "18846", "17972", "1407", "51988", "2478", "13766", "51707", "4201", "7777", "25886", "8317", "26115", "16852", "48061", "30831", "53004", "46413", "11301", "19660", "45825", "40460", "17405", "49682", "51671", "7034", "33926", "43577", "54247", "18998", "46399", "19304", "9055", "25040", "4796", "45150", "21580", "3254", "47234", "44800", "27112", "6736", "52723", "50277", "23541", "37888", "23537", "54204", "2633", "49399", "17020", "19719", "59053", "8541", "44854", "54478", "55552", "25980", "57981", "36459", "25686", "33306", "37286", "11976", "37471", "30936", "10987", "14519", "32676", "56101", "23418", "30191", "45196", "4153", "39187", "46723", "46860", "25535", "41137", "50244", "26976", "42038", "4688", "49032", "31645", "57537", "39355", "14015", "30974", "59206", "39893", "55803", "21624", "57502", "6455", "19237", "41374", "21388", "15610", "257", "50663", "2962", "54453", "58705", "21104", "7827", "10356", "31451", "599", "1233", "25057", "6677", "37180", "55032", "36741", "4574", "37118", "1676", "54774", "15608", "48279", "37465", "51721", "33340", "29134", "46880", "19306", "15334", "36413", "32911", "2519", "56036", "20243", "19515", "1323", "23359", "5720", "59545", "53608", "17756", "49280", "17971", "48932", "28021", "13110", "56981", "27688", "16099", "43141", "4035", "37169", "20490", "2046", "53035", "33016", "35164", "24293", "54497", "39577", "26281", "54940", "9105", "35941", "2474", "53806", "55619", "22981", "22360", "32407", "34549", "8752", "1690", "25207", "43936", "16197", "28279", "5558", "28224", "51860", "33179", "9966", "17919", "6812", "29519", "48068", "32975", "43349", "35811", "31360", "31118", "51115", "28665", "30016", "2026", "29494", "9809", "39808", "35845", "18222", "50344", "18142", "31876", "29846", "20860", "51366", "56182", "19011", "27450", "16", "41516", "32217", "42209", "19775", "27321", "12514", "38445", "57760", "57810", "35438", "15588", "35719", "34958", "32191", "19969", "13942", "33059", "44753", "32329", "28849", "31181", "6764", "21182", "56145", "53810", "47569", "23858", "55584", "609", "54711", "50219", "46482", "20465", "18104", "6204", "1715", "13507", "21465", "26124", "35697", "18553", "40940", "14785", "36569", "57566", "55592", "12089", "44313", "19662", "8286", "56665", "13057", "50183", "34527", "13638", "1219", "22052", "4169", "45754", "31458", "58230", "37058", "11504", "59678", "9845", "46031", "27214", "55503", "55226", "7114", "13950", "36871", "26819", "3745", "53691", "53927", "4227", "57631", "55052", "47591", "421", "10301", "58567", "24", "58762", "12460", "13292", "45002", "20926", "38421", "28119", "41805", "36452", "9033", "3818", "39392", "55236", "15841", "1060", "11554", "55505", "45406", "22863", "9480", "45635", "19353", "20047", "1489", "43277", "43449", "30234", "12759", "56513", "36429", "58812", "40233", "20086", "55263", "54530", "41150", "49563", "57145", "42928", "17224", "16531", "5460", "14496", "54558", "53178", "34886", "12186", "597", "27764", "59326", "35419", "30045", "7170", "19283", "194", "5379", "24775", "37632", "13755", "1256", "17691", "23409", "49561", "3035", "29704", "56462", "21006", "42077", "51185", "13642", "50021", "36460", "28307", "54847", "40747", "34500", "45190", "41712", "32800", "37045", "24904", "14277", "20497", "39722", "27816", "49372", "41950", "12391", "8497", "33455", "54630", "40387", "14061", "45942", "27035", "15500", "26706", "23262", "750", "40578", "36174", "6318", "34127", "37504", "2201", "20232", "58231", "38022", "43144", "20499", "6678", "12354", "9553", "13835", "58784", "24273", "48156", "42412", "5975", "33503", "437", "1492", "26079", "36580", "26336", "58532", "51973", "35501", "49464", "5545", "826", "878", "39920", "30915", "42188", "17188", "14212", "26937", "23522", "17156", "28124", "32733", "57008", "15050", "19140", "27314", "2675", "44375", "41467", "42569", "12993", "34188", "3989", "28213", "13552", "54085", "40777", "47968", "24951", "49886", "29977", "7492", "50385", "37751", "38840", "25033", "42150", "12179", "20325", "19426", "59471", "31105", "24294", "34166", "59551", "2679", "46757", "17596", "34482", "36701", "43483", "42945", "36348", "55359", "37345", "45397", "19750", "19330", "51143", "44792", "7654", "53364", "4189", "30421", "37695", "7780", "41788", "38039", "41744", "25404", "884", "6374", "52095", "19447", "42586", "50574", "5490", "24074", "41175", "45542", "38336", "38796", "11644", "6879", "32137", "23723", "15902", "2717", "49629", "29653", "20173", "16473", "57775", "29446", "45026", "52901", "14748", "43201", "40850", "30206", "23384", "2343", "25133", "58653", "58920", "42102", "8566", "20005", "6772", "36224", "46432", "51489", "45292", "10099", "41487", "48935", "51774", "10035", "38966", "39002", "51999", "22206", "42026", "37603", "3848", "29661", "23707", "49849", "39882", "50495", "24313", "6684", "12558", "49233", "43290", "49836", "35374", "25339", "32426", "37675", "41774", "32885", "47205", "51335", "27457", "57885", "41186", "11165", "34933", "51380", "48585", "27345", "33139", "204", "52176", "57811", "15937", "52290", "33497", "44245", "22452", "51847", "30733", "57938", "31227", "43341", "8683", "25578", "12162", "25400", "15133", "48344", "40143", "40101", "41018", "46599", "39744", "58746", "19873", "40717", "44250", "18985", "32235", "13978", "2296", "54258", "27474", "47824", "35500", "54427", "32305", "22829", "7031", "17471", "8079", "55872", "44642", "10094", "40708", "43022", "26924", "56103", "26267", "72", "51941", "31613", "900", "12544", "25967", "3677", "33786", "10465", "39241", "29407", "31338", "58829", "37070", "22905", "26933", "42531", "48869", "25410", "35046", "42983", "37707", "9310", "29336", "1961", "4872", "40688", "17547", "17881", "54796", "6842", "35456", "42614", "18067", "44302", "30870", "58151", "12533", "49317", "28637", "17984", "7674", "7285", "48556", "29150", "230", "57115", "52410", "40837", "3028", "50585", "7074", "40228", "28033", "45928", "24180", "17737", "9651", "11864", "36496", "54170", "22055", "11729", "33907", "56257", "7008", "42488", "27360", "12905", "56072", "30701", "29286", "38249", "45524", "7808", "5654", "30236", "20957", "34351", "7520", "27509", "18467", "38922", "44074", "48251", "48941", "38049", "22416", "39995", "30906", "12058", "43873", "11815", "24776", "11130", "18039", "50462", "54410", "14398", "38789", "56025", "40027", "45582", "29131", "57905", "39876", "38167", "42401", "37947", "22356", "54715", "55353", "40072", "27645", "39130", "57474", "32089", "58523", "46166", "13151", "15564", "24321", "59178", "27306", "24001", "50085", "49170", "24105", "5479", "10370", "36550", "35718", "27612", "47045", "27441", "45525", "26399", "3380", "32040", "32006", "13351", "30853", "8507", "282", "28176", "2433", "52605", "23879", "2848", "48582", "45708", "58647", "42223", "20579", "19791", "51699", "17745", "38847", "12399", "13970", "29022", "55401", "22147", "7414", "11041", "48284", "15877", "22541", "14854", "52368", "20560", "35444", "26771", "58506", "56148", "20936", "29316", "11996", "13399", "845", "50454", "27664", "36523", "12713", "22073", "42133", "15580", "47754", "58959", "34015", "49929", "16195", "32097", "22777", "10451", "40398", "20529", "24563", "21237", "56034", "15006", "34949", "19332", "7145", "4087", "38072", "28777", "47075", "58206", "49622", "45001", "24811", "28867", "21894", "10737", "2544", "20532", "58513", "52691", "28768", "33607", "6372", "6912", "32069", "20190", "9582", "41276", "17031", "34825", "23789", "43091", "57552", "10043", "21807", "5951", "26869", "48494", "54137", "54226", "3185", "44626", "41588", "147", "16663", "6755", "17184", "56784", "28094", "42543", "53874", "49878", "15622", "57129", "45670", "56817", "13092", "44028", "41238", "58382", "47198", "38303", "54076", "33502", "17482", "9945", "26456", "31709", "35747", "48719", "13119", "39764", "50808", "25489", "56680", "39884", "1036", "21852", "4061", "45369", "42893", "42464", "51837", "29250", "47867", "47324", "17438", "41888", "45409", "12010", "28065", "48944", "35965", "33101", "23997", "7538", "50800", "44691", "35976", "59755", "42975", "13348", "14440", "46354", "1726", "42064", "38899", "16857", "20699", "36798", "48893", "34525", "30289", "24777", "54336", "18434", "6430", "11282", "5439", "8117", "27127", "43382", "259", "15090", "50455", "50726", "38722", "10292", "40048", "29787", "8735", "37992", "52705", "40885", "19328", "21313", "34853", "29314", "11696", "46070", "37190", "43281", "55921", "22449", "57447", "29850", "51519", "24988", "15378", "19867", "56804", "38002", "40174", "26398", "10118", "21100", "27996", "53681", "35898", "5404", "10568", "22431", "29252", "24426", "43523", "35100", "58159", "14857", "58056", "8471", "5128", "51769", "21942", "57914", "23769", "2336", "57579", "30226", "13711", "35207", "16638", "9719", "43134", "41552", "38774", "15655", "33783", "18021", "46176", "32519", "28363", "12597", "3403", "17383", "17212", "36218", "53199", "7668", "59343", "7886", "35586", "28957", "2646", "10884", "27102", "27856", "5652", "39103", "12271", "52069", "1053", "24668", "54157", "7844", "26779", "472", "47089", "3669", "45470", "25042", "22866", "4975", "17701", "16460", "56459", "30556", "31556", "42738", "55162", "10970", "40495", "38838", "20169", "29917", "25358", "54735", "21841", "24038", "25280", "41272", "13562", "31534", "15567", "37082", "38205", "16084", "43279", "15814", "23938", "42244", "6054", "20461", "50093", "19162", "12620", "4917", "6301", "2709", "30597", "52300", "11510", "34", "39963", "48815", "45377", "46191", "42578", "18923", "28455", "37113", "57956", "57105", "21107", "59942", "18662", "54823", "8195", "43214", "9726", "10106", "8438", "40117", "25557", "42251", "30731", "38571", "13676", "23422", "51839", "59400", "3696", "661", "50718", "49719", "56089", "43225", "33438", "33957", "31397", "6271", "27116", "3738", "12097", "39347", "38808", "4438", "23616", "29634", "58161", "41441", "22790", "49267", "41069", "33052", "25901", "14403", "17763", "37042", "45453", "37830", "16577", "24738", "11069", "53191", "56583", "50294", "35559", "2986", "7283", "48479", "14865", "10453", "31802", "43731", "12772", "16551", "55086", "26496", "55139", "827", "56964", "32200", "22466", "18449", "16107", "5393", "17258", "49116", "54286", "19504", "48622", "5245", "53711", "42270", "37284", "27024", "27508", "6836", "8194", "15774", "58432", "31425", "56396", "23355", "55414", "35302", "39541", "1189", "268", "21049", "44713", "24436", "20111", "25810", "42449", "3506", "14799", "1716", "50779", "48435", "16277", "34871", "29706", "16729", "28354", "39660", "14842", "40874", "46192", "51612", "32924", "39156", "32514", "8331", "12429", "12516", "37673", "35028", "2065", "57406", "30606", "38797", "58505", "51449", "17665", "28643", "41765", "36683", "33950", "9575", "7764", "37553", "17197", "13837", "48513", "42606", "44223", "36080", "45879", "38079", "39561", "54861", "22877", "31960", "9307", "26631", "18117", "28058", "13932", "36949", "49160", "4187", "12351", "16503", "22911", "29418", "11286", "4536", "48799", "1313", "29491", "46326", "56768", "37225", "35429", "43856", "23215", "32451", "13437", "15068", "28463", "49529", "20844", "12", "54802", "28478", "2277", "16424", "58469", "1178", "18797", "47243", "23147", "48514", "20881", "41842", "38489", "29076", "33146", "49714", "16275", "1194", "39200", "59004", "28932", "11228", "7160", "46725", "8383", "6600", "11522", "24134", "35041", "48081", "33381", "48362", "59133", "11436", "918", "2777", "29400", "21755", "16683", "28872", "7360", "10906", "8508", "3977", "43282", "24644", "22594", "46395", "59339", "50626", "47775", "10339", "17254", "34349", "31073", "21936", "8524", "15440", "39850", "34767", "49626", "44119", "34457", "9679", "29712", "24931", "52892", "47732", "56377", "22123", "4080", "13620", "11758", "35412", "11956", "19158", "30966", "19136", "46391", "21316", "43387", "5107", "56734", "20328", "12774", "12135", "71", "40844", "54463", "4823", "53554", "10579", "43612", "21372", "34463", "20356", "19108", "8763", "30132", "25650", "56228", "17395", "49662", "48291", "49092", "41395", "51184", "42929", "41348", "11171", "3051", "39695", "41000", "53535", "4055", "34032", "57194", "17270", "20321", "55733", "35410", "31738", "434", "30752", "11998", "36577", "39715", "52354", "29042", "21113", "6559", "39852", "20380", "17963", "47819", "50547", "12495", "14948", "7073", "54272", "33718", "7245", "6594", "6939", "51424", "28668", "34610", "32621", "11089", "25351", "41459", "38247", "43071", "29548", "47664", "57723", "58225", "45587", "54728", "2723", "19911", "55868", "1718", "35094", "40755", "26189", "55895", "49568", "36815", "51075", "31606", "364", "57113", "9463", "2461", "3262", "46819", "55028", "15949", "29910", "57327", "21491", "54693", "16787", "20258", "29858", "56759", "42756", "14525", "49669", "31091", "19557", "34981", "49361", "26364", "48317", "52678", "9233", "55544", "1183", "5456", "23524", "39904", "40806", "28072", "25527", "54093", "56846", "48694", "48853", "58478", "54344", "5552", "23464", "33181", "44598", "1573", "40823", "53537", "4789", "22998", "45454", "57148", "41241", "37359", "6840", "12562", "24794", "46383", "40150", "19336", "27211", "2480", "42097", "48364", "32059", "782", "35058", "40635", "19069", "15670", "30074", "46359", "38281", "27467", "16283", "53866", "13707", "25832", "59641", "17878", "11688", "2376", "6792", "41515", "16164", "202", "18623", "8186", "12649", "47397", "32348", "29513", "59992", "40207", "4235", "3418", "25688", "15522", "1129", "59535", "59154", "6091", "33200", "4053", "35363", "27778", "31532", "49400", "33965", "24190", "28546", "11200", "12617", "34305", "55737", "42933", "48710", "9760", "57352", "40522", "3084", "57808", "4913", "52597", "17145", "29456", "53207", "30827", "35531", "30173", "12795", "29740", "35354", "46411", "337", "40322", "47821", "31269", "836", "34967", "42725", "34906", "44966", "4324", "594", "53808", "13031", "40924", "12435", "30119", "46190", "163", "10047", "26419", "571", "32091", "57290", "32707", "43759", "21340", "16913", "18408", "30193", "13000", "30376", "47283", "43890", "26465", "44581", "35327", "709", "4148", "23690", "20235", "30494", "48150", "5481", "30188", "34173", "55679", "18097", "46987", "21273", "47101", "55488", "28700", "34678", "5202", "58390", "31955", "49506", "3070", "9145", "15208", "39060", "3268", "58848", "19935", "21390", "57336", "54616", "46403", "29821", "36007", "15683", "50510", "57993", "31862", "46652", "14729", "8899", "54", "13085", "544", "36168", "40545", "11790", "19612", "22212", "23849", "12589", "1731", "16682", "15645", "6828", "45432", "33787", "2577", "7101", "36457", "54175", "19964", "37944", "1853", "26091", "27497", "21693", "47792", "48078", "43353", "35860", "785", "41490", "11759", "15861", "4353", "58837", "18397", "17540", "18243", "53264", "2324", "17873", "15390", "38359", "14258", "57732", "27634", "26254", "43014", "45316", "30391", "47449", "2473", "34033", "40459", "54838", "24719", "13076", "33167", "44080", "43952", "3141", "37005", "19617", "47621", "20970", "41979", "46580", "5841", "18724", "10119", "12503", "30131", "6124", "34655", "46472", "58666", "14100", "22376", "26513", "27495", "58311", "16731", "158", "42994", "11806", "52619", "35669", "2253", "58496", "10267", "45451", "14271", "53115", "35889", "22992", "57154", "10182", "11798", "50831", "49925", "7475", "968", "49796", "55657", "2208", "59341", "28951", "109", "57240", "15946", "57554", "14136", "4491", "6880", "24620", "6620", "54486", "47181", "8663", "18379", "18569", "14366", "31363", "17465", "12879", "59177", "37310", "36120", "30619", "1833", "53304", "33448", "38793", "21150", "19805", "7461", "10319", "22404", "5485", "9371", "2327", "30362", "4805", "2500", "33696", "43027", "35689", "39229", "8793", "48055", "48333", "23316", "55208", "19550", "57366", "42748", "28405", "59748", "18582", "25562", "4932", "56911", "45368", "46260", "34037", "53699", "24963", "46702", "9567", "9158", "58483", "37523", "3579", "7949", "4624", "585", "29090", "51061", "17388", "18471", "51029", "2059", "21738", "28828", "57665", "56585", "54755", "23203", "55876", "13115", "54307", "57802", "55113", "4306", "33655", "24259", "48397", "59562", "34454", "36248", "32515", "47741", "28593", "55120", "37938", "38693", "1892", "40908", "35616", "32580", "41783", "5429", "17084", "21934", "24480", "53243", "14803", "3057", "9188", "25386", "32021", "58290", "22884", "17815", "17076", "20742", "33668", "40860", "24756", "24917", "35320", "31730", "44103", "30809", "987", "14836", "20591", "30624", "12563", "51069", "50834", "18910", "45182", "57047", "53689", "41732", "18828", "14738", "1080", "9427", "50447", "35717", "34191", "42324", "41643", "26380", "57568", "9013", "7774", "35853", "17015", "40576", "8645", "13664", "57288", "1375", "17346", "44890", "4175", "32811", "1907", "28313", "41778", "58409", "29544", "5721", "12246", "26062", "41505", "28523", "18040", "4401", "26603", "7943", "30152", "40992", "53882", "23095", "19379", "9227", "44608", "57426", "27058", "27232", "46340", "5403", "13189", "24698", "8344", "39490", "45945", "37808", "59967", "57049", "14765", "7870", "28101", "2861", "54452", "14522", "43819", "24130", "49543", "47740", "43518", "46082", "50342", "37794", "22224", "52761", "53107", "3179", "42762", "51484", "47303", "50655", "40412", "36567", "44470", "41281", "13206", "29246", "42161", "9342", "18903", "29784", "12817", "38204", "7484", "23746", "51520", "18856", "39279", "35128", "46459", "45467", "21135", "37147", "33335", "28914", "38568", "53394", "29464", "45766", "28136", "41587", "35386", "49469", "18713", "5395", "4569", "8757", "54563", "36162", "11024", "2851", "53180", "51253", "4638", "13134", "43894", "36648", "41688", "17171", "54408", "19434", "14885", "12855", "37479", "29535", "9000", "59284", "8361", "55668", "9834", "41228", "20058", "38780", "31944", "33458", "34403", "11211", "24536", "21398", "27514", "16068", "9683", "26410", "11206", "41052", "50028", "35865", "7746", "262", "35576", "21482", "45598", "50448", "8595", "26705", "20186", "57367", "43805", "51363", "53605", "34851", "41984", "52036", "11827", "56541", "33449", "45629", "6153", "12365", "430", "55924", "7856", "34448", "9613", "10719", "47245", "14291", "15612", "48447", "2077", "2980", "43362", "58502", "38634", "57107", "50247", "48310", "2590", "56827", "3778", "51353", "18507", "47799", "2531", "17865", "19730", "31898", "34153", "59656", "10659", "51326", "52151", "9434", "53233", "59194", "57060", "18107", "56168", "40527", "54853", "16541", "19706", "58579", "57355", "604", "5532", "33203", "49185", "18932", "11948", "12719", "26546", "8382", "34101", "3793", "44075", "14555", "10405", "57511", "42925", "2123", "43359", "34167", "59498", "15245", "41166", "50347", "46295", "1978", "38177", "17740", "27348", "39516", "2163", "11112", "6624", "50356", "25890", "11958", "27640", "37776", "3922", "4859", "58926", "57460", "820", "7135", "40040", "14773", "255", "42722", "28457", "37764", "449", "39907", "32409", "24464", "23398", "38875", "21946", "28926", "17932", "15840", "58931", "13547", "54398", "23948", "54932", "623", "52005", "54515", "55193", "7136", "51787", "53077", "59429", "31018", "6323", "4511", "58538", "7272", "27059", "42702", "1632", "31368", "45060", "41022", "28812", "41414", "8213", "630", "35239", "37385", "40906", "28727", "29897", "25573", "26889", "16869", "21861", "30072", "58652", "59307", "46629", "30669", "23854", "37323", "12541", "24479", "1394", "16567", "43674", "48124", "36653", "32764", "10271", "29710", "48504", "22639", "31015", "14526", "31104", "13788", "11294", "59970", "21516", "9270", "153", "41649", "36275", "5295", "2066", "7273", "2497", "28736", "2090", "29835", "39770", "22186", "13784", "50311", "6569", "27305", "32619", "39540", "41777", "59888", "50825", "45497", "14164", "18757", "38483", "49161", "28111", "15039", "21748", "30858", "37392", "15606", "54260", "5111", "58068", "31871", "22634", "21702", "9499", "36510", "26716", "55762", "9667", "2873", "634", "55305", "21210", "53775", "56107", "1662", "29638", "23496", "56883", "43695", "59731", "43414", "23465", "45819", "12957", "16714", "31765", "23565", "34369", "32898", "47100", "44432", "15998", "53230", "13349", "11677", "4837", "18829", "23966", "37379", "43435", "43197", "5237", "6126", "3749", "58869", "12008", "52900", "17537", "33356", "56177", "38804", "31830", "14014", "3845", "9221", "16476", "52382", "34779", "38337", "20587", "19886", "21279", "10010", "29007", "55319", "6391", "4302", "48436", "59929", "28821", "26321", "14984", "48357", "4542", "13986", "34507", "6961", "41684", "52074", "40927", "49611", "36240", "20314", "52676", "51587", "20030", "13967", "38832", "51261", "34021", "26068", "8258", "41939", "27784", "11902", "16278", "49222", "3058", "15085", "45644", "32258", "11232", "17496", "33295", "7024", "26542", "8923", "23996", "20033", "20653", "24261", "29055", "30850", "26607", "6683", "23832", "331", "27406", "38195", "58935", "3890", "35807", "9876", "41752", "47588", "31861", "58449", "25818", "22956", "15824", "46199", "17247", "56523", "15514", "15752", "6212", "45092", "58189", "14571", "57692", "24519", "5770", "45632", "2809", "34849", "57975", "50199", "28519", "20045", "563", "29682", "17479", "14649", "24770", "22828", "36845", "53183", "20382", "49355", "59332", "4006", "50651", "20595", "57678", "50535", "26896", "17058", "59846", "43445", "48769", "27889", "57813", "55485", "20730", "32051", "6655", "16349", "57968", "56552", "6867", "10905", "51314", "8388", "51989", "1652", "14230", "27971", "43864", "41260", "30386", "6002", "17839", "17581", "8472", "36521", "11031", "44586", "43787", "25071", "28132", "23186", "44547", "58358", "43041", "56858", "20217", "21420", "52108", "27849", "25159", "30178", "5987", "42280", "52713", "870", "45758", "19060", "18682", "33881", "40756", "46202", "32721", "380", "40050", "3139", "47753", "16599", "7955", "154", "51509", "59852", "39772", "5083", "18279", "7697", "41077", "35585", "54914", "48626", "8030", "58783", "3289", "21671", "18185", "49921", "14107", "16925", "59653", "3800", "15428", "36297", "23277", "3764", "5551", "51501", "56014", "4308", "30738", "20032", "38342", "46569", "29746", "47225", "29999", "18863", "58402", "56982", "28788", "40461", "1307", "18442", "11149", "6460", "39439", "42684", "2052", "49553", "13207", "55413", "28840", "22072", "55695", "59448", "11342", "46560", "33066", "846", "18451", "41943", "5292", "52424", "23436", "4206", "6853", "20501", "4880", "25964", "50479", "14715", "12419", "21574", "56716", "45298", "44194", "29913", "48302", "329", "30852", "55224", "35191", "47147", "13592", "50185", "45326", "47164", "22660", "2684", "42453", "33404", "53468", "24597", "5813", "58992", "34283", "49018", "6591", "5723", "42298", "19393", "44838", "10437", "46956", "39112", "45047", "45259", "18547", "28953", "10364", "49979", "39532", "906", "6496", "44601", "22928", "1151", "40305", "17825", "2716", "56674", "42604", "16905", "47505", "731", "17176", "4024", "18778", "47942", "45972", "19711", "53983", "38496", "59572", "48320", "5683", "32696", "17463", "7839", "50335", "12433", "17202", "53578", "42355", "38934", "22490", "59986", "18536", "44343", "54806", "47512", "50671", "48994", "44755", "15731", "25402", "20", "39385", "46706", "1480", "1463", "21712", "14764", "36200", "48575", "55620", "29230", "29572", "11907", "16692", "17790", "37922", "42277", "38592", "35206", "10933", "58258", "493", "32882", "52059", "51172", "44754", "2207", "52991", "31103", "21246", "54632", "12343", "8210", "47690", "16512", "1078", "40366", "59201", "59569", "7501", "14992", "15639", "43010", "25911", "56316", "5254", "4170", "51420", "28037", "26887", "39677", "53146", "10898", "38011", "1727", "29895", "29520", "4409", "58593", "33917", "36370", "1011", "13431", "21305", "31990", "48785", "44029", "9784", "23874", "55699", "18095", "53818", "47296", "41205", "45545", "40773", "47716", "2351", "50549", "8025", "26687", "43725", "12777", "51895", "41917", "7237", "44289", "43408", "44829", "53170", "59267", "53235", "17008", "32963", "21512", "56325", "11046", "46588", "7309", "53757", "10404", "58964", "35170", "47128", "22234", "152", "44044", "383", "46224", "15276", "16842", "22204", "5913", "35985", "9370", "40965", "41017", "46385", "20920", "9344", "16342", "29536", "58983", "44561", "14901", "52493", "6634", "45013", "56296", "48259", "51902", "11840", "53291", "18850", "41821", "7716", "25330", "3355", "38927", "58621", "4665", "51306", "12978", "21430", "12458", "25507", "48492", "23982", "15178", "8122", "51310", "2920", "4336", "15302", "26152", "9820", "465", "31193", "28845", "21284", "20485", "53276", "5457", "11936", "9698", "41332", "17682", "19217", "11145", "41358", "9378", "28474", "19111", "37580", "8877", "36950", "42713", "8161", "2612", "25196", "52266", "48539", "2445", "36747", "28459", "9731", "50254", "37189", "30147", "42362", "2365", "43680", "39259", "5613", "17580", "1019", "19354", "14146", "11639", "18495", "27264", "20038", "58383", "1552", "34731", "26307", "49319", "21906", "6959", "43975", "51676", "31393", "17052", "40115", "24559", "4734", "869", "24821", "25328", "33567", "31436", "8417", "10580", "4475", "29058", "3446", "1698", "54673", "37812", "54987", "42989", "23601", "49728", "59930", "50680", "23118", "25066", "19602", "48463", "41630", "29541", "57172", "16320", "18049", "43502", "29006", "45251", "26391", "16605", "50037", "31945", "29392", "33990", "27479", "5932", "5873", "47998", "37072", "16419", "12755", "30122", "52647", "4504", "20774", "38242", "27606", "1799", "23911", "21189", "41881", "9865", "34328", "13813", "21837", "58194", "29256", "52743", "16483", "15981", "13937", "18575", "37383", "35067", "12283", "23697", "34447", "46792", "29509", "56431", "56524", "58580", "28036", "2784", "928", "26536", "35617", "5231", "20748", "56047", "53398", "29451", "9409", "9050", "8876", "21072", "680", "33134", "27958", "43203", "47697", "59434", "51472", "19863", "23892", "45287", "37670", "26878", "46619", "32173", "56553", "55342", "54886", "456", "26155", "30618", "9928", "40087", "3872", "41123", "25834", "14889", "8404", "26177", "53820", "56598", "33735", "43028", "16302", "31411", "50143", "51286", "51346", "41853", "53033", "11990", "18267", "15079", "8706", "57090", "55865", "25272", "31719", "47647", "38836", "34443", "30372", "27847", "19526", "46935", "39289", "32471", "35557", "51778", "57761", "51773", "21222", "24543", "36742", "48653", "25856", "6558", "35736", "8276", "44516", "42633", "4256", "51709", "7408", "45673", "4370", "45161", "40636", "19739", "57304", "58278", "20813", "25644", "31562", "9446", "20306", "32863", "39705", "43397", "40378", "51106", "31816", "14108", "13849", "25706", "19368", "14314", "41120", "54119", "135", "35507", "38656", "29401", "22343", "15886", "37088", "9671", "36189", "47063", "14568", "2040", "49450", "48855", "49638", "5278", "18382", "25290", "52764", "45780", "37000", "12780", "33246", "38628", "39079", "29950", "23138", "33513", "23494", "56278", "33274", "47613", "10727", "22969", "59508", "58839", "8345", "55965", "19259", "29172", "49459", "2592", "54133", "15858", "50113", "14304", "3593", "57482", "51348", "730", "55771", "6023", "9402", "19958", "6274", "47326", "5852", "36365", "50559", "57815", "16051", "57582", "56740", "56549", "41898", "7983", "37077", "9171", "31101", "1991", "33785", "28315", "33007", "26248", "50031", "28844", "18926", "36195", "3830", "52974", "45510", "9986", "36229", "29381", "23514", "40123", "5198", "47913", "56239", "42936", "36302", "24797", "42919", "27824", "49974", "15753", "14165", "6708", "36374", "50371", "14489", "15690", "2516", "59750", "38306", "28062", "22436", "28814", "28240", "14661", "45010", "37859", "38626", "12796", "41964", "21143", "16021", "14964", "29040", "40630", "24705", "50687", "28447", "45334", "28720", "4555", "18060", "48969", "42575", "17773", "53590", "47923", "57061", "39014", "13899", "53982", "3513", "36156", "18710", "11591", "32624", "15882", "4633", "34310", "16889", "20543", "22964", "34496", "3999", "33932", "25046", "14329", "53615", "31831", "8953", "55999", "50934", "38843", "28248", "54579", "32507", "21300", "19540", "40904", "50693", "34365", "52390", "33899", "16718", "16548", "44324", "57280", "44450", "27812", "13357", "45195", "40905", "31922", "49145", "44712", "28919", "52693", "2610", "30461", "16957", "21973", "5900", "59573", "52101", "48472", "42631", "33995", "39826", "53954", "31267", "43738", "32128", "29463", "36076", "38013", "2179", "59054", "18468", "56972", "22130", "15755", "48921", "54031", "19115", "6283", "53415", "39492", "57003", "47830", "54214", "58240", "41235", "37540", "23713", "37444", "8112", "38937", "57988", "32322", "45725", "45612", "6670", "27710", "13436", "54759", "16001", "24669", "18150", "51751", "5762", "14970", "31515", "8694", "12664", "35957", "38850", "7257", "12577", "33992", "57880", "55636", "47403", "15186", "28223", "2007", "54160", "41980", "49214", "33260", "11692", "56613", "4396", "43944", "41995", "51289", "30747", "27136", "11594", "43907", "18160", "17475", "40942", "1368", "57870", "12797", "28993", "44762", "32268", "28423", "22209", "8461", "57847", "4797", "42838", "48050", "57158", "13157", "29060", "5519", "1358", "35676", "33308", "15263", "10804", "47367", "57402", "11880", "52040", "25843", "59027", "5697", "30925", "33688", "27212", "45618", "52038", "35204", "11531", "19798", "12467", "43308", "24365", "48732", "55583", "33551", "58321", "30058", "27759", "52305", "32517", "2799", "23599", "17627", "17226", "20621", "5483", "23298", "7155", "5146", "3676", "30424", "36895", "6223", "23127", "41935", "39321", "8408", "24562", "57224", "55126", "17002", "39314", "44823", "53769", "43939", "27857", "56697", "52215", "9805", "13808", "19517", "33977", "17065", "17165", "12728", "24356", "47803", "36492", "35159", "56440", "25603", "15022", "24633", "1560", "40760", "57797", "33473", "58361", "38488", "8958", "42976", "53787", "29532", "26630", "30279", "58461", "27095", "31431", "35201", "3184", "44147", "19637", "34688", "43063", "1988", "58178", "25414", "50587", "41647", "8741", "26253", "37516", "12115", "4718", "7560", "47949", "36378", "12358", "52267", "39659", "20641", "8668", "34543", "30620", "54506", "55155", "59182", "52730", "10185", "41615", "44758", "59387", "33092", "53294", "6595", "51878", "50979", "20671", "48555", "24408", "20686", "37097", "28451", "23872", "12929", "1426", "57636", "30930", "58126", "54331", "23612", "56306", "44276", "28144", "56700", "14547", "46451", "26430", "21264", "59331", "55534", "1940", "14608", "4414", "30648", "31718", "21541", "21344", "50190", "56117", "8350", "19943", "5287", "13272", "46284", "47617", "1406", "48990", "1828", "44940", "47698", "44188", "45387", "36844", "16973", "37782", "26529", "31729", "27315", "51543", "50283", "35522", "59976", "42340", "30360", "22844", "33221", "56297", "549", "25694", "50327", "26512", "11896", "29912", "53147", "1105", "5044", "43505", "28555", "27310", "17575", "54372", "37780", "35723", "23923", "2984", "52615", "24902", "1793", "45569", "9937", "35027", "41078", "24460", "10635", "20973", "32662", "8457", "25919", "46912", "7120", "21980", "11084", "22950", "58912", "40685", "5859", "1558", "22850", "11042", "33552", "39935", "38920", "55153", "30819", "37807", "20670", "12802", "11764", "50069", "30947", "52837", "40690", "31433", "40864", "59401", "49357", "3439", "8072", "41020", "5178", "7940", "53549", "36792", "56401", "43127", "7519", "5620", "27993", "15012", "19204", "36678", "41192", "57942", "55399", "42829", "19089", "55218", "22478", "38416", "18601", "9603", "15324", "51980", "17622", "40646", "55989", "6473", "21648", "39249", "32269", "38442", "44523", "20901", "34363", "45217", "18616", "39692", "25389", "8557", "58004", "33640", "40304", "28659", "1029", "8403", "9424", "21419", "32609", "59877", "11532", "29880", "45814", "6561", "22058", "40621", "36924", "38326", "35241", "16410", "15355", "57489", "59181", "11901", "45412", "25849", "17112", "20008", "39349", "2443", "50274", "44719", "23149", "27280", "40111", "46958", "32830", "50673", "37880", "21405", "7344", "53652", "21992", "41133", "27565", "37935", "52595", "54739", "30983", "41080", "8458", "45242", "3642", "50394", "27815", "58981", "17018", "33337", "46027", "39615", "56956", "2628", "45018", "52154", "18034", "13833", "44747", "8564", "55309", "48219", "32735", "21788", "1862", "59038", "55194", "38261", "50594", "56262", "55212", "49183", "12579", "49535", "47180", "16630", "8201", "20158", "36406", "50725", "32653", "57487", "18389", "52931", "3545", "53248", "25625", "2350", "36738", "1626", "23620", "57770", "18198", "30516", "51944", "35753", "3349", "24333", "22752", "28506", "9214", "7047", "25164", "32880", "55712", "52610", "59815", "45691", "13558", "15569", "12291", "55364", "26483", "46780", "55850", "558", "10467", "26895", "27758", "1429", "28407", "17288", "8558", "48306", "50712", "44695", "56032", "21875", "1349", "18073", "44739", "52323", "40569", "17766", "14813", "5926", "29169", "9912", "13564", "28905", "22832", "42939", "49954", "29862", "18938", "36417", "23051", "29493", "2719", "34754", "54916", "13824", "29549", "53835", "21351", "12071", "27642", "39630", "20353", "45490", "53953", "14252", "9264", "335", "15478", "21938", "30029", "56236", "42765", "16465", "3046", "50697", "55774", "15513", "18920", "45172", "14871", "20745", "19438", "58667", "42398", "50393", "9175", "23433", "51354", "42812", "11772", "21180", "50493", "45559", "35117", "57774", "43425", "8372", "25143", "58304", "32995", "24660", "8817", "40801", "39417", "24469", "52861", "23459", "49199", "57773", "28782", "43666", "29084", "44207", "40611", "34440", "40507", "44788", "48976", "14086", "35931", "59473", "11503", "58791", "45058", "4162", "57420", "46936", "12201", "3116", "46645", "45095", "4708", "58111", "20964", "16272", "27703", "41068", "24449", "56923", "17974", "8605", "54619", "46256", "52334", "59521", "50981", "56334", "40914", "42615", "56136", "524", "20127", "50609", "23681", "51429", "45664", "41051", "25969", "42799", "39305", "41636", "53872", "4593", "13023", "50053", "38518", "9835", "51729", "38557", "39507", "23484", "37696", "622", "23385", "7235", "57536", "36705", "46180", "38711", "48120", "38607", "13328", "39875", "409", "12765", "32777", "5778", "2255", "16676", "7936", "40752", "31755", "45871", "32208", "57784", "23596", "22388", "39069", "39413", "50158", "10127", "17762", "23684", "44911", "38515", "5505", "10881", "36598", "21824", "48895", "15636", "38925", "53475", "9257", "32577", "9771", "22968", "59651", "29624", "1943", "6398", "18296", "13323", "56071", "16779", "57522", "24908", "53530", "25952", "24399", "16784", "13692", "36592", "46055", "49747", "31032", "54077", "19408", "2921", "24532", "40395", "14147", "36754", "28312", "38448", "50476", "31947", "38767", "47172", "23020", "21476", "54234", "28743", "11094", "22426", "46950", "26627", "9384", "18286", "55891", "35124", "14436", "48531", "50977", "232", "19957", "47151", "33206", "22387", "15968", "52702", "37876", "57253", "26856", "14030", "13686", "2513", "59557", "48783", "43420", "32026", "14637", "44678", "53105", "22386", "16004", "4392", "31667", "1483", "42720", "23951", "22757", "16722", "56497", "11213", "59303", "14346", "19134", "27117", "14769", "22121", "25685", "48856", "20418", "40187", "6246", "14211", "46034", "16644", "55969", "32692", "2459", "5605", "21077", "59800", "30333", "11233", "20972", "14523", "45656", "26109", "58452", "36804", "28865", "8592", "55648", "10357", "31785", "15237", "34204", "10289", "38169", "47312", "20293", "30994", "46267", "6004", "56092", "34974", "22229", "42673", "58800", "22053", "32443", "58081", "12295", "7749", "39453", "36259", "59610", "56707", "9023", "59635", "48844", "43059", "23104", "25100", "42831", "59583", "49559", "50665", "54481", "31335", "26781", "27846", "4122", "34704", "24909", "54883", "8480", "51216", "17572", "27513", "49685", "10234", "6045", "52552", "27416", "38141", "29240", "10219", "53722", "54505", "22256", "14444", "8229", "53145", "34542", "53678", "40724", "59618", "53999", "27942", "47638", "32466", "24522", "33495", "21976", "59351", "49094", "21073", "50686", "59082", "50074", "29166", "22348", "24123", "58870", "15138", "53761", "33377", "315", "14356", "5540", "43133", "28293", "42010", "18415", "44337", "339", "33031", "19789", "6921", "54997", "41237", "15264", "46300", "6757", "47972", "31090", "12622", "33143", "28266", "5564", "35961", "47107", "57858", "8822", "2818", "33150", "51210", "56766", "1871", "18047", "40126", "25267", "26", "24501", "32929", "798", "6022", "16792", "21890", "38685", "29615", "30026", "20572", "46622", "586", "52185", "22975", "20212", "7011", "7946", "45872", "19933", "28442", "14377", "14380", "53071", "53669", "49713", "34446", "18100", "4641", "53625", "41142", "16885", "523", "23518", "37499", "56022", "35735", "17741", "52113", "33301", "59245", "18287", "43620", "38089", "51654", "49851", "16861", "13901", "50744", "9882", "31488", "24760", "8205", "41933", "58668", "33354", "24407", "17108", "41521", "29692", "37975", "26990", "45021", "26798", "50295", "42061", "37512", "24866", "3991", "40274", "38288", "52054", "31438", "35465", "2836", "29080", "59195", "9762", "55015", "26011", "39691", "17411", "11833", "54688", "44455", "14713", "47086", "18174", "14052", "14082", "19762", "20436", "8660", "37702", "59985", "53766", "44730", "11480", "811", "2073", "51685", "37979", "54879", "31323", "46382", "5763", "46053", "31000", "17976", "48453", "20390", "38125", "18996", "3709", "11858", "13647", "39990", "11811", "17938", "48295", "46980", "29971", "31367", "18070", "32678", "32731", "26520", "56274", "32756", "41035", "27701", "10541", "9787", "29529", "27574", "11509", "59642", "2647", "36635", "57372", "16190", "27009", "50040", "36826", "47466", "27835", "1588", "40494", "10697", "23822", "14962", "57096", "45961", "12678", "14772", "4034", "38709", "42589", "25779", "21713", "18086", "39472", "20513", "5399", "46967", "12580", "35385", "30629", "11539", "1949", "55044", "29241", "20676", "33242", "5030", "43326", "37116", "8987", "45834", "47290", "39525", "2349", "23341", "30548", "1969", "37610", "13215", "53088", "84", "23136", "31295", "49240", "37869", "2372", "56050", "17893", "21058", "250", "17412", "40963", "21691", "19286", "39034", "56386", "53957", "20803", "22680", "50569", "35783", "16057", "58763", "9804", "40071", "57826", "59957", "41103", "59946", "48433", "48058", "26549", "37179", "32823", "40254", "50786", "57034", "24827", "16172", "27187", "25387", "26514", "22222", "35244", "21948", "45645", "15403", "39317", "30617", "8936", "31319", "29302", "52842", "18666", "45658", "38222", "37318", "53373", "1989", "51786", "25539", "847", "7387", "47722", "7755", "36800", "58018", "2316", "23426", "46024", "29738", "45534", "50460", "14868", "25075", "21767", "23530", "50803", "46469", "58001", "51629", "27673", "36098", "24015", "3394", "59224", "46614", "54744", "25083", "31248", "58000", "29909", "30644", "27480", "16969", "12807", "57028", "24148", "49610", "47709", "10803", "28063", "51929", "58535", "37588", "6951", "19907", "33107", "27529", "822", "14005", "8045", "42069", "40934", "28805", "18730", "40700", "49122", "48214", "23423", "45700", "33726", "52513", "57676", "35802", "18657", "5197", "47791", "17024", "24487", "46720", "13793", "57796", "22989", "9216", "24948", "26266", "15028", "42352", "22831", "18485", "56207", "38404", "300", "908", "57563", "32314", "56398", "9985", "56994", "48641", "29675", "37182", "16976", "14365", "13930", "29068", "47484", "51219", "46776", "42331", "15972", "35962", "40062", "26249", "17887", "26119", "51575", "39574", "34942", "41867", "14391", "13965", "34657", "39023", "43826", "37874", "24465", "12301", "49132", "46241", "33891", "35999", "6298", "54021", "39327", "46941", "53643", "44793", "28638", "15049", "17035", "24039", "20024", "53883", "34406", "1674", "3879", "4027", "12251", "23895", "46930", "34606", "24175", "53764", "31157", "34646", "28775", "42282", "41696", "55957", "6052", "13995", "50041", "44864", "54047", "38642", "23878", "20129", "31787", "53791", "24814", "5523", "3388", "50557", "8379", "23999", "11872", "18978", "3023", "26894", "57857", "45750", "55840", "30686", "43105", "38106", "22812", "1854", "23", "57021", "5360", "30483", "36575", "54303", "45070", "4411", "15474", "35452", "41163", "20538", "31828", "56005", "12289", "45634", "8614", "2383", "32320", "19050", "42301", "25029", "33471", "43082", "29344", "50253", "18129", "14333", "49098", "40367", "39482", "17071", "15873", "57298", "57576", "21616", "7089", "32710", "15788", "17204", "16932", "18744", "58463", "10836", "7063", "26269", "30245", "229", "1120", "7178", "13506", "12453", "8464", "15554", "29063", "40365", "1376", "52933", "40744", "3536", "14410", "36533", "26178", "18763", "18375", "42512", "41741", "6462", "19782", "15338", "59122", "40032", "40946", "55760", "29641", "53444", "41761", "16808", "58336", "44144", "3323", "56141", "12415", "41549", "46944", "26221", "50503", "19875", "15964", "58098", "11822", "55693", "58740", "41700", "57324", "23347", "5345", "48948", "5642", "30908", "51372", "47757", "47277", "33251", "28901", "35183", "16176", "21494", "12494", "46691", "38632", "26818", "41994", "41400", "27077", "30189", "59821", "26418", "4397", "43966", "25761", "22740", "26159", "8830", "46114", "2165", "15850", "18637", "58117", "4403", "30325", "48228", "19044", "28121", "3982", "8765", "38055", "43086", "44532", "16789", "736", "27761", "12623", "13886", "8188", "4550", "42490", "25664", "5623", "14610", "56318", "47433", "45555", "53060", "34618", "42810", "19686", "58543", "44951", "33353", "8252", "16137", "20921", "41161", "54056", "815", "5080", "19402", "14001", "12255", "5591", "51583", "32820", "49597", "17303", "23585", "8267", "8543", "13228", "16085", "55757", "6285", "52518", "15944", "55308", "36375", "25413", "37980", "32962", "38134", "11219", "3331", "8059", "52558", "11304", "22601", "24849", "2118", "53256", "48960", "46029", "5252", "51487", "2180", "7238", "38892", "11875", "57279", "35432", "12447", "42855", "52875", "11205", "59533", "7649", "56509", "47828", "46335", "6846", "35492", "37627", "17306", "38714", "33140", "2285", "19792", "15748", "31663", "57471", "41586", "27878", "34128", "11632", "20690", "5584", "49287", "14908", "19219", "52264", "51311", "6902", "14673", "17085", "2757", "32433", "9783", "58577", "6086", "57624", "50309", "7068", "17433", "2239", "4142", "42153", "38149", "9908", "38239", "6584", "44542", "25577", "54106", "4054", "58924", "13934", "37354", "10581", "59117", "26870", "55006", "16145", "20004", "19190", "59041", "13222", "30630", "23162", "12644", "59298", "5601", "20617", "32687", "30172", "48303", "43500", "13511", "33898", "15512", "20655", "17857", "35533", "39582", "4149", "14025", "6556", "38166", "7127", "36332", "59576", "210", "12853", "10423", "18381", "3681", "43484", "43519", "46365", "31483", "24454", "37031", "32494", "59098", "51955", "33698", "6510", "50456", "2203", "15597", "32761", "24279", "2950", "30862", "19615", "57206", "7286", "4361", "16639", "2819", "56828", "9485", "42643", "37821", "55107", "42537", "7173", "225", "58884", "46784", "29242", "49688", "31875", "6802", "6984", "9748", "22765", "49163", "9185", "34714", "34741", "55272", "59961", "4194", "53907", "19575", "27013", "55085", "46884", "50050", "23899", "41563", "22088", "50375", "56861", "4757", "29049", "50380", "40438", "19606", "57580", "25821", "55493", "6647", "14926", "12326", "22519", "26640", "38614", "19383", "34577", "57013", "52449", "17394", "25198", "48903", "52444", "18861", "58960", "42317", "14786", "55419", "6822", "23897", "46920", "34782", "49946", "26431", "29036", "32779", "11403", "28056", "5468", "13324", "47042", "26182", "42417", "3961", "8225", "747", "51457", "2849", "15894", "39756", "48203", "52380", "47634", "15108", "56283", "37136", "54636", "6358", "11498", "15179", "57827", "11672", "55554", "31011", "25302", "19490", "15235", "45032", "8818", "52777", "48464", "44027", "32159", "51687", "19375", "8728", "23081", "8690", "53698", "58348", "24049", "15653", "17788", "55546", "44208", "23703", "48601", "11792", "312", "28621", "28999", "50464", "50810", "19794", "26163", "12655", "17505", "4986", "37976", "12237", "56656", "10748", "51656", "3721", "29167", "7615", "19492", "47969", "35623", "37164", "43697", "34108", "29421", "38761", "50038", "37453", "46494", "29801", "55406", "10917", "49915", "13287", "20958", "39624", "28707", "2918", "59249", "44816", "36634", "4740", "9144", "49313", "24568", "45385", "20446", "40103", "52687", "57454", "11910", "3440", "34143", "51557", "23272", "46427", "26945", "4946", "37978", "38373", "47978", "30038", "32246", "35924", "46095", "19659", "19731", "36746", "37556", "10123", "42315", "54648", "46404", "58691", "44181", "39095", "44941", "54323", "14612", "44585", "9974", "52567", "16954", "57698", "46975", "43753", "20444", "40374", "43603", "18974", "20968", "26953", "37493", "57484", "4680", "12459", "23303", "35940", "20264", "57137", "44201", "50120", "26947", "22441", "36902", "50226", "7020", "6063", "1660", "47921", "49231", "22284", "50403", "41526", "34341", "46343", "39521", "40795", "23704", "15170", "44121", "37168", "17134", "40683", "98", "41243", "34756", "38978", "59311", "52580", "58788", "11653", "47953", "269", "56590", "42191", "25148", "41090", "22826", "4451", "13817", "1245", "32330", "27254", "27245", "682", "24774", "51600", "41550", "30527", "55559", "47287", "46134", "47399", "42346", "41861", "57559", "32081", "31655", "21387", "47646", "18272", "19419", "12799", "28544", "57952", "5215", "12214", "55487", "48381", "8554", "7211", "37555", "42128", "36629", "30917", "33062", "8957", "7222", "33193", "36362", "47369", "28177", "28649", "13048", "21727", "10551", "32923", "51433", "14719", "51244", "52828", "13589", "3700", "25526", "83", "27527", "30225", "644", "44932", "4529", "32315", "29328", "4603", "9365", "55328", "37469", "47030", "34925", "22353", "49755", "57851", "21802", "5768", "8539", "24616", "23297", "55536", "46815", "14129", "9934", "3985", "32254", "42235", "43067", "53300", "6730", "45414", "31048", "9654", "7797", "59360", "21640", "6636", "8016", "51486", "36899", "45458", "43253", "28255", "46903", "49971", "54931", "59617", "37317", "21535", "56010", "54032", "35671", "57458", "2765", "14762", "7826", "54653", "58450", "54965", "21136", "30275", "37260", "54901", "43222", "41716", "8996", "40542", "48478", "17629", "18217", "1742", "36702", "17178", "59391", "48092", "7251", "6609", "52811", "39877", "34300", "37232", "52148", "44906", "16764", "48157", "21619", "10817", "16615", "10287", "57548", "45666", "32331", "5223", "50281", "4642", "8024", "59090", "57646", "36731", "17269", "2145", "28349", "34231", "17757", "38530", "27924", "47623", "43771", "21308", "45264", "41239", "23506", "27870", "49422", "2398", "3373", "52035", "43742", "55253", "36", "57188", "408", "6082", "49136", "27901", "40912", "590", "16732", "17967", "13290", "41887", "54835", "32465", "24242", "59039", "59466", "10201", "2576", "57898", "14751", "9857", "55182", "41072", "33801", "36859", "32579", "6897", "46445", "58204", "49857", "52786", "1016", "52566", "37575", "14515", "31633", "29711", "5817", "53556", "35945", "27236", "21003", "24082", "41698", "37711", "3901", "43019", "32486", "50046", "27119", "43003", "59345", "30366", "53880", "53846", "35283", "19339", "23024", "52373", "34612", "30721", "49945", "46416", "17041", "21181", "17765", "39008", "21888", "57783", "58755", "11375", "27767", "25459", "6759", "33368", "38775", "58919", "51885", "5705", "55667", "2019", "42249", "3101", "3597", "13234", "11642", "46593", "20200", "32220", "52793", "42634", "19230", "29834", "18822", "35110", "51255", "50110", "46600", "55580", "2456", "5668", "59823", "9831", "12260", "30190", "26004", "46730", "20143", "9880", "41868", "28084", "39857", "3348", "14630", "26689", "36078", "11195", "34150", "5639", "50537", "38962", "29133", "4772", "28952", "34199", "26909", "24444", "16502", "14635", "20521", "53703", "41457", "52100", "10157", "12049", "53777", "45860", "47474", "5959", "55346", "4372", "40937", "35115", "46462", "3434", "58295", "13559", "9153", "24238", "22895", "39839", "37304", "40", "18353", "6061", "39809", "37339", "3588", "21959", "10955", "20404", "40018", "48455", "27209", "13539", "47693", "11870", "7233", "17424", "41310", "59308", "56442", "41208", "4332", "26748", "7594", "15817", "11927", "2591", "41145", "36022", "20161", "44095", "4088", "57925", "54135", "59958", "49451", "31311", "13125", "58441", "35544", "53282", "57450", "17561", "30154", "5227", "20112", "37640", "55943", "50989", "32045", "28212", "36056", "45380", "9297", "10617", "44834", "23366", "54659", "36028", "53577", "58400", "15667", "10502", "51783", "40336", "32602", "15865", "26583", "28416", "35710", "9093", "5626", "57769", "34536", "49327", "54383", "56069", "21160", "47103", "20292", "44106", "46615", "41495", "58123", "45653", "30728", "26298", "50669", "56098", "15278", "6020", "34297", "45083", "50848", "48598", "5931", "50443", "49175", "44084", "38750", "28510", "7868", "26101", "12411", "26592", "18341", "1216", "4653", "55178", "1261", "43024", "39802", "38042", "45520", "20696", "57328", "4698", "26863", "16800", "15592", "37587", "33276", "59708", "5029", "50017", "43772", "765", "4309", "37034", "12806", "52336", "22371", "38093", "47318", "45113", "14323", "56287", "49524", "46951", "39912", "22963", "11009", "37545", "33548", "54366", "45595", "50316", "54645", "58224", "46020", "50898", "31859", "30418", "35439", "34458", "41486", "17268", "40181", "24368", "34179", "23378", "2215", "46281", "54818", "42254", "27349", "35592", "1", "131", "43544", "24420", "47305", "39452", "10830", "59421", "20635", "50352", "30890", "54051", "23765", "59443", "14524", "27648", "40811", "5580", "50429", "40838", "7960", "35891", "31471", "58064", "13184", "15185", "8739", "21000", "31057", "49403", "59349", "57149", "5843", "39137", "59791", "21002", "30135", "41626", "2987", "35227", "37261", "48633", "1600", "30184", "23130", "3640", "53780", "49630", "58903", "9094", "24230", "21685", "25893", "10665", "10279", "53724", "56226", "25344", "59428", "31482", "2079", "16604", "51203", "34461", "8264", "33679", "13238", "45164", "58338", "41268", "34975", "46477", "54803", "52258", "39704", "50128", "16334", "9204", "51855", "37883", "19638", "2389", "10963", "57534", "33492", "21690", "13701", "45329", "19159", "16992", "31665", "14", "47515", "55417", "26289", "13907", "2137", "51332", "23602", "28575", "17252", "5808", "32156", "1763", "51105", "45327", "35311", "5994", "3886", "27274", "45448", "57246", "15094", "57134", "30230", "48597", "24658", "4099", "26838", "41865", "49619", "33187", "5736", "30348", "43858", "23605", "15819", "26118", "39720", "17379", "26892", "7701", "18069", "47715", "2715", "49916", "6954", "27718", "7234", "11179", "29127", "40821", "9848", "38876", "41924", "15139", "37160", "37820", "16340", "10269", "2236", "29922", "47875", "43065", "7407", "57878", "17611", "55124", "23848", "51263", "40049", "23743", "17369", "17198", "3629", "35864", "51199", "31341", "40839", "2362", "24070", "16135", "53240", "15674", "44408", "59827", "40680", "33423", "43433", "10224", "1920", "36026", "49299", "8607", "24435", "36212", "18751", "29065", "51028", "15383", "58488", "3363", "27535", "11203", "1594", "53595", "7190", "15398", "25532", "27831", "35485", "38513", "57887", "674", "27183", "29854", "59442", "18196", "12210", "11493", "59714", "43566", "27623", "48293", "32043", "28019", "2146", "10474", "55221", "46156", "36251", "57120", "12319", "15122", "41773", "53210", "37851", "16037", "4813", "28040", "42124", "20937", "4131", "34075", "56301", "27632", "30839", "15365", "56721", "52846", "88", "28398", "2235", "25388", "52063", "55809", "16566", "48845", "25550", "23846", "16948", "53159", "30160", "30202", "54205", "23485", "24336", "48149", "42305", "32834", "56403", "42391", "51470", "43005", "51422", "47801", "27982", "12054", "2096", "55064", "47816", "30297", "9903", "27210", "46127", "20687", "51446", "58649", "43668", "3273", "10869", "48807", "5837", "21060", "45306", "35454", "30183", "10131", "55539", "13508", "43452", "14863", "49668", "23302", "27926", "34608", "52145", "8324", "39938", "39701", "11863", "1755", "42909", "30213", "39467", "19889", "8488", "24174", "35629", "59762", "2753", "48610", "16047", "52132", "38289", "5864", "19282", "40832", "20518", "19041", "30155", "13587", "46292", "47023", "52538", "26755", "50745", "45008", "7204", "27644", "16867", "8509", "23047", "31595", "12726", "22382", "52224", "36136", "23449", "36434", "14780", "47102", "57320", "33284", "1054", "48268", "59673", "14603", "55833", "30205", "33248", "2435", "22231", "45114", "51811", "19184", "3993", "53457", "30018", "59259", "44382", "38947", "19930", "26977", "10507", "40364", "52542", "53948", "41746", "45716", "43125", "7022", "40406", "14374", "15555", "37326", "55169", "7338", "59629", "49608", "5002", "14180", "39883", "23528", "42278", "29033", "47554", "16535", "10316", "21333", "36325", "35428", "43853", "55215", "20815", "20453", "48138", "47518", "54121", "48537", "53164", "29492", "6869", "15831", "49625", "16080", "24716", "35960", "26328", "15382", "22920", "29386", "44908", "37126", "30269", "38667", "13063", "32812", "40514", "36055", "44026", "40849", "17715", "32272", "48549", "19403", "55720", "4014", "34078", "34982", "31265", "17203", "59971", "1453", "55590", "23052", "44894", "12856", "27422", "24623", "32970", "52908", "19384", "43978", "18918", "33550", "59743", "31316", "44110", "4315", "53449", "47444", "56642", "3162", "33097", "46259", "21248", "34268", "57881", "22092", "47597", "52306", "1629", "41283", "36381", "9163", "11987", "37562", "55314", "17167", "22232", "50732", "54387", "59105", "15384", "27427", "37873", "30217", "50400", "29860", "5844", "10521", "36372", "45093", "53345", "464", "2246", "4133", "28290", "16876", "27837", "9962", "48701", "889", "33400", "48212", "23907", "32418", "38742", "3483", "46870", "7850", "18578", "30242", "41857", "8405", "14327", "44631", "46683", "15898", "32196", "13164", "21759", "36914", "19866", "58704", "22021", "9639", "6234", "3951", "26501", "54749", "20222", "18336", "37908", "9322", "45119", "57301", "26928", "16300", "14550", "46650", "32058", "8469", "36309", "33869", "17151", "44271", "29792", "34324", "29471", "52919", "21324", "22004", "23231", "4273", "45305", "29210", "29962", "2805", "36886", "24005", "48613", "49819", "59782", "55893", "45300", "52486", "15011", "55994", "5214", "414", "5436", "17422", "52718", "14214", "6467", "42239", "4687", "34505", "33704", "3606", "50155", "38851", "37144", "48502", "46264", "22267", "47794", "59541", "392", "16420", "16793", "22473", "32285", "1180", "759", "39523", "22711", "47594", "52420", "23265", "59826", "21016", "40738", "16515", "9754", "51275", "41076", "59711", "17892", "53185", "45796", "13170", "25844", "44969", "18849", "2990", "23772", "31035", "13435", "18327", "2200", "48158", "28188", "49912", "40600", "30334", "16844", "8565", "58351", "3589", "5241", "12325", "18571", "31314", "56792", "22423", "40012", "24953", "21382", "36976", "46809", "44018", "31702", "39830", "55050", "15118", "45615", "39366", "44066", "44435", "3754", "38262", "36312", "20452", "40830", "48102", "42045", "11926", "56737", "56475", "47321", "3437", "42258", "47222", "50590", "49471", "13009", "40900", "39859", "35256", "44555", "18332", "43774", "1248", "24580", "23260", "54858", "38743", "12110", "21931", "52609", "18683", "9531", "14502", "28322", "38020", "1572", "59369", "2514", "17047", "29432", "49716", "14800", "57359", "18581", "15872", "47159", "13622", "34285", "21873", "2320", "2048", "44798", "6660", "21932", "44775", "23165", "32488", "2725", "22117", "28218", "53255", "27511", "15457", "7306", "8104", "46641", "48440", "33760", "20051", "2011", "6733", "23419", "29035", "34120", "36305", "56851", "55735", "16971", "40290", "44912", "19343", "5528", "9284", "19179", "5716", "19895", "18318", "38807", "7917", "14642", "2151", "50743", "27850", "7904", "51049", "51067", "37462", "29888", "55697", "34014", "8278", "38747", "34512", "5742", "42958", "33233", "42220", "12340", "27263", "10633", "43035", "37641", "5235", "28274", "2564", "54726", "15770", "45148", "49104", "39307", "1282", "42774", "46489", "43319", "53408", "45228", "22187", "5672", "42267", "53978", "14295", "58149", "43031", "52631", "50303", "33183", "44389", "24069", "12449", "16054", "11449", "45614", "47790", "32183", "42286", "21089", "2515", "33507", "7330", "41756", "16086", "24064", "14825", "55214", "54961", "14145", "7508", "55379", "24088", "54126", "24366", "55507", "30059", "22326", "20464", "39213", "38973", "10527", "23038", "1651", "25257", "35741", "42917", "34118", "24741", "52212", "55003", "28950", "3978", "54607", "11215", "54842", "22612", "55709", "50995", "33982", "25968", "54612", "15313", "13411", "16158", "28016", "1668", "1713", "38132", "11917", "22042", "22511", "34042", "27231", "25866", "54075", "57055", "26181", "8459", "24292", "17886", "21457", "33563", "20290", "14450", "22538", "48891", "39091", "50905", "35988", "13957", "4720", "28624", "8548", "44511", "46892", "55008", "7741", "22736", "56263", "1303", "56122", "7156", "20209", "51050", "56116", "34178", "41511", "47915", "23862", "37148", "19993", "28234", "34193", "30222", "49480", "6230", "37718", "16744", "24073", "48096", "23820", "655", "24220", "49182", "14338", "6431", "4863", "34398", "15700", "13794", "9948", "36586", "851", "59497", "33927", "37319", "17498", "25072", "39217", "35348", "25697", "53749", "22949", "612", "14312", "47567", "15933", "21105", "180", "7067", "22244", "25502", "44752", "55606", "58531", "17772", "41527", "9207", "1890", "10266", "23607", "30251", "52180", "24956", "40590", "51302", "7148", "49850", "47802", "15503", "41818", "29191", "50872", "11770", "44380", "34408", "3602", "19656", "41236", "51790", "56043", "6689", "396", "44850", "34591", "32542", "59322", "20780", "30171", "1973", "7817", "21965", "16879", "43338", "25161", "14075", "24739", "27125", "58645", "14958", "7624", "56481", "6474", "31537", "13353", "48209", "51369", "39680", "41564", "44358", "21186", "25568", "12072", "49983", "50287", "18460", "22827", "47215", "27281", "19715", "33768", "43499", "23358", "39165", "38186", "5689", "46966", "41785", "3198", "21684", "38507", "19642", "52263", "49734", "24402", "93", "44220", "44795", "6265", "53285", "37677", "29639", "47338", "57519", "21660", "16752", "14837", "24982", "31794", "53979", "4016", "16000", "38931", "46347", "38538", "21784", "6925", "23309", "5670", "38731", "16838", "8622", "43392", "22330", "18291", "39689", "12767", "27613", "58955", "24166", "25018", "15920", "58715", "5412", "29288", "54896", "51986", "29181", "28007", "47555", "14913", "1669", "17392", "41877", "23427", "10910", "9839", "1953", "27948", "50422", "26910", "41874", "483", "39372", "6976", "37384", "52413", "40058", "14383", "23166", "4870", "44986", "49190", "46471", "53860", "47195", "2758", "16492", "33842", "26047", "14116", "35080", "16511", "39099", "17336", "39018", "12197", "21431", "9829", "17619", "1467", "45699", "45543", "18403", "14887", "46453", "48823", "55391", "15839", "8183", "14073", "29146", "16358", "8168", "7821", "13764", "33975", "4577", "28285", "39518", "59702", "21744", "22574", "40441", "55556", "58399", "57319", "39121", "58373", "31271", "25765", "29268", "55266", "45322", "43039", "19588", "38815", "39093", "31195", "54923", "53610", "5478", "39966", "35480", "18651", "28005", "46945", "23064", "14597", "53496", "34636", "37171", "45102", "31797", "20633", "1607", "33332", "57177", "39412", "30879", "8502", "7977", "14851", "50755", "16032", "50406", "41368", "33682", "13641", "15889", "9368", "1721", "35984", "46367", "8931", "12195", "31707", "18579", "47239", "16977", "1044", "101", "17778", "29096", "16788", "49306", "35092", "45646", "4395", "1772", "13243", "29323", "7464", "4439", "45473", "19561", "45156", "59707", "38185", "18517", "50197", "8272", "55495", "6232", "41869", "5418", "24861", "39276", "6329", "38297", "36529", "920", "12865", "36501", "33994", "33922", "54386", "1864", "14203", "6106", "57845", "55235", "55070", "8470", "4530", "12127", "3359", "37355", "49251", "27792", "5104", "6377", "5038", "22778", "23012", "38823", "2039", "53649", "45697", "18343", "17428", "35947", "37165", "14454", "46543", "2143", "42556", "26921", "17232", "30977", "23757", "17517", "48073", "59747", "16019", "27725", "26507", "43600", "3576", "50926", "14565", "54361", "30359", "3838", "46045", "2454", "25567", "31513", "57510", "55060", "25736", "56214", "43415", "4339", "49952", "33054", "23744", "3776", "19096", "25463", "5095", "57628", "51064", "43736", "44918", "11073", "28807", "58511", "1944", "25941", "39951", "16922", "15144", "22690", "38133", "54834", "29376", "25125", "28358", "48296", "49461", "33529", "24516", "57419", "39205", "28299", "14371", "15958", "8237", "49903", "40480", "4704", "9018", "446", "59007", "43012", "8512", "56324", "22524", "36719", "9919", "20945", "11331", "30349", "28528", "32551", "58431", "10751", "50832", "29794", "4648", "3224", "56762", "7219", "2517", "59273", "28500", "26820", "52463", "7115", "13779", "37062", "23452", "45314", "24709", "59100", "45313", "44632", "47306", "10338", "56372", "4120", "37018", "42480", "14170", "46110", "43241", "24825", "21539", "57333", "541", "11062", "30356", "21495", "24016", "10649", "6199", "10781", "41848", "29582", "44149", "3405", "35727", "56240", "23408", "31142", "44172", "17205", "51626", "52949", "52835", "58688", "27402", "9440", "56217", "39133", "35619", "1684", "38102", "57123", "49219", "13769", "18509", "17161", "39485", "2535", "8976", "51892", "46093", "37439", "50160", "50258", "14902", "54269", "26985", "11248", "19352", "55608", "16219", "31842", "40074", "36483", "1559", "11627", "51975", "31013", "55484", "27687", "32587", "4330", "48762", "35972", "51679", "39202", "44274", "18470", "795", "4066", "51197", "34190", "56038", "56907", "46431", "44288", "29559", "28988", "17263", "9707", "28171", "17798", "31123", "20515", "7656", "57201", "6273", "20050", "23833", "56189", "20463", "35852", "37645", "24611", "59210", "52138", "5206", "42694", "32374", "44038", "7876", "56013", "25627", "42269", "12575", "57430", "7147", "56084", "46278", "44928", "57989", "44743", "26920", "36391", "3920", "16759", "36347", "49817", "23382", "46492", "56389", "3671", "34465", "17929", "24985", "20250", "49637", "25056", "34772", "42620", "31573", "13203", "31377", "35678", "19790", "28831", "24055", "12013", "12576", "55301", "10410", "44153", "45282", "28631", "46575", "52598", "4818", "9044", "55599", "9242", "2835", "12721", "19019", "4020", "44914", "57376", "40041", "43597", "49698", "21007", "5924", "33460", "38769", "10794", "27597", "36733", "36596", "39820", "44633", "46764", "2470", "3397", "38683", "15427", "5168", "35504", "15772", "8348", "14652", "29722", "47041", "24048", "5688", "10288", "6180", "32948", "36622", "7599", "18417", "6131", "39504", "26310", "14647", "9990", "29113", "19788", "28347", "31643", "1922", "13368", "38666", "46486", "8890", "42952", "29769", "15394", "11021", "17780", "26397", "13931", "47882", "27973", "29534", "21261", "38291", "6667", "10815", "46700", "41425", "34962", "45987", "59737", "31602", "9039", "35225", "6257", "17489", "6943", "36606", "58618", "18667", "32441", "19992", "53570", "25586", "52430", "22233", "13046", "2479", "52529", "27206", "36152", "32984", "42710", "3685", "19801", "48218", "39971", "23536", "51884", "52938", "26923", "2275", "35362", "41438", "40557", "6793", "6049", "7001", "1058", "38559", "27905", "57889", "981", "32216", "7866", "36544", "52228", "32417", "17055", "22807", "51033", "57836", "37532", "13723", "9565", "12380", "1279", "37464", "11333", "40812", "6469", "27780", "53949", "52791", "3706", "23440", "35814", "35569", "33996", "12634", "47682", "3072", "59579", "53628", "56877", "31679", "41750", "11527", "38860", "23242", "7086", "2721", "56485", "26791", "33259", "9527", "18801", "32060", "53023", "36572", "25982", "441", "32864", "8475", "15000", "43199", "12506", "43609", "27300", "12213", "6447", "38917", "54722", "2307", "57600", "56012", "57828", "6141", "38395", "56925", "45806", "27015", "14883", "5840", "17299", "29186", "45590", "56264", "20510", "10941", "53051", "52183", "49435", "27418", "12073", "49448", "38600", "997", "34251", "2949", "57734", "910", "52689", "29458", "56501", "20952", "51093", "13983", "56830", "35455", "7726", "54381", "22483", "45253", "12961", "43247", "52775", "32048", "43009", "5947", "53182", "34596", "52988", "50775", "15910", "19892", "6394", "52316", "19590", "49785", "59231", "28863", "11048", "30397", "4501", "39456", "49241", "39702", "42344", "26320", "30340", "28675", "58949", "3267", "51638", "42742", "10540", "43538", "59643", "21627", "15078", "6178", "1421", "33637", "58515", "54721", "26066", "42263", "47262", "8585", "12386", "7384", "33885", "17117", "26616", "33402", "687", "27118", "7093", "53", "7129", "41638", "40641", "45226", "38553", "52986", "54190", "15857", "10080", "46085", "17733", "26222", "30816", "52294", "41999", "23470", "49788", "24976", "28428", "42171", "59166", "26811", "49334", "29826", "32657", "44228", "42410", "54411", "19406", "28705", "15088", "36104", "46052", "6093", "15217", "10726", "14105", "42528", "4095", "33256", "13166", "11476", "51677", "37399", "30255", "21814", "46080", "41026", "43723", "8313", "45062", "48567", "45918", "54144", "38755", "54441", "2677", "41763", "19474", "35301", "14110", "47674", "26339", "18639", "50701", "12870", "31938", "32966", "37667", "19763", "50111", "740", "5650", "4420", "38675", "15874", "44156", "24572", "47847", "4074", "56623", "25004", "7138", "45214", "28730", "1086", "33542", "1839", "42858", "57075", "34773", "49520", "6366", "28031", "32783", "8806", "26065", "5531", "6010", "21608", "21969", "38905", "4759", "9850", "48991", "57550", "34027", "38862", "12287", "51447", "25731", "39789", "12401", "7799", "59130", "48318", "59086", "53936", "42837", "28630", "29530", "28623", "9443", "54968", "57170", "14272", "40484", "41963", "20811", "4433", "22279", "10807", "23983", "32738", "28891", "45418", "57116", "58300", "46775", "38101", "11019", "34013", "17296", "29757", "51660", "53534", "15899", "9596", "37721", "38192", "11839", "38351", "30211", "26451", "34069", "19744", "31069", "6303", "48200", "12286", "7723", "55426", "7885", "58249", "45914", "27991", "41592", "55497", "46361", "57523", "50917", "29872", "26061", "43615", "27698", "52199", "53148", "41044", "6765", "14245", "7424", "37799", "1598", "33467", "56855", "11446", "6198", "50528", "1693", "57948", "5380", "8486", "41396", "41155", "23432", "7853", "23591", "8239", "30817", "10417", "15214", "30144", "35174", "33777", "45843", "43235", "7327", "57151", "13665", "13305", "22637", "3017", "49940", "11844", "10298", "10354", "6259", "44844", "47249", "32484", "12891", "18527", "10491", "8571", "31350", "36425", "59123", "19818", "53117", "32661", "17029", "52142", "25904", "15055", "34578", "18385", "35942", "55974", "16906", "54676", "24582", "5882", "47493", "26499", "50635", "19640", "8960", "42731", "51820", "58738", "47628", "45880", "30361", "10293", "6713", "59923", "18958", "4469", "22973", "15930", "22897", "59595", "52907", "33376", "46563", "16146", "34174", "38737", "32530", "9483", "42846", "59068", "50417", "44174", "6393", "16106", "12363", "53141", "42154", "34770", "29748", "4084", "33877", "42339", "2981", "11335", "25461", "8423", "19940", "6611", "11687", "42011", "13977", "25116", "6616", "39396", "2565", "21480", "10204", "5653", "58106", "3496", "6907", "41160", "38287", "33164", "9328", "34214", "9298", "39638", "27081", "18315", "58961", "34434", "27247", "18884", "831", "15646", "18091", "35884", "57616", "42330", "56299", "43252", "47320", "26560", "19026", "39595", "38555", "8929", "28364", "14148", "7470", "59995", "10509", "22332", "13803", "16173", "32711", "11221", "37064", "23653", "32553", "54099", "19692", "45067", "15993", "1703", "43851", "42058", "44264", "57872", "19734", "47251", "22292", "41206", "33832", "38466", "49007", "41453", "31805", "57904", "14090", "25988", "40003", "2139", "29078", "32606", "27739", "33158", "39142", "31691", "56056", "29733", "14690", "51669", "42126", "34624", "15775", "2697", "55023", "15033", "17945", "28961", "12928", "46676", "9479", "12889", "42902", "1135", "33576", "32960", "29439", "29472", "44108", "5189", "51174", "5553", "18501", "19554", "53125", "38650", "28073", "50566", "46942", "15516", "45347", "7487", "37145", "3519", "48406", "22965", "30552", "17532", "31340", "12750", "57268", "24323", "8504", "3930", "44457", "13900", "51191", "39097", "40349", "1960", "32891", "15335", "56884", "34716", "59228", "4377", "336", "3384", "21925", "9271", "1658", "15909", "54733", "41748", "25319", "48461", "58277", "46497", "43457", "25283", "19288", "54561", "52643", "39593", "20528", "21729", "22297", "46244", "13857", "33947", "21606", "13727", "26674", "22788", "51040", "31168", "33019", "33734", "54538", "20623", "26028", "5796", "39869", "22302", "47879", "13424", "27801", "20955", "5004", "48465", "50119", "31065", "50573", "6378", "9299", "46863", "44939", "46331", "21736", "37748", "16590", "38240", "9811", "23054", "20106", "18545", "44263", "32996", "49234", "43733", "44707", "9841", "14117", "32512", "23152", "42027", "26385", "32", "26505", "18411", "30015", "5341", "44268", "53887", "35808", "57849", "46217", "51138", "18893", "48848", "52127", "31440", "43345", "48888", "7695", "12086", "53351", "30024", "1262", "14805", "36691", "9861", "48270", "56329", "56528", "29299", "54250", "9583", "4617", "5618", "52752", "58706", "40563", "27186", "3670", "6668", "45840", "17678", "20445", "16813", "38523", "36730", "43902", "52320", "46827", "7768", "12757", "9022", "52879", "36615", "38328", "45539", "26170", "17550", "9842", "55174", "44400", "13851", "42476", "14209", "44667", "40265", "10499", "17785", "26475", "4486", "18454", "48413", "26835", "4093", "27297", "49266", "33385", "38017", "25469", "7996", "25293", "26023", "42696", "34359", "25611", "30998", "18726", "11689", "49486", "54337", "21905", "21350", "52028", "40297", "50386", "8560", "6978", "17549", "18394", "12056", "35483", "16654", "7734", "31641", "57806", "31593", "47487", "58374", "20287", "31060", "49604", "9287", "40628", "4076", "56083", "223", "57824", "6981", "28245", "27196", "11100", "16797", "22006", "50554", "25781", "45073", "46229", "27336", "49341", "37967", "26623", "15024", "31391", "56568", "44750", "49224", "54360", "49257", "16872", "1861", "48488", "56351", "48133", "17936", "26485", "37420", "47048", "14629", "53998", "2068", "14281", "16576", "44259", "37094", "27586", "41360", "55284", "21549", "36382", "54898", "34197", "27253", "23749", "8906", "33234", "8019", "56984", "33738", "1569", "39751", "9971", "14337", "55145", "17784", "42667", "16781", "56123", "19721", "7402", "40006", "29817", "18424", "24978", "52134", "24591", "22817", "25590", "20163", "15379", "51396", "28324", "25038", "53098", "20911", "49046", "26582", "39662", "16560", "9530", "30346", "18512", "8310", "56157", "59933", "33136", "52914", "33138", "54346", "15386", "57444", "14995", "6304", "33516", "10544", "45424", "27026", "57776", "24416", "14592", "56924", "16134", "27877", "48643", "45705", "20156", "12897", "49744", "42137", "52286", "1024", "26833", "1225", "32877", "29045", "27741", "15623", "28167", "30426", "25497", "42014", "528", "50991", "55612", "40553", "30315", "51590", "52853", "7153", "7364", "34537", "47979", "19255", "30570", "874", "33478", "59051", "25156", "10020", "15525", "3935", "57335", "13933", "28600", "24095", "22662", "17823", "29890", "13136", "38338", "37853", "12886", "39360", "5267", "326", "18703", "39487", "20149", "11127", "32181", "12508", "46360", "27891", "52982", "53654", "5177", "5855", "48009", "7152", "37389", "56360", "31164", "58498", "32690", "1190", "17562", "48793", "9541", "55371", "38021", "33901", "6278", "55468", "14263", "9600", "35442", "46198", "22801", "47748", "5234", "14384", "37069", "32228", "26897", "30814", "30830", "48276", "5838", "19270", "5022", "29538", "34908", "38842", "18576", "48861", "34455", "34934", "4916", "2154", "7473", "10993", "30101", "54502", "54324", "36549", "1667", "50629", "22512", "54891", "12485", "24144", "53412", "20034", "32131", "50084", "38449", "13949", "33870", "6203", "8228", "32794", "5535", "2605", "23025", "50886", "45832", "58873", "13953", "15791", "54720", "11984", "24773", "39816", "29300", "48995", "40474", "53479", "47193", "156", "5024", "40792", "19130", "45466", "10081", "20905", "33930", "4600", "52660", "40434", "46666", "26213", "27331", "33971", "7409", "55683", "57603", "57934", "43998", "40293", "59749", "6225", "9969", "24384", "36323", "36839", "58350", "35277", "42881", "53603", "42375", "17710", "17529", "35335", "23460", "48983", "22024", "57286", "16760", "16736", "27400", "29636", "58694", "13731", "40285", "37981", "14009", "14936", "53844", "4564", "3129", "12145", "39243", "39125", "38586", "52408", "50266", "41300", "8116", "27466", "2197", "58816", "32561", "59918", "16756", "56139", "2132", "36016", "39573", "31631", "58168", "27859", "31569", "25617", "27134", "25948", "49129", "9862", "14845", "7079", "15495", "13265", "37243", "21646", "57199", "51797", "51492", "2639", "5376", "18786", "3711", "21760", "32343", "9034", "29066", "5562", "31578", "16831", "3976", "13396", "58254", "56675", "37945", "38987", "32601", "52488", "49425", "35344", "29037", "56634", "18673", "55119", "52648", "44357", "8998", "4652", "16042", "45123", "11194", "32829", "8510", "1161", "55393", "18245", "29995", "47437", "58997", "6554", "28987", "58938", "28054", "44979", "13846", "32785", "43453", "1777", "39501", "46893", "5856", "36929", "30595", "59058", "16257", "57103", "26302", "16360", "23558", "43221", "53399", "54877", "53486", "44791", "49462", "32408", "58414", "48067", "57539", "19880", "28679", "48491", "8584", "2769", "47633", "58303", "21551", "39547", "44723", "26789", "29413", "44837", "11080", "10478", "43921", "31936", "49883", "23928", "50353", "24694", "17779", "1165", "19527", "27960", "7558", "30636", "17616", "56469", "46552", "18925", "11101", "12406", "26282", "58713", "38504", "6039", "10342", "10818", "13751", "34361", "57018", "34876", "34342", "54178", "57297", "30373", "39781", "48724", "13783", "3081", "58405", "26168", "16841", "30141", "59373", "45533", "10214", "27601", "9078", "3637", "9967", "54405", "52230", "53302", "55920", "33941", "8214", "33623", "46452", "42169", "4334", "29449", "52433", "17327", "18083", "6186", "49393", "47797", "9512", "20838", "21167", "43278", "49284", "33321", "16348", "33808", "5288", "33633", "59312", "37122", "53753", "43605", "39236", "12717", "43159", "23137", "19687", "24540", "40240", "19528", "15913", "20458", "47938", "38278", "26297", "34969", "1538", "25913", "6066", "40877", "8902", "16264", "25331", "15790", "42660", "31772", "11735", "57121", "48630", "16886", "42478", "17253", "24304", "47316", "10891", "31445", "24784", "50171", "51154", "54592", "10455", "44300", "31733", "7491", "1775", "10945", "57101", "5423", "53030", "40535", "35365", "40121", "49048", "23363", "1697", "37208", "43621", "17040", "26642", "12747", "8780", "47925", "30987", "57051", "10485", "42648", "56566", "59275", "19571", "38263", "32516", "11143", "9620", "54869", "59924", "31984", "59510", "20208", "23623", "30507", "4712", "36506", "14508", "28430", "18690", "58865", "8017", "58966", "45907", "53161", "4249", "31549", "5754", "45526", "36810", "22787", "11447", "56100", "43636", "2795", "53490", "3082", "24656", "38608", "58571", "59426", "34653", "57378", "41226", "33558", "38779", "33886", "23413", "1056", "11916", "14446", "7650", "931", "28051", "40052", "59500", "39894", "28253", "27164", "20971", "17699", "36338", "31845", "56082", "32234", "20994", "32633", "43385", "2230", "28934", "33465", "33893", "40104", "35527", "7616", "25709", "10488", "4179", "27053", "58453", "37494", "7954", "52050", "14722", "10145", "436", "37167", "17460", "28001", "17368", "58458", "33934", "12878", "40357", "50310", "41197", "43021", "3560", "12046", "56577", "17641", "35118", "44293", "12968", "56387", "4030", "58438", "57433", "13683", "9904", "39020", "38921", "25210", "4097", "8680", "5017", "3347", "41308", "41501", "55541", "5385", "57643", "43026", "17799", "34599", "18444", "30920", "28239", "33440", "13578", "49320", "56112", "54044", "23539", "12981", "46272", "25384", "44904", "43914", "29388", "17831", "23306", "23438", "11966", "5534", "44040", "6801", "16812", "7181", "35829", "55931", "33190", "56175", "56190", "14784", "8892", "5281", "10277", "51845", "7881", "985", "19833", "3463", "19954", "12345", "38712", "48271", "44499", "1827", "39712", "58398", "18577", "26502", "31215", "56630", "53081", "28328", "24247", "22369", "30419", "51752", "41760", "54812", "59269", "8012", "48873", "32092", "14480", "25168", "12232", "24539", "30671", "36173", "38094", "24003", "58773", "53371", "46253", "59494", "5798", "46231", "32190", "14943", "40562", "7998", "23840", "32708", "56271", "11053", "20181", "30100", "56550", "44897", "22319", "53508", "50521", "7804", "39711", "217", "7512", "19411", "47394", "5355", "41179", "50877", "52491", "3408", "38944", "59431", "20299", "57095", "58677", "29986", "7292", "30707", "18304", "54839", "51588", "51962", "22500", "48267", "12842", "47348", "27010", "28191", "32914", "14057", "39767", "53579", "3912", "30837", "30867", "49482", "40516", "20023", "20979", "34719", "59671", "38878", "59746", "25146", "4070", "19119", "14240", "28351", "35964", "45759", "28099", "28269", "12707", "18251", "26415", "20552", "30772", "3679", "20548", "25721", "17626", "6034", "43512", "55718", "30267", "43769", "31892", "18774", "2468", "41673", "17708", "25917", "133", "41685", "30820", "18600", "41652", "1705", "38604", "54844", "15835", "47593", "47168", "31243", "5224", "49605", "50275", "5684", "25794", "53322", "36280", "11963", "25670", "31731", "54843", "45225", "24835", "52140", "18640", "28289", "14438", "28336", "23283", "55480", "1131", "32012", "32112", "46648", "4453", "7693", "24715", "43292", "51673", "38474", "18299", "15754", "25292", "1786", "31148", "14684", "790", "33330", "15538", "33361", "6491", "45237", "2609", "2333", "58334", "9825", "11290", "14038", "2227", "13688", "29138", "32790", "42811", "49550", "43403", "59926", "55425", "35855", "55233", "19622", "9295", "37885", "14850", "51682", "12518", "21820", "33639", "16287", "50713", "53338", "7239", "47187", "44365", "22367", "20120", "51726", "13923", "41005", "4415", "11091", "10030", "22344", "26873", "50969", "7062", "3375", "31624", "31099", "23327", "54055", "11188", "29966", "25392", "16818", "7913", "23533", "8233", "34810", "46698", "25809", "8681", "24119", "32142", "53084", "4040", "16399", "49458", "50868", "54004", "4960", "37759", "56289", "12437", "4941", "50433", "34081", "56921", "50055", "25804", "13127", "40758", "13695", "32651", "50272", "51546", "45578", "33505", "43269", "18414", "44886", "47149", "20245", "48499", "10256", "56218", "4606", "1746", "5042", "57050", "3849", "47450", "8343", "47531", "48265", "26363", "46066", "1354", "52221", "23747", "20993", "9036", "7864", "58802", "41435", "41978", "16092", "17544", "13143", "53214", "10798", "42008", "52042", "28466", "20801", "47987", "35763", "51134", "48870", "40326", "39155", "23622", "22847", "40977", "42189", "41951", "56681", "16916", "30169", "46011", "37375", "24670", "14876", "36609", "41274", "17982", "26420", "44251", "37382", "57700", "53555", "4507", "10913", "50933", "9542", "35922", "46677", "46014", "42240", "57099", "29919", "56254", "2744", "17693", "14409", "45828", "16706", "2252", "20567", "41947", "8401", "3376", "7037", "23610", "10320", "42492", "19284", "30535", "22238", "57725", "35033", "13881", "57147", "49993", "7964", "10029", "9965", "56969", "33878", "53788", "53915", "7633", "37856", "12662", "46538", "22019", "11559", "27083", "11826", "56545", "50513", "36407", "20992", "27750", "23663", "4912", "4726", "7934", "14726", "3222", "19921", "32174", "52157", "34548", "25213", "39066", "51454", "48908", "48563", "40042", "59230", "27800", "12482", "15374", "38594", "14056", "45585", "24357", "8181", "43333", "37990", "50365", "15357", "1269", "37241", "10858", "5630", "21978", "19618", "56342", "7279", "28464", "5003", "22185", "10774", "53588", "54230", "30880", "6287", "45362", "41595", "42511", "43545", "33411", "48993", "13341", "39651", "22960", "43660", "4362", "35897", "59457", "665", "26526", "57178", "6405", "17237", "23450", "48014", "41766", "55925", "13902", "58729", "5579", "12026", "43151", "52186", "31413", "35824", "32034", "4676", "33290", "37301", "24789", "9230", "20064", "7600", "10216", "44733", "43984", "51930", "35949", "7573", "47637", "20763", "45572", "11370", "53624", "9589", "52204", "51189", "41496", "16880", "28422", "43473", "49008", "24008", "24185", "51516", "59655", "51077", "50592", "42293", "23399", "18328", "30542", "59034", "53261", "39707", "13122", "30815", "51564", "5949", "25000", "4261", "58503", "34517", "19071", "43846", "26938", "27122", "49673", "24270", "47745", "37755", "13322", "4890", "3012", "40718", "8080", "30448", "40650", "36639", "17378", "56587", "55152", "46769", "3335", "28717", "51674", "23211", "29539", "54139", "42597", "7916", "52447", "7862", "38071", "1002", "4782", "34232", "3572", "9246", "56876", "24320", "28059", "31298", "5772", "51466", "8496", "14335", "37021", "36068", "33989", "46646", "50102", "12142", "13310", "49156", "14614", "50014", "33572", "2727", "55361", "31487", "55645", "29953", "42080", "37662", "53414", "49944", "26590", "51267", "13678", "53172", "12109", "26495", "1470", "3796", "40872", "58075", "22202", "29271", "37831", "31748", "24225", "58273", "20374", "9259", "30781", "635", "475", "39587", "12989", "18532", "10814", "37609", "18994", "4763", "56361", "51158", "3034", "30601", "12535", "2820", "31768", "49017", "48973", "50322", "58972", "7541", "40581", "3590", "16833", "50904", "16215", "28744", "2551", "55980", "24769", "324", "23131", "1901", "37900", "8429", "25914", "34571", "52692", "47525", "35303", "51716", "36430", "17157", "41019", "41454", "49202", "38674", "55299", "18010", "33754", "46308", "6345", "10299", "27363", "25561", "1894", "56546", "11593", "24843", "6795", "15321", "44431", "47253", "8872", "16163", "31113", "51151", "1945", "27652", "30244", "6315", "46836", "59613", "17181", "4779", "42135", "17152", "31233", "55498", "35842", "24844", "43973", "17189", "57583", "2471", "28704", "19542", "4295", "3826", "9107", "30311", "58", "52815", "57604", "32685", "23920", "3667", "56963", "54467", "36201", "35664", "55289", "27070", "4311", "5218", "9345", "16596", "20121", "49293", "14591", "26245", "31943", "59003", "12585", "13679", "5411", "37802", "12102", "8092", "37176", "4670", "6606", "3616", "12279", "36092", "6146", "7831", "10688", "13377", "56868", "4867", "29196", "58836", "40841", "1274", "49752", "22012", "53918", "57667", "8177", "46062", "41709", "42883", "56256", "10548", "53271", "41075", "33775", "40538", "12820", "18857", "39398", "458", "30833", "17523", "32643", "134", "36956", "29951", "50420", "6031", "6029", "33399", "39642", "8184", "42104", "5515", "8589", "10652", "3757", "14452", "36412", "20924", "3301", "51482", "8499", "33099", "59830", "13309", "7346", "32063", "13141", "58270", "51387", "30605", "49562", "10102", "55784", "51207", "12474", "5517", "52724", "54622", "1631", "7059", "56706", "38245", "59397", "33744", "56645", "12956", "22719", "9231", "35944", "37854", "36053", "18311", "16096", "11938", "8898", "1344", "55532", "10880", "17504", "26701", "29571", "21902", "1649", "30098", "35877", "17391", "10523", "35069", "10755", "8803", "38752", "50133", "2751", "56840", "15531", "44987", "43592", "33289", "28918", "52146", "1781", "27399", "48796", "18574", "8097", "886", "47334", "39220", "56498", "53027", "58039", "36333", "39889", "9469", "5072", "4889", "27170", "6353", "40674", "12172", "40014", "33863", "10347", "32674", "40759", "48392", "24862", "54927", "39407", "10218", "36386", "4090", "23140", "53658", "54436", "25820", "33096", "18729", "16017", "35737", "45126", "5633", "14638", "54492", "56152", "33848", "4508", "52882", "34589", "37692", "5588", "44280", "56643", "3379", "17481", "14063", "49854", "40742", "37044", "33517", "32223", "13550", "12254", "23154", "556", "42668", "11040", "28761", "25673", "27516", "47235", "56359", "46181", "4285", "35550", "43494", "32595", "40511", "16417", "47520", "46003", "43444", "20829", "2736", "2922", "2635", "28866", "15654", "4523", "31628", "31337", "18159", "42697", "11816", "3441", "26980", "30729", "31647", "23307", "39842", "49840", "12440", "19051", "2930", "31291", "47445", "8895", "26533", "8695", "32178", "37893", "746", "17618", "10749", "49900", "24362", "34497", "20534", "50212", "31929", "46047", "13780", "11944", "34725", "58969", "17752", "3925", "43788", "38479", "57445", "56687", "3874", "7859", "48454", "9950", "46522", "49379", "29160", "49236", "59087", "25076", "4630", "10663", "15431", "13607", "54872", "127", "24725", "56327", "59318", "37002", "25435", "50749", "129", "49493", "12462", "13868", "45707", "45043", "20077", "36968", "20304", "57780", "50556", "16622", "37952", "31325", "51485", "8046", "24850", "32164", "21941", "39017", "30401", "24096", "36187", "5420", "28786", "12038", "40196", "57052", "12228", "14008", "54777", "55738", "29595", "1197", "57996", "406", "19879", "29561", "38653", "35738", "56703", "27699", "39530", "5310", "23788", "13633", "10469", "14589", "59414", "44717", "38562", "49804", "25465", "54963", "40929", "13954", "52124", "54836", "28666", "51932", "7905", "32078", "2903", "15124", "28300", "22551", "22548", "23578", "38295", "50990", "54855", "41203", "37123", "32000", "51421", "29202", "9751", "19067", "38627", "59960", "34807", "36063", "39263", "21096", "30861", "54678", "31190", "31296", "8049", "31409", "28997", "49987", "43229", "31696", "29363", "23146", "3207", "56538", "41557", "23454", "31846", "28179", "20520", "43809", "22767", "58857", "17934", "38574", "21990", "34400", "22770", "54261", "15546", "33006", "55010", "52177", "53150", "15965", "1113", "28308", "46234", "35827", "36594", "49801", "930", "54709", "52553", "32510", "11328", "29894", "46621", "6268", "11254", "1778", "1430", "3219", "10163", "37407", "45517", "41678", "17734", "26234", "50988", "13534", "34972", "15781", "9477", "59885", "54880", "43232", "36006", "18679", "46608", "34340", "52246", "15070", "51226", "26009", "11940", "44270", "38818", "55978", "15812", "33012", "22395", "56473", "15974", "18585", "8775", "6028", "17667", "10014", "3795", "9103", "25918", "19154", "15182", "6261", "19172", "41697", "15951", "55675", "9445", "6347", "9143", "23869", "25717", "34431", "1158", "56192", "52269", "54300", "40244", "3881", "6776", "55184", "40899", "2913", "48389", "10129", "34922", "50501", "7770", "33802", "19048", "41437", "43797", "30327", "3189", "47260", "39113", "30967", "43775", "29229", "42629", "40614", "57862", "10829", "37716", "1636", "42877", "1206", "59250", "54512", "47362", "24382", "40783", "4737", "10598", "35481", "12417", "51649", "31710", "53522", "182", "1800", "19714", "34513", "12549", "17066", "23019", "5649", "8465", "28325", "1739", "41421", "17435", "32032", "56015", "58888", "39773", "25943", "17274", "9988", "17677", "20289", "39153", "29772", "13371", "2768", "6181", "20985", "30692", "34635", "29650", "20491", "43042", "35426", "27766", "40981", "54793", "6763", "47584", "24681", "35199", "50941", "38872", "48438", "37650", "39460", "13451", "27607", "24191", "53352", "19093", "57221", "43251", "50870", "9474", "8922", "35655", "32170", "35791", "50899", "59361", "39855", "41410", "36664", "17225", "38983", "17771", "27052", "57316", "58032", "5999", "52403", "19386", "47543", "35642", "5702", "44019", "17417", "17396", "45837", "54293", "38286", "29453", "50326", "4013", "52794", "50997", "4984", "10394", "17925", "34953", "24281", "50657", "2888", "20681", "52796", "38330", "15950", "13929", "22632", "42347", "46261", "49410", "55179", "47658", "37803", "58184", "21745", "11739", "20729", "31807", "30081", "27714", "13627", "15853", "56554", "37075", "39878", "29755", "5995", "9859", "36293", "8620", "17650", "45817", "1401", "55676", "59449", "5428", "6087", "18933", "18061", "12446", "41986", "50312", "28550", "59939", "35508", "12698", "18858", "49315", "43812", "15453", "30660", "53257", "36502", "58378", "43334", "53839", "50613", "35089", "18957", "27337", "26377", "36446", "52160", "10399", "39972", "9579", "53003", "41295", "14045", "42654", "44060", "8583", "22377", "15084", "12240", "1048", "53205", "28087", "11676", "22781", "57469", "19977", "33372", "26793", "752", "24373", "48580", "44640", "5685", "8366", "54061", "5398", "37535", "58087", "52917", "146", "39669", "15328", "21144", "22248", "32666", "48616", "44780", "28029", "54394", "32447", "3914", "45903", "45676", "12735", "52685", "40532", "5627", "1257", "18819", "37095", "13315", "52409", "54857", "56054", "28691", "41380", "58962", "37247", "57396", "11807", "58615", "10079", "27139", "56532", "49229", "47455", "24361", "16432", "7959", "31400", "58860", "25370", "2341", "59253", "13448", "50267", "55395", "21816", "7727", "23178", "1025", "41708", "11037", "37421", "40213", "46825", "54754", "54488", "29227", "56290", "22208", "25545", "58928", "42764", "7796", "34692", "48650", "13636", "41876", "12011", "45741", "35287", "30486", "27985", "59459", "10460", "30635", "32839", "51857", "39344", "51620", "21943", "10257", "47549", "703", "51627", "53657", "9406", "8114", "52626", "6205", "51577", "1240", "33470", "49617", "56723", "48222", "29462", "55686", "49191", "7731", "31632", "48379", "21945", "59574", "14362", "19441", "50206", "24781", "25970", "30055", "19369", "43318", "4374", "9326", "16098", "58107", "55856", "52078", "21700", "39661", "47197", "11115", "51867", "54100", "49824", "28343", "19183", "18602", "48063", "57703", "34940", "38713", "854", "28874", "9097", "25662", "25891", "934", "57111", "41431", "42724", "9973", "26907", "1633", "41475", "39081", "55441", "38636", "27743", "39244", "23151", "58055", "52608", "51883", "10615", "9628", "48939", "2092", "9167", "1372", "47600", "45976", "45994", "50607", "35314", "27555", "21602", "57985", "23722", "6293", "1934", "52969", "15101", "10714", "11988", "9694", "10176", "12944", "59164", "16495", "31042", "54473", "7763", "10928", "46018", "22583", "12760", "57036", "16559", "25862", "38897", "14492", "9148", "47485", "5467", "45171", "30539", "34065", "23645", "43527", "19757", "43514", "25637", "3196", "30826", "17331", "3088", "12444", "55202", "42940", "9087", "25359", "52094", "12288", "8971", "22320", "49481", "56064", "13050", "49446", "57748", "15066", "19244", "5536", "46886", "7970", "42360", "19409", "47771", "5686", "41427", "21315", "58560", "53028", "26231", "50937", "57065", "13393", "42851", "10427", "39838", "24037", "38589", "2386", "37409", "11778", "31585", "26093", "15472", "32896", "11388", "55132", "32997", "16346", "7291", "15318", "20749", "11521", "18596", "8573", "12720", "32725", "13124", "28530", "39471", "53184", "26591", "46230", "39426", "32047", "37454", "35179", "7887", "39406", "40092", "35407", "34913", "29632", "33819", "2213", "53783", "53944", "27175", "3775", "18475", "36414", "38890", "39608", "55725", "26021", "12117", "43806", "29658", "19103", "42428", "34997", "50346", "47340", "51684", "59554", "14216", "44465", "15280", "5899", "13263", "15003", "34273", "56095", "51173", "9139", "54924", "57256", "7477", "137", "2261", "32888", "36748", "38756", "10304", "18136", "26941", "2318", "39930", "39410", "25852", "9360", "35554", "13097", "24525", "28115", "38430", "12300", "20590", "51057", "52967", "43811", "17948", "12143", "18895", "58591", "40549", "25691", "52392", "24898", "36450", "28181", "3412", "34665", "6786", "46474", "29943", "59297", "51680", "3893", "34991", "53057", "2016", "13716", "34196", "40980", "59263", "21400", "603", "54280", "29905", "43118", "50964", "59882", "26137", "18621", "56471", "19633", "11707", "50012", "18875", "33597", "46357", "8909", "41251", "27057", "40381", "48775", "52163", "59590", "40258", "46949", "49697", "18283", "40335", "28583", "19707", "16380", "5822", "30381", "2250", "2883", "6807", "27519", "45619", "33076", "4839", "52779", "39021", "12964", "28175", "17808", "199", "25294", "34321", "47082", "20663", "166", "53585", "32376", "28458", "892", "13973", "12609", "9096", "9591", "8247", "38371", "4280", "53841", "6211", "36583", "21091", "2932", "44778", "4348", "20355", "12434", "51292", "52563", "24796", "42091", "24498", "49565", "18031", "15073", "52918", "54849", "43535", "7010", "39785", "9461", "46512", "30780", "15704", "8941", "51760", "18971", "33436", "9656", "3247", "23350", "50571", "1857", "15140", "49006", "10321", "12450", "38292", "11404", "30696", "3181", "39549", "48836", "30741", "21095", "13852", "7193", "23731", "3641", "20928", "10539", "25501", "42341", "13336", "20931", "26039", "33403", "4868", "25648", "44879", "30089", "58845", "37832", "49508", "33594", "56478", "8521", "42048", "38402", "52857", "7700", "40206", "54741", "52666", "28053", "47702", "53332", "20847", "6438", "51479", "22060", "50986", "6829", "23050", "11813", "32981", "42718", "55810", "37709", "30436", "37198", "37661", "23431", "21258", "5356", "38605", "27150", "23961", "46712", "2361", "9121", "48015", "16999", "21053", "44346", "11931", "29868", "40994", "24063", "47194", "39149", "30435", "18164", "40328", "18716", "42544", "32425", "39199", "1699", "13484", "42630", "24745", "18859", "38388", "15523", "21470", "57597", "45735", "36658", "40002", "51020", "39527", "20326", "24824", "17385", "23058", "6551", "40086", "1661", "7363", "23208", "32093", "11865", "33195", "12920", "22133", "49502", "34955", "4490", "34149", "35202", "49409", "38341", "9828", "12139", "49723", "6777", "4761", "33364", "52061", "9229", "23886", "4071", "23344", "30186", "2159", "46732", "43564", "20478", "20300", "50827", "27142", "8391", "52439", "15521", "50676", "32492", "91", "8854", "51551", "48661", "42923", "48912", "31094", "52701", "16629", "52375", "18401", "10317", "51697", "37534", "45433", "12267", "56355", "25120", "34007", "54657", "52514", "53404", "28326", "58568", "34715", "15510", "31017", "1957", "30843", "55622", "13014", "49896", "37440", "31457", "24249", "35511", "45724", "54814", "55066", "7460", "16291", "16321", "8123", "50534", "3727", "46006", "20041", "38419", "28516", "24215", "58934", "14736", "52880", "51118", "13165", "9612", "13655", "20417", "33683", "35127", "14193", "13735", "39815", "20260", "3409", "45422", "15671", "42033", "10576", "47308", "16929", "36528", "19376", "26260", "3607", "33223", "27720", "43272", "59769", "4498", "31932", "59042", "5677", "13668", "43340", "7671", "6414", "58223", "46310", "16942", "21287", "7828", "21005", "33170", "3322", "41871", "18783", "52630", "12456", "35859", "49077", "21409", "36073", "40020", "13346", "55200", "42363", "49144", "35121", "43669", "26063", "19173", "46035", "38181", "38399", "9917", "57504", "35869", "13729", "4656", "53676", "37294", "36917", "54811", "30935", "31570", "25833", "53523", "56216", "41370", "35887", "4366", "22959", "33018", "37015", "41715", "1643", "8439", "425", "52826", "30128", "50638", "22311", "41325", "56865", "59908", "43543", "53194", "5537", "30685", "49424", "13512", "28964", "29018", "59726", "23167", "57601", "42303", "13232", "58534", "17914", "32100", "57712", "23860", "15498", "888", "7713", "2328", "15979", "43299", "15452", "59056", "37193", "12711", "10980", "45157", "9775", "22566", "35150", "31577", "53504", "23482", "14905", "6175", "28673", "11386", "5242", "24979", "56756", "58766", "39631", "2568", "18085", "42772", "20229", "19626", "19709", "13196", "26637", "8596", "44137", "43692", "12845", "46039", "45475", "47402", "32913", "30882", "28150", "44239", "46707", "23736", "7762", "25517", "52114", "47587", "45753", "48667", "26876", "22528", "17101", "47114", "1152", "57540", "20255", "9563", "46263", "38803", "27028", "32020", "44690", "44197", "34887", "51882", "21352", "13582", "34664", "21231", "17093", "8770", "14669", "3140", "17729", "39306", "10824", "23026", "53826", "39308", "8369", "53274", "2746", "30091", "25192", "52819", "18669", "46147", "7587", "58349", "19157", "48173", "23018", "13502", "8717", "33307", "42408", "39138", "28316", "24975", "25484", "23925", "3722", "21332", "10564", "43237", "37312", "57704", "11918", "2032", "20197", "40369", "10700", "48693", "20141", "48833", "56686", "44810", "16143", "43052", "56248", "10205", "49742", "43850", "55343", "42034", "55404", "27043", "10842", "27913", "51147", "14888", "12999", "35119", "55247", "28897", "30919", "44412", "4521", "26786", "30120", "37348", "47295", "38334", "37950", "20721", "56863", "21676", "44385", "39968", "9458", "45045", "17717", "2067", "48774", "36395", "18119", "3759", "12302", "7236", "53482", "24006", "51060", "53509", "42120", "47993", "30353", "58572", "49985", "36520", "41841", "53935", "12715", "59155", "29832", "31245", "51876", "56772", "16316", "53771", "28642", "37146", "29220", "23563", "55963", "22210", "2087", "57029", "59838", "25486", "53278", "1091", "36231", "4431", "30134", "58226", "21634", "47135", "28260", "26000", "26241", "7893", "34315", "50888", "44530", "45939", "20196", "35993", "50545", "57317", "57560", "23908", "38208", "10260", "28994", "58701", "52774", "44374", "51900", "38679", "49019", "11542", "49164", "16211", "32858", "11155", "41718", "35925", "17726", "59410", "23728", "13912", "39089", "51647", "28013", "43616", "9521", "1378", "16633", "27678", "33345", "17834", "8561", "3972", "138", "42515", "40845", "20199", "23592", "42059", "55443", "6642", "59282", "24447", "39316", "37060", "58071", "40132", "13075", "20481", "18873", "17874", "57104", "52926", "7480", "44964", "20159", "55354", "20707", "44921", "20291", "12985", "17319", "30432", "42839", "51623", "19732", "33587", "15330", "2763", "30818", "15201", "34024", "51037", "44112", "38977", "22099", "36301", "15540", "26613", "14432", "41181", "14473", "30761", "59540", "1022", "6737", "276", "27213", "45266", "36646", "23148", "22720", "1167", "17062", "4378", "43296", "30959", "18035", "56900", "21500", "22855", "1382", "27326", "56811", "30829", "42142", "47891", "48824", "47483", "44636", "40682", "9860", "11314", "21009", "18546", "39000", "43033", "18987", "35968", "16240", "19674", "41966", "38308", "36202", "30319", "3651", "29681", "35645", "5336", "11481", "46056", "43006", "23268", "46579", "17264", "29814", "46339", "57142", "58479", "3562", "23182", "24534", "53376", "57798", "48760", "6209", "24686", "1464", "42219", "36410", "3753", "41991", "45679", "695", "46548", "12118", "11044", "28154", "25234", "3549", "48450", "54253", "34372", "1170", "3974", "22090", "18573", "16299", "7901", "59112", "34479", "2370", "3630", "39127", "26565", "54773", "19819", "57332", "39227", "31626", "15826", "32691", "17566", "35076", "44751", "58996", "12101", "55983", "9159", "51007", "2898", "22743", "48512", "21518", "36698", "57238", "29293", "56464", "28226", "7628", "28122", "11288", "41322", "16688", "36551", "31097", "16165", "21356", "52105", "55869", "3216", "38066", "41500", "4195", "27591", "53602", "1396", "48883", "33960", "22999", "33943", "57269", "16370", "59435", "52092", "10731", "48778", "31402", "48866", "9165", "46290", "26117", "55635", "55953", "6900", "21913", "51294", "21584", "54770", "29016", "53405", "11007", "31994", "6843", "53937", "12859", "30945", "50683", "7060", "48526", "48294", "56169", "33130", "44372", "41589", "6975", "11086", "20679", "17227", "14381", "35875", "42591", "21926", "16727", "3270", "919", "15271", "59900", "22128", "29427", "59608", "32833", "7525", "4729", "12090", "52324", "54646", "9525", "27585", "42813", "49791", "19841", "28344", "9901", "21063", "55216", "43971", "21142", "50691", "27936", "37192", "21219", "1065", "57027", "50408", "26433", "21987", "35392", "46328", "35672", "39503", "33045", "34051", "13240", "46800", "40161", "42861", "59381", "7991", "28081", "28413", "20167", "30385", "27038", "23088", "267", "39718", "9352", "35954", "24325", "46374", "8260", "7738", "12445", "2636", "46995", "47890", "29482", "47409", "20902", "6640", "53267", "14324", "4548", "18148", "766", "45052", "20908", "2790", "23868", "42854", "40973", "38550", "18349", "23364", "30111", "6720", "57513", "51517", "24674", "34926", "13611", "30637", "29457", "44162", "52574", "28235", "44041", "52219", "5590", "49407", "15991", "33028", "4925", "55751", "30841", "34279", "17150", "30377", "15443", "14058", "42859", "25690", "13497", "57477", "18588", "56494", "56823", "19887", "23718", "28746", "39395", "38854", "9814", "15389", "21379", "10801", "13594", "17510", "38175", "3272", "41528", "27094", "30102", "18620", "47422", "52616", "48386", "26670", "6051", "34656", "11780", "57682", "49394", "27581", "7657", "54956", "38611", "29422", "15052", "30019", "14349", "31552", "30192", "78", "8692", "36722", "58139", "51725", "25162", "47494", "46748", "39840", "5524", "26646", "50766", "8685", "20103", "25795", "55548", "46115", "37173", "43339", "30215", "14018", "14395", "25528", "45601", "30909", "26931", "757", "32197", "56144", "29735", "58702", "157", "49270", "40920", "16384", "1398", "52592", "41703", "1454", "45358", "37373", "43721", "31559", "33569", "5819", "31511", "59035", "21399", "18350", "49430", "15305", "24581", "41392", "18944", "11722", "18961", "52981", "47788", "5883", "40999", "13039", "41106", "57662", "17103", "36094", "16379", "22011", "10085", "49022", "42790", "52374", "44130", "1277", "18354", "11603", "12610", "32233", "27945", "29281", "36599", "29326", "20276", "25297", "42627", "13649", "31273", "25335", "5549", "7511", "4215", "34382", "40836", "48605", "55165", "26401", "56363", "1655", "49710", "3337", "52944", "15515", "32079", "41351", "12368", "70", "57629", "32050", "55031", "35813", "42379", "1810", "19668", "14549", "47276", "42516", "15448", "53108", "22880", "50880", "35147", "32659", "26098", "21270", "41189", "901", "35082", "3465", "6821", "37814", "29849", "43079", "59226", "49842", "22843", "8570", "35543", "31121", "52175", "26033", "31915", "9238", "9816", "2089", "4452", "10850", "31535", "37518", "17121", "8831", "23906", "15046", "13672", "44622", "23638", "38869", "17233", "3016", "2558", "25613", "1089", "25950", "26624", "42475", "50702", "2681", "12883", "32169", "14330", "5299", "9226", "10647", "29341", "42751", "52653", "50130", "56328", "14463", "21663", "9035", "15387", "52189", "54825", "50340", "50010", "52889", "45124", "238", "33396", "23900", "19685", "9704", "36602", "52754", "52302", "53366", "45757", "8236", "53006", "57893", "36601", "1663", "18033", "7121", "10782", "7342", "13581", "51476", "7568", "21042", "40550", "11293", "13997", "5155", "12931", "20219", "28818", "1231", "13944", "38191", "34428", "54746", "1680", "29245", "22691", "6397", "19881", "3311", "2597", "56167", "23699", "43471", "30520", "7322", "17845", "59616", "15723", "52789", "30545", "57276", "27158", "8634", "244", "52011", "12333", "39077", "28000", "45577", "17256", "27805", "10302", "32230", "10908", "19878", "37325", "54723", "41706", "15348", "13781", "33048", "12962", "44331", "28610", "56369", "14968", "12239", "8522", "43060", "24110", "56813", "20849", "55435", "14359", "2659", "57706", "34255", "53163", "37643", "9369", "26194", "23280", "13190", "19431", "15739", "43089", "13188", "5167", "33147", "38770", "20472", "35621", "59843", "4460", "18386", "2021", "19698", "58353", "41669", "15888", "36313", "22214", "13981", "55140", "43088", "15566", "21076", "3152", "15807", "55882", "36652", "46528", "4909", "10991", "57848", "1062", "31714", "24104", "47962", "23513", "26012", "21012", "44826", "18264", "10072", "52184", "9653", "35103", "39987", "7593", "10627", "17183", "57804", "11587", "10458", "59913", "53811", "2128", "40157", "17470", "12832", "10972", "5861", "33959", "43111", "52227", "57634", "35025", "22015", "1976", "1182", "56679", "43982", "16637", "1591", "3673", "26314", "3831", "44861", "30203", "50368", "42661", "18956", "57076", "11960", "16661", "47040", "15292", "11625", "34498", "28208", "18765", "2368", "48041", "50731", "11102", "21677", "11505", "48065", "28992", "45808", "40425", "26584", "37226", "34102", "35350", "5275", "26663", "1762", "13391", "5091", "26424", "4381", "45489", "3486", "40827", "6311", "22592", "53128", "28887", "28120", "28690", "38244", "21436", "19202", "56379", "44703", "37340", "8390", "8503", "55932", "59320", "4233", "58797", "21903", "32403", "51602", "57606", "42998", "45744", "940", "37206", "29144", "25875", "36959", "8308", "31908", "40168", "25903", "52337", "16790", "42641", "8238", "24693", "21339", "10739", "46660", "23589", "28644", "47093", "8846", "4906", "51376", "6137", "216", "45421", "45800", "58648", "11169", "57344", "58138", "2552", "50776", "8859", "7249", "43146", "37664", "11890", "36479", "59370", "50953", "12881", "43618", "46046", "14727", "595", "41023", "55790", "2348", "4557", "9832", "17011", "34151", "37104", "38406", "28738", "19713", "35618", "45019", "27283", "15380", "18274", "3099", "6511", "15767", "14829", "19323", "2498", "8277", "58654", "33841", "45954", "6096", "37697", "53020", "16016", "42292", "27793", "20752", "58592", "1371", "39908", "44326", "40597", "53388", "25252", "6567", "3285", "37906", "18626", "42822", "59594", "27672", "56698", "59008", "8658", "5158", "29829", "1495", "53925", "24462", "9800", "45813", "54575", "23124", "44015", "52255", "48111", "20666", "29190", "35745", "27518", "9851", "13208", "42015", "42572", "40638", "37674", "25874", "39196", "48516", "40073", "46169", "11077", "15446", "11530", "35725", "9607", "5722", "37133", "44354", "31486", "2150", "7183", "54067", "5638", "10039", "25496", "28219", "44885", "38545", "46611", "32905", "26639", "21765", "9786", "46304", "47267", "3917", "1121", "8450", "2803", "1935", "30722", "13740", "6453", "22814", "11025", "49771", "31708", "43238", "54349", "22864", "18422", "8676", "15408", "2221", "41126", "36097", "21540", "55376", "41265", "48025", "44439", "59863", "56429", "19754", "32518", "10389", "13974", "54568", "3691", "17676", "48408", "37621", "42969", "40342", "47566", "248", "3497", "4058", "7485", "49656", "53456", "7646", "38469", "56001", "39224", "52174", "30918", "39537", "48094", "47917", "22741", "56315", "4810", "15969", "32808", "10489", "3870", "35792", "29881", "55791", "27500", "43365", "9436", "25815", "37775", "38057", "50863", "43101", "46210", "37459", "2037", "36774", "29898", "31034", "47533", "7578", "21675", "45953", "50922", "13588", "30544", "14159", "50632", "59860", "38547", "20974", "47292", "56183", "11197", "4270", "50867", "10552", "49953", "27364", "8282", "7395", "28425", "2534", "42332", "24482", "9426", "19347", "34732", "37965", "58958", "17516", "12427", "28968", "7841", "20875", "35457", "29440", "34696", "39362", "3163", "28473", "10448", "41343", "9529", "486", "59106", "50516", "905", "12568", "39428", "16908", "19252", "29998", "30302", "56702", "42992", "24044", "9028", "41751", "52634", "41906", "56569", "1806", "15942", "59741", "32798", "20381", "29497", "52658", "50601", "9285", "42245", "51634", "37515", "28963", "15481", "12571", "18469", "30118", "19796", "50586", "44328", "15169", "35998", "53893", "23776", "22014", "50512", "16398", "6797", "12499", "22398", "44952", "24206", "18491", "48901", "26208", "6868", "43999", "43075", "32945", "1156", "8980", "32513", "30655", "42328", "25109", "49128", "45642", "40772", "13456", "4296", "20063", "33417", "23108", "21496", "17774", "23711", "35644", "44216", "5203", "39035", "13566", "22178", "43931", "11953", "59278", "30588", "57083", "8490", "47007", "29972", "13452", "57325", "11545", "38622", "32594", "32501", "40957", "6412", "52733", "27967", "41411", "14579", "23913", "33462", "34622", "33120", "37428", "49242", "24235", "34090", "54841", "9700", "24877", "50568", "55262", "34629", "16820", "14099", "59356", "57189", "12983", "15957", "38810", "24181", "5675", "35116", "18767", "29112", "45107", "51918", "1432", "56948", "38183", "52983", "46542", "58911", "24732", "33691", "23299", "39735", "47439", "12212", "11728", "10612", "17295", "34145", "4694", "56417", "16225", "58421", "56466", "33166", "57002", "48159", "43743", "8944", "33412", "57220", "39805", "17168", "32499", "36107", "37199", "46883", "54404", "21835", "23468", "20341", "56438", "32068", "22550", "1209", "16877", "20837", "58594", "38152", "23261", "28320", "17136", "2490", "42143", "46655", "1069", "41892", "25725", "46197", "21394", "47691", "47221", "57341", "13880", "4860", "31143", "34399", "12751", "7595", "39179", "23562", "58454", "38655", "32908", "17800", "54601", "5119", "53797", "15064", "21311", "54235", "27861", "43472", "30531", "21130", "46505", "22205", "55586", "19728", "12770", "5717", "31199", "24046", "35754", "25415", "18410", "33537", "56632", "32229", "12667", "36591", "10967", "37623", "28093", "40120", "10835", "27029", "26299", "40684", "3904", "9900", "33338", "46849", "47874", "45423", "46794", "5729", "49250", "52885", "41447", "2634", "29052", "53873", "9235", "48840", "13217", "7472", "59723", "24218", "11399", "40889", "13915", "53836", "34487", "16854", "16243", "19582", "33278", "54284", "48171", "17063", "41512", "5136", "29484", "34100", "55641", "32350", "3481", "41974", "15856", "56062", "33519", "27858", "54255", "26293", "56572", "27486", "39567", "36279", "48951", "48865", "40603", "34228", "57611", "42777", "15786", "44600", "48085", "23218", "19599", "47092", "42383", "44768", "45279", "9234", "10226", "39795", "10922", "40607", "14720", "35243", "22392", "54593", "41220", "39586", "24651", "10434", "52868", "5465", "38879", "9925", "6128", "20743", "59867", "18457", "33504", "37108", "5526", "43638", "41742", "7971", "21391", "54528", "36116", "23811", "43956", "38304", "25755", "6886", "19489", "11173", "13445", "45355", "37847", "32737", "19700", "17234", "3092", "33064", "17501", "3697", "52225", "32521", "4466", "7434", "16403", "55888", "42809", "33715", "2807", "34319", "44312", "26334", "856", "33004", "12913", "31931", "43095", "27394", "8420", "58044", "8039", "50236", "23476", "24260", "40275", "21544", "51834", "20001", "11148", "58708", "11483", "33782", "19454", "44575", "33845", "50011", "51136", "22264", "53283", "4083", "32554", "50057", "1124", "2989", "51896", "5958", "15661", "57461", "37114", "48487", "59614", "58778", "15161", "41926", "6157", "18882", "30491", "12930", "34590", "8354", "55471", "37737", "40354", "54638", "39246", "51153", "21692", "43741", "51395", "23219", "47940", "46786", "3805", "33697", "39941", "56835", "50661", "53965", "57681", "46397", "45147", "19975", "42676", "43176", "36771", "16411", "44588", "6762", "39911", "47696", "8075", "4176", "56556", "55412", "51202", "19390", "9222", "57373", "21916", "15454", "44369", "6407", "332", "19027", "41691", "17730", "55524", "40227", "53938", "29024", "10639", "41156", "2872", "24483", "22924", "43575", "32585", "21257", "28047", "52307", "57411", "15086", "43411", "20343", "57379", "9767", "22916", "22045", "14308", "16463", "42880", "34854", "4089", "54775", "40401", "3381", "46401", "43231", "92", "26963", "41908", "32478", "37841", "42506", "12461", "36298", "8983", "39376", "44433", "8322", "36336", "48561", "49980", "14421", "35422", "43687", "11649", "33584", "18175", "54808", "32971", "33161", "32413", "52836", "20166", "40170", "47689", "41284", "24296", "10328", "44397", "28616", "8221", "19736", "49986", "30022", "35065", "32211", "38465", "22305", "55183", "24398", "26572", "17922", "50044", "9106", "34186", "44096", "6305", "49778", "53673", "12775", "1477", "13139", "50176", "35894", "4115", "51481", "21473", "13724", "27433", "30369", "14256", "34277", "35403", "8466", "16597", "32132", "26220", "7355", "54734", "22773", "59982", "34793", "49154", "18936", "54665", "33973", "1794", "11161", "35948", "15044", "6446", "36808", "19885", "7466", "52923", "2229", "51052", "5881", "972", "5340", "37676", "43863", "56514", "53072", "53497", "10947", "15460", "20150", "26749", "54520", "36640", "56873", "54145", "5748", "41591", "26831", "45836", "22822", "40139", "20794", "12869", "17727", "25846", "23249", "40512", "2268", "47037", "11067", "7689", "45091", "58904", "54860", "3875", "53896", "59647", "54736", "49081", "21974", "33015", "51815", "42533", "22303", "6841", "30257", "46498", "31618", "13998", "59544", "58998", "17669", "13179", "52659", "57980", "52722", "51047", "52751", "51256", "49765", "35356", "53000", "35371", "37729", "763", "57348", "13714", "25802", "38958", "34795", "6313", "59221", "17941", "42770", "51727", "23781", "7054", "5001", "15681", "35633", "22819", "23924", "2640", "36071", "31781", "11398", "53823", "15109", "42023", "27748", "29948", "39759", "1073", "35927", "13282", "51917", "15765", "56321", "23630", "4063", "33398", "54700", "42987", "31466", "55445", "29667", "46479", "44790", "5007", "18562", "29285", "43061", "8647", "33794", "15596", "46409", "39193", "49615", "8903", "22048", "41808", "53369", "17506", "4509", "7865", "12922", "53709", "58956", "54291", "52086", "14613", "35175", "3015", "42744", "54976", "37505", "12169", "46076", "40485", "1946", "52298", "57167", "46211", "46508", "11592", "35026", "51437", "48841", "27452", "39478", "521", "27216", "16549", "44769", "31694", "51688", "10297", "5982", "41270", "18096", "37051", "47464", "57627", "52547", "24111", "34066", "49321", "19117", "7816", "24708", "3682", "1456", "27109", "53633", "52980", "14484", "8050", "10533", "34267", "50317", "13959", "25666", "54169", "58833", "10152", "40808", "25954", "12432", "8968", "17432", "34006", "1191", "30231", "228", "10482", "44888", "34814", "21015", "43876", "33129", "47344", "26450", "58495", "37200", "59632", "38855", "32199", "7499", "58125", "18982", "17638", "38816", "46664", "6062", "37984", "55160", "21328", "14055", "39710", "42793", "34930", "13086", "48368", "48255", "26808", "22047", "962", "55317", "39599", "16609", "25493", "53970", "18699", "34070", "47259", "22746", "25399", "1414", "33672", "42", "47115", "50231", "15761", "44133", "24800", "38433", "36926", "29312", "26107", "54438", "56295", "10863", "40280", "32326", "30314", "54384", "51000", "36466", "52170", "32809", "13492", "19484", "59902", "12837", "11912", "14071", "6686", "9959", "49418", "46962", "20305", "32612", "1351", "36711", "26145", "25249", "30453", "32104", "26823", "53329", "11168", "5582", "8970", "20206", "42438", "25254", "59856", "34600", "10268", "37166", "1639", "3401", "4208", "27171", "55977", "38665", "52975", "12606", "16878", "58165", "3857", "11438", "8921", "28385", "43226", "33752", "51056", "14559", "6545", "56185", "5386", "43352", "58545", "27988", "30412", "31688", "6857", "47590", "23813", "36449", "14651", "59287", "1312", "8001", "14062", "27367", "41798", "52735", "22935", "20090", "55672", "37654", "30736", "40933", "37048", "35847", "31856", "3834", "17758", "54973", "24834", "1296", "8520", "22940", "34453", "41303", "39529", "56000", "46701", "55378", "30174", "35043", "36287", "57940", "36870", "18137", "10285", "25228", "23890", "53015", "11755", "52963", "45400", "41461", "4715", "59334", "38320", "41916", "6317", "17090", "34385", "24941", "833", "53231", "3645", "32671", "18523", "41968", "3414", "30927", "41383", "23783", "13853", "6727", "2107", "52740", "9053", "58172", "41107", "24481", "49652", "24870", "17445", "16070", "2455", "31752", "14781", "1023", "18052", "2994", "14623", "56967", "22703", "8425", "56233", "36827", "24684", "25676", "8400", "22322", "55285", "24643", "12081", "3334", "23659", "53176", "21028", "34718", "52561", "34836", "38052", "24092", "28215", "42816", "33642", "43874", "42864", "12670", "57693", "32615", "17854", "32937", "18845", "2573", "30755", "40043", "16799", "27553", "18793", "36665", "29109", "7632", "50412", "22098", "58758", "37961", "31208", "42716", "46550", "32940", "22962", "3494", "47592", "34084", "18168", "50956", "41359", "31857", "44217", "50095", "19827", "8013", "44822", "48792", "33602", "57957", "34338", "3071", "35347", "50107", "45116", "39653", "41007", "14196", "8609", "27240", "22545", "52272", "15306", "10415", "2167", "41067", "39856", "19644", "18892", "35620", "6270", "44997", "53451", "12490", "18725", "34309", "1856", "18072", "58619", "27225", "6160", "11478", "53156", "11716", "55518", "2722", "44083", "11794", "48160", "53313", "1678", "53930", "31927", "27266", "56542", "56362", "48449", "22983", "46059", "41759", "46237", "650", "45933", "7557", "45842", "36122", "5137", "35360", "49137", "25226", "50297", "56376", "21664", "7631", "24610", "27722", "38746", "4724", "12991", "7429", "14208", "13314", "8254", "15163", "4123", "21368", "56294", "33165", "37846", "58900", "24228", "41027", "21410", "55843", "51724", "18768", "57449", "10670", "16494", "36138", "12900", "32416", "52120", "38758", "5967", "26901", "45017", "28577", "4434", "11889", "59060", "8157", "50298", "21581", "29593", "3865", "55170", "29778", "24404", "39194", "29662", "9438", "49391", "22101", "43712", "12583", "49659", "52703", "6671", "17835", "14798", "52788", "29613", "46174", "46", "59014", "50018", "48323", "56097", "36633", "46874", "52635", "40458", "958", "14665", "54433", "31107", "3783", "42390", "33831", "13911", "11108", "25184", "33303", "2124", "352", "53293", "10724", "12679", "5150", "14425", "33809", "45617", "30357", "51385", "53886", "35126", "39552", "35088", "42524", "23942", "1970", "16710", "6292", "33615", "5733", "14181", "56849", "13033", "21756", "9555", "49864", "58289", "25551", "20320", "59534", "2877", "10981", "58906", "2206", "12822", "36045", "38105", "36256", "5342", "56586", "36981", "20897", "11239", "53950", "51536", "5732", "53804", "44749", "30464", "7336", "8615", "6930", "24736", "29391", "4350", "7610", "25857", "18466", "4217", "21195", "46100", "23643", "9423", "15994", "8689", "19110", "23799", "49830", "8993", "59427", "19198", "39451", "30911", "48481", "42155", "3949", "43532", "25744", "18548", "42954", "25368", "27334", "59220", "40059", "9780", "17541", "7642", "31818", "29213", "20627", "40264", "27079", "39814", "39145", "26190", "13772", "25975", "58940", "12527", "17697", "1108", "46183", "57982", "31614", "20072", "41143", "34547", "54181", "55264", "14470", "49120", "58558", "53862", "26720", "17235", "4423", "46934", "48432", "5648", "29120", "23388", "27628", "46963", "23216", "4539", "27832", "58851", "20689", "5390", "43062", "49052", "57577", "41294", "18880", "10247", "32160", "7348", "47448", "6872", "6392", "55527", "59844", "30153", "3240", "9936", "30786", "7608", "17528", "18188", "4307", "33121", "48001", "26362", "51317", "689", "29651", "22979", "6067", "46668", "15191", "44035", "2737", "1923", "31609", "38715", "49573", "27840", "14288", "33351", "56764", "53309", "52829", "39152", "8632", "3957", "45998", "54694", "23975", "47491", "59725", "5096", "59704", "39604", "53670", "49419", "1232", "47028", "42640", "38061", "17075", "45831", "13883", "21051", "52097", "10999", "9415", "36428", "50096", "45375", "23555", "45141", "50910", "44876", "27271", "5025", "26140", "50261", "21797", "17917", "28697", "10281", "37511", "59403", "23114", "30468", "26268", "32861", "52297", "29264", "2796", "43366", "53311", "15436", "22840", "32845", "58923", "52321", "28585", "5193", "17910", "32066", "7164", "53581", "23394", "39583", "59898", "12334", "14505", "44613", "34762", "4766", "52640", "5297", "49992", "4146", "45963", "54947", "49950", "19241", "51518", "15533", "39511", "10241", "32084", "8875", "53971", "22715", "36607", "43611", "59396", "25615", "28162", "33942", "33605", "18840", "5992", "25748", "50480", "15117", "21137", "24857", "37936", "11710", "39272", "42164", "6590", "21786", "1642", "28740", "44725", "43935", "413", "36744", "34259", "55713", "52948", "1942", "45506", "13248", "13111", "45895", "17100", "51618", "10411", "57001", "23053", "49666", "30429", "56507", "55815", "8633", "17049", "6750", "5163", "34206", "54231", "19135", "35524", "23013", "3214", "38463", "14020", "26703", "35317", "12861", "10466", "542", "52030", "44710", "25204", "44490", "53287", "5220", "55294", "41976", "40001", "12403", "27298", "49115", "38477", "35862", "18595", "23252", "39420", "4981", "2761", "56787", "7052", "2828", "7924", "44151", "15291", "22659", "19583", "31619", "32077", "55022", "43163", "57717", "36234", "53911", "5037", "25445", "52850", "44936", "59772", "35973", "15316", "14808", "30392", "22218", "52639", "23139", "50572", "39871", "35746", "11117", "59792", "46473", "52767", "20863", "11343", "24269", "19516", "11356", "38019", "34439", "32087", "3821", "11416", "42062", "9453", "22527", "29478", "13303", "36182", "23059", "16382", "22912", "46721", "7894", "46257", "15927", "5530", "7576", "18538", "7993", "5929", "26843", "32614", "27772", "29933", "40656", "44673", "53354", "56996", "1974", "3992", "41955", "59073", "31045", "19900", "33524", "11329", "25239", "43962", "37503", "34031", "23456", "25984", "39105", "32713", "19341", "10695", "29434", "3880", "26104", "15936", "46330", "47695", "4722", "25383", "7379", "41754", "31176", "28941", "2968", "39228", "18812", "39169", "43202", "49131", "22525", "8202", "32617", "1985", "53738", "31841", "16220", "39499", "59302", "24959", "42094", "24967", "5750", "6483", "2375", "31616", "5889", "29298", "2168", "57879", "36036", "57026", "50315", "57265", "37552", "28397", "7103", "43953", "9243", "48189", "42675", "38154", "40524", "1112", "14096", "1127", "45556", "48735", "52262", "15985", "15591", "46067", "21610", "31345", "33806", "4046", "48083", "9980", "9888", "53321", "40891", "53921", "59624", "31378", "950", "34996", "9217", "27865", "17386", "11774", "55329", "12613", "42519", "14985", "58679", "20607", "46525", "24987", "51993", "15116", "9164", "16983", "42180", "21828", "31791", "49955", "53929", "18777", "18712", "39337", "46924", "54086", "24423", "40595", "16933", "16254", "54615", "20332", "54210", "12125", "16539", "26370", "16582", "25154", "2027", "3068", "46030", "39403", "8756", "46135", "58985", "34207", "5272", "53718", "15561", "16184", "52968", "35498", "21024", "474", "3164", "22665", "32802", "46885", "53664", "38316", "51223", "27928", "2689", "11615", "23061", "42536", "57749", "32109", "6416", "16409", "52818", "40551", "22687", "10228", "51621", "22031", "23853", "41992", "37052", "5055", "58539", "46914", "34973", "37653", "56129", "5323", "31526", "33366", "43820", "59394", "35476", "38124", "8175", "10449", "37706", "1729", "39059", "33914", "5614", "31957", "40241", "46112", "18675", "49295", "51241", "37238", "30474", "28702", "12710", "14127", "42972", "33707", "41389", "3085", "44819", "42377", "33586", "56806", "43327", "8364", "44298", "47807", "33795", "19013", "57343", "51568", "57159", "35167", "50209", "59727", "9720", "17464", "30077", "51563", "36522", "26850", "1302", "50936", "55926", "42645", "7648", "4992", "58939", "30767", "35316", "34140", "51068", "12618", "56574", "35920", "3053", "16890", "40439", "10088", "26227", "41354", "57063", "44193", "43495", "25618", "26940", "39900", "8644", "44873", "51006", "6833", "36183", "8215", "45947", "22730", "27791", "41157", "1526", "10494", "37335", "6177", "36005", "44785", "17844", "19197", "12951", "4681", "21599", "2663", "21723", "17177", "5886", "49687", "55846", "38631", "46186", "19141", "3320", "16894", "40262", "44009", "58844", "16911", "47950", "30076", "35339", "12587", "15283", "51282", "40190", "38282", "15248", "38708", "31247", "48688", "55238", "11579", "44422", "5806", "49930", "8389", "38561", "43732", "44002", "36281", "35373", "6896", "30581", "29570", "11377", "5550", "38936", "56710", "9120", "49805", "39076", "42103", "59701", "26429", "17716", "22370", "43350", "41390", "38910", "26577", "21529", "9467", "8969", "19189", "51364", "58002", "40520", "54273", "17513", "37929", "6571", "42327", "15582", "6509", "7872", "18024", "55797", "49749", "404", "45042", "20604", "4064", "9146", "51644", "43961", "46640", "17349", "20016", "28895", "31913", "6451", "30408", "22970", "19508", "33241", "2301", "10706", "741", "51441", "43405", "44034", "50499", "34534", "19830", "3701", "49406", "5632", "19009", "11310", "28374", "37393", "35446", "8771", "23446", "55574", "39365", "52048", "27569", "40938", "41128", "20377", "34602", "25342", "26116", "39354", "15536", "1099", "14264", "22139", "44360", "50436", "47850", "45044", "21919", "51749", "36970", "18029", "883", "46141", "11051", "31904", "46763", "23635", "48926", "17332", "5374", "52807", "17322", "40947", "25889", "13388", "43556", "11699", "12231", "54929", "22869", "25747", "37591", "59990", "41178", "34008", "51280", "39812", "54862", "15239", "18241", "46350", "41747", "52709", "5378", "49146", "42905", "35425", "19929", "56404", "57302", "54440", "11607", "42918", "20843", "34299", "57599", "14378", "6825", "57439", "11006", "43401", "53580", "6009", "37353", "44071", "46601", "44445", "34074", "58890", "46976", "21327", "42456", "21365", "15045", "56965", "5879", "35573", "21761", "38548", "12801", "18062", "38206", "3994", "694", "40766", "25926", "59580", "52364", "37236", "53802", "54903", "45105", "22419", "29417", "42279", "11146", "55278", "16009", "50895", "22151", "54725", "428", "14770", "31503", "10761", "18022", "1837", "31518", "25074", "45132", "19569", "23308", "18991", "26423", "24571", "24132", "26965", "17955", "32669", "8974", "4428", "49839", "38700", "7168", "43451", "29399", "29151", "36146", "34966", "4202", "22390", "2153", "44138", "16161", "29352", "38918", "16155", "57514", "14899", "517", "22097", "30371", "30344", "24594", "7324", "31305", "49067", "35970", "16801", "29372", "27366", "13442", "37394", "18915", "42826", "3921", "47335", "36062", "48734", "16555", "22071", "5761", "26906", "42805", "36784", "23511", "18986", "19168", "25132", "29053", "26905", "4390", "35595", "32739", "52895", "27782", "46628", "22986", "16466", "10238", "46480", "1234", "20101", "18192", "32065", "54177", "50218", "49140", "29887", "48443", "40971", "11334", "32290", "4015", "11824", "13140", "26073", "30899", "36085", "30212", "58422", "3592", "24915", "53424", "18400", "38146", "44479", "9451", "9262", "44818", "48625", "52769", "17975", "14369", "57718", "58362", "45143", "34856", "18854", "35000", "57652", "19512", "10902", "55368", "50478", "26329", "6905", "21863", "43461", "47861", "49674", "47059", "1157", "10937", "11318", "47223", "15152", "11494", "35135", "47228", "58520", "29255", "10725", "37063", "32450", "8704", "2463", "53684", "12810", "9693", "29126", "5343", "22617", "7081", "48907", "6958", "27463", "15154", "14040", "59666", "50838", "27276", "36247", "41895", "9534", "2619", "41433", "8145", "59286", "58497", "10973", "47451", "36817", "3170", "18631", "34691", "39025", "31900", "42170", "30934", "24927", "57896", "53815", "57894", "39575", "48", "46733", "59412", "9497", "54098", "49881", "13370", "5944", "40875", "56222", "28822", "4623", "11004", "27520", "47524", "10390", "43152", "21265", "19683", "226", "23656", "50628", "58342", "57368", "55231", "51320", "38434", "1954", "58557", "47098", "55928", "22673", "28165", "21444", "45296", "46607", "8643", "17624", "14311", "45152", "28842", "673", "3558", "46882", "19980", "41503", "41641", "39888", "52456", "26130", "7349", "39974", "58603", "22049", "41851", "31130", "8642", "40345", "53090", "43213", "57971", "8416", "14032", "34729", "32098", "50643", "1998", "29409", "47694", "50445", "26303", "37217", "1812", "24622", "23904", "1184", "53809", "45849", "11972", "14655", "48470", "9621", "2171", "40263", "25581", "56055", "7725", "55098", "9633", "43726", "7944", "21310", "2507", "52700", "54465", "35122", "46771", "4316", "55795", "733", "6264", "38725", "55106", "58088", "35494", "95", "15277", "10715", "40195", "53109", "54752", "37592", "20662", "8945", "31902", "35208", "4819", "58121", "54155", "55499", "44387", "35274", "51843", "46532", "21116", "49858", "36540", "17393", "15692", "30165", "11257", "51585", "39927", "43434", "43838", "27584", "11379", "26746", "25229", "18909", "34905", "26383", "4647", "22595", "20029", "5851", "40856", "52079", "28890", "40625", "12866", "16269", "20486", "33110", "25138", "44957", "32400", "10565", "8232", "53576", "6120", "46449", "27228", "31052", "59066", "23250", "26090", "45597", "46507", "20710", "34080", "50961", "10566", "39092", "15247", "31580", "52757", "24414", "2947", "35844", "6919", "46258", "6355", "41053", "21851", "29429", "17956", "54442", "28075", "45324", "9830", "16404", "40250", "25883", "5830", "47291", "25059", "43381", "47087", "45159", "5204", "1244", "57022", "49810", "29132", "28128", "40257", "25106", "3625", "41596", "9747", "32468", "56208", "20756", "6322", "11640", "6597", "19075", "36991", "30454", "41210", "51628", "19898", "44085", "29485", "56812", "43181", "4050", "32951", "47586", "2967", "51665", "11128", "55330", "8441", "43814", "3100", "8856", "11289", "17484", "21438", "38321", "55453", "29396", "40644", "41852", "31332", "54288", "33415", "7139", "36937", "27909", "5917", "38352", "2839", "13626", "55150", "42927", "57915", "23745", "52750", "51666", "33986", "10788", "32349", "6854", "31754", "19983", "28365", "58003", "12619", "57801", "51578", "38669", "46026", "14050", "20533", "27189", "46657", "36888", "51142", "36992", "54801", "50242", "7158", "41668", "13096", "40882", "19101", "2606", "58987", "34971", "32022", "43928", "38464", "1046", "30923", "56435", "45289", "41481", "10812", "12465", "51759", "848", "49167", "43342", "42481", "19455", "37102", "5910", "31678", "43477", "26497", "771", "12708", "32556", "16990", "40070", "13042", "37311", "54035", "48286", "33559", "28824", "18610", "23891", "23498", "3000", "38992", "23478", "31356", "30095", "34366", "51234", "51753", "10360", "17519", "38532", "51936", "20479", "19338", "38438", "24062", "27742", "21477", "19483", "57722", "12748", "43744", "42012", "41353", "50308", "50442", "19498", "4139", "50562", "53010", "35297", "17007", "26103", "9272", "55276", "57941", "53892", "51488", "16154", "7829", "2665", "40292", "41100", "36035", "46678", "24675", "45011", "43558", "53996", "48272", "53634", "7561", "36254", "34720", "50019", "59668", "37377", "47929", "24524", "39560", "15659", "21407", "7301", "21963", "25888", "23441", "13993", "20869", "26864", "32017", "23391", "6746", "19372", "13975", "55779", "10878", "15507", "29152", "39570", "27711", "10041", "11709", "28568", "11757", "9618", "31910", "47034", "35022", "57960", "54875", "50907", "30551", "20115", "26427", "34965", "49149", "19082", "35711", "8110", "23323", "34667", "39442", "36013", "13815", "24896", "43794", "24390", "38276", "49343", "17045", "14500", "5489", "16354", "28020", "42079", "57714", "16416", "6756", "50508", "53251", "28180", "53319", "19148", "23343", "9858", "24743", "24317", "25623", "17742", "28570", "2401", "4874", "5474", "3970", "7015", "1424", "1959", "11167", "38159", "11001", "33347", "21318", "34493", "46539", "23221", "59017", "2906", "10968", "43284", "27230", "18114", "21883", "26711", "5434", "24282", "14321", "23700", "33105", "51529", "18525", "55365", "10144", "48026", "56106", "55225", "46773", "34601", "59710", "7871", "49479", "19838", "5527", "5981", "38996", "5494", "28855", "40797", "10607", "16488", "29574", "51547", "52084", "45638", "6152", "15607", "32856", "40503", "29883", "11843", "16742", "15423", "42321", "5370", "46926", "4579", "29774", "16470", "9398", "37635", "19281", "28427", "8523", "23664", "33669", "8384", "39188", "18310", "2054", "28565", "28069", "9130", "35951", "10498", "26135", "58901", "57283", "31321", "3941", "58673", "36495", "59075", "39400", "54172", "40176", "30159", "19677", "52473", "19479", "32283", "57575", "30103", "25221", "13138", "54266", "30505", "51309", "20508", "51571", "27356", "56515", "55209", "32489", "37106", "46591", "49176", "28105", "10759", "19684", "764", "25393", "15174", "22613", "42500", "7400", "57931", "43323", "44765", "18420", "13055", "39819", "32049", "10156", "10646", "57308", "32537", "9675", "22299", "55254", "55217", "3712", "4575", "23224", "10994", "57237", "5603", "12315", "8700", "57204", "3166", "30555", "57954", "7565", "30848", "22678", "57950", "38490", "29566", "151", "24893", "9716", "57472", "589", "27391", "17357", "18351", "27014", "26111", "7321", "43922", "59759", "56896", "45950", "23143", "57728", "43748", "25079", "18030", "19548", "4356", "8937", "41912", "15652", "27728", "47133", "44062", "13540", "51403", "28875", "18584", "25661", "5871", "23641", "43513", "36086", "7493", "57257", "18837", "39381", "18441", "59652", "20512", "5236", "3527", "22346", "21036", "29135", "5077", "15891", "48753", "47746", "26409", "25879", "5010", "47064", "47264", "35172", "1106", "28954", "20705", "50915", "1000", "33847", "54368", "15320", "5751", "10187", "55563", "59479", "30219", "34621", "54992", "29674", "4919", "32590", "13360", "32993", "48482", "1657", "45239", "43016", "8742", "13187", "50648", "58035", "52226", "30204", "26244", "34674", "33828", "19842", "39329", "1318", "22888", "214", "14167", "52905", "9305", "8448", "11928", "47710", "18831", "28813", "28508", "41597", "55899", "43804", "14755", "34011", "49171", "30528", "32497", "6289", "26045", "51733", "28633", "48789", "49408", "41937", "39083", "46159", "56992", "25253", "1411", "38760", "33579", "14313", "41071", "57088", "16649", "2511", "36177", "8068", "11134", "28078", "42384", "55149", "24920", "28185", "49436", "43938", "40735", "41531", "6487", "47572", "17455", "2574", "13785", "5746", "10957", "33389", "10363", "21720", "24632", "12269", "26619", "5194", "14707", "53029", "19547", "47713", "43216", "43371", "45610", "47095", "22124", "49232", "48590", "39730", "59698", "48348", "33325", "11857", "5953", "59650", "1880", "6479", "28369", "35514", "40729", "40706", "2607", "47714", "43563", "12868", "24654", "53314", "12003", "46495", "11942", "14328", "35871", "2840", "29584", "701", "8726", "6471", "58464", "17643", "21568", "36952", "38519", "44401", "49217", "26186", "26173", "41299", "12768", "18007", "21426", "12860", "27986", "24027", "36209", "25243", "3891", "49542", "21080", "34113", "6918", "56805", "28251", "55475", "36587", "21519", "25895", "18609", "6973", "11280", "17182", "1192", "37222", "14878", "47145", "19761", "1335", "38718", "3307", "18404", "13230", "59984", "49556", "39466", "55123", "10685", "54634", "106", "22262", "36163", "30740", "54677", "46063", "53638", "59095", "5699", "56533", "55653", "45551", "26286", "4995", "5405", "51341", "46406", "19996", "5301", "45252", "47453", "38329", "54625", "51180", "33323", "39084", "42359", "560", "11663", "25013", "51762", "44655", "56670", "22701", "26128", "29693", "45667", "42844", "52569", "39389", "16022", "4239", "34554", "35773", "10407", "13916", "55244", "13625", "4386", "31153", "42934", "6575", "21111", "5450", "36710", "41219", "17334", "51994", "47897", "867", "15773", "3535", "5774", "21157", "17431", "45235", "12035", "18483", "54003", "28582", "52133", "19451", "29338", "35375", "30735", "44745", "37264", "31134", "881", "23905", "7080", "23887", "47284", "56269", "29089", "52745", "20346", "9014", "42247", "12901", "1822", "18313", "56775", "50121", "53760", "3843", "37496", "19171", "28318", "38001", "59005", "41653", "41866", "30825", "58586", "49311", "53428", "41672", "45911", "24558", "2291", "28517", "57323", "55913", "1123", "29877", "31587", "52243", "7438", "56480", "51530", "10709", "13199", "10628", "35384", "11492", "6053", "14828", "34675", "31576", "8447", "29965", "5071", "52959", "25882", "1479", "47213", "27455", "3417", "37280", "16691", "1126", "48765", "7100", "53179", "52966", "22628", "51301", "42242", "48153", "34499", "51873", "775", "37551", "45751", "23509", "35154", "22618", "22994", "12657", "23448", "27934", "53529", "37548", "4291", "29317", "14558", "9906", "32422", "22290", "55823", "20788", "36750", "13654", "27292", "18780", "25233", "59889", "43574", "27968", "15212", "37968", "16837", "37974", "8023", "44497", "37191", "27239", "49106", "6015", "32520", "49856", "48462", "2982", "55252", "18555", "57235", "6284", "11058", "10669", "37995", "25215", "58631", "13733", "43714", "38392", "17690", "49601", "33283", "8981", "22515", "21127", "42819", "40560", "56684", "49583", "17625", "50857", "8878", "23087", "26578", "37154", "1205", "35739", "34318", "21573", "27530", "23710", "41285", "46043", "56831", "20527", "35624", "26036", "31422", "36057", "49786", "25556", "58634", "28602", "49814", "15252", "12478", "35570", "50035", "32042", "55189", "11725", "59811", "47233", "42935", "29039", "6437", "19250", "22261", "13297", "56260", "57586", "21416", "31205", "46705", "46806", "54523", "27799", "22355", "26743", "4858", "9810", "15687", "41546", "38139", "49516", "39519", "39131", "50109", "34920", "4532", "1819", "50889", "16985", "41379", "58274", "57020", "42368", "44131", "4425", "34758", "14152", "51171", "51949", "38299", "26382", "10203", "28026", "16052", "54598", "7765", "21737", "28658", "51950", "44424", "37495", "39513", "35608", "23031", "52450", "10008", "55077", "35155", "54350", "25318", "47755", "58771", "41553", "19176", "27930", "13747", "362", "6964", "53632", "56878", "50125", "29284", "45775", "43718", "44386", "47649", "56547", "53955", "30719", "23717", "13219", "29642", "50290", "11275", "23455", "29121", "50286", "44427", "54590", "28127", "40884", "17858", "51051", "59118", "45613", "19771", "43210", "34864", "47077", "40269", "24374", "42465", "9091", "54714", "52673", "42904", "36697", "9140", "7300", "28589", "12559", "40629", "7611", "2910", "16899", "46318", "12025", "55151", "9373", "53805", "37103", "54176", "31928", "14316", "42316", "46400", "8284", "14562", "43840", "22166", "35104", "15595", "29038", "32469", "58811", "13395", "7782", "55573", "19006", "41269", "18391", "45522", "55350", "31873", "29747", "42172", "15406", "1388", "29222", "39147", "34798", "6100", "50882", "13440", "13885", "20759", "3813", "37771", "1171", "43529", "27618", "36198", "31745", "18103", "25195", "28662", "56330", "1247", "13475", "21225", "49278", "59848", "58213", "22383", "15476", "47021", "37924", "47508", "52181", "14351", "7596", "59543", "40486", "42732", "34330", "45639", "19537", "40807", "46225", "15345", "45349", "18700", "16617", "51694", "41406", "17399", "23582", "46816", "34843", "19591", "39051", "38000", "7526", "34210", "9943", "9147", "34647", "39239", "37320", "30609", "51780", "54654", "35767", "49340", "11126", "28261", "43790", "4811", "15017", "12588", "52655", "36938", "1909", "33836", "30012", "2604", "11348", "30892", "58047", "37937", "26019", "31656", "39737", "44025", "53695", "29725", "55363", "36548", "25772", "57146", "6741", "12924", "46151", "19340", "38733", "30451", "13338", "31444", "742", "31098", "8479", "40703", "10290", "6523", "25175", "54263", "28254", "50494", "2759", "58219", "47655", "16835", "9475", "25365", "50331", "11736", "51842", "24463", "59585", "40308", "26262", "148", "59266", "9416", "23952", "32307", "19994", "2391", "56543", "35190", "42448", "11512", "20637", "24973", "44536", "51890", "4783", "19718", "25680", "40886", "49087", "44495", "36703", "22802", "16507", "4587", "34060", "18265", "3663", "1981", "16192", "3061", "25582", "31866", "48243", "51992", "2035", "58768", "30975", "33953", "2332", "2875", "52276", "2403", "27078", "5156", "14518", "52366", "28797", "47196", "20044", "1181", "33521", "9963", "10840", "25269", "57661", "40888", "1458", "10640", "59464", "7531", "25916", "51469", "39267", "18398", "301", "57421", "39368", "3205", "17796", "4134", "33900", "34099", "49748", "58689", "21384", "53171", "35695", "46356", "11225", "50560", "18068", "57838", "1487", "28622", "13259", "19853", "53008", "46206", "24331", "24271", "41781", "53661", "45583", "1293", "13461", "8537", "32435", "27100", "38688", "15988", "22022", "45261", "14910", "31800", "473", "37324", "39119", "41686", "12441", "20890", "18750", "43097", "49661", "45449", "34435", "39343", "14819", "21620", "39872", "34630", "27144", "1576", "15948", "34900", "42174", "24865", "16100", "1536", "6088", "27631", "53525", "21971", "57390", "32434", "4613", "48533", "3591", "58086", "40667", "14611", "1930", "8982", "6862", "44420", "58205", "6256", "46023", "33304", "48483", "56797", "29874", "58253", "26207", "30347", "47958", "29754", "40011", "22064", "36115", "49598", "50117", "15801", "33172", "30238", "55117", "10661", "10516", "37770", "28035", "55560", "6724", "38995", "34869", "48755", "21198", "32192", "34541", "40035", "578", "48079", "19810", "19020", "13713", "29734", "4683", "44185", "50551", "1460", "48417", "28383", "8718", "48288", "26875", "15715", "2775", "26464", "54627", "50029", "11786", "16062", "21579", "11193", "25431", "37954", "51974", "15344", "36100", "40044", "35541", "22611", "37120", "22893", "55874", "17889", "9827", "53856", "4931", "40462", "13065", "54330", "38964", "1795", "53201", "5144", "26579", "9548", "53795", "43848", "48013", "30284", "22217", "6819", "38025", "49211", "28071", "15530", "56419", "48234", "29918", "6703", "30138", "35218", "33392", "15658", "40862", "14688", "47736", "5848", "44607", "43926", "43488", "50377", "15971", "13943", "39933", "33057", "16093", "31074", "58780", "58885", "40397", "37797", "22553", "57786", "10133", "3556", "37836", "39459", "5810", "27552", "32067", "30956", "27834", "20322", "28774", "34488", "57702", "49244", "65", "19701", "5026", "41506", "2231", "46693", "15349", "13533", "18997", "21029", "23119", "38117", "4443", "18025", "25397", "22941", "22542", "42434", "40216", "39250", "31173", "21796", "16111", "54014", "8531", "55457", "939", "12272", "45428", "6928", "44098", "41613", "48980", "553", "1595", "16284", "59088", "33835", "25466", "22408", "23207", "26257", "48505", "21528", "6521", "50100", "41532", "3567", "18732", "1557", "29655", "14423", "3030", "16050", "50526", "10783", "54461", "5284", "32102", "32837", "32930", "57144", "34727", "3160", "53516", "9695", "51565", "21159", "12312", "43720", "43098", "6906", "50784", "53067", "6674", "13337", "27651", "25447", "29931", "43478", "58403", "52073", "29583", "31698", "23453", "56945", "37688", "51872", "49263", "28070", "50463", "30783", "48282", "16405", "17594", "58293", "31249", "50809", "52309", "12177", "7362", "29906", "10479", "33336", "55356", "52402", "24662", "17407", "46466", "27462", "14474", "51758", "54557", "34592", "59274", "819", "17298", "12381", "31075", "59851", "27355", "22602", "53952", "49290", "20798", "36241", "33464", "11408", "47686", "41248", "5825", "20327", "19233", "21017", "37508", "39913", "54159", "15923", "58148", "44363", "15678", "29793", "51168", "24442", "47536", "31312", "7468", "57819", "4700", "29878", "41416", "18477", "47254", "14695", "42472", "32472", "21064", "45687", "8320", "16296", "438", "46537", "24734", "29445", "7523", "23859", "49042", "3634", "9634", "11204", "11172", "20768", "35926", "7547", "12400", "11268", "16635", "48683", "18736", "7372", "57270", "32149", "30462", "32186", "8397", "31288", "39965", "54762", "6566", "28803", "4184", "12632", "3316", "3937", "5977", "41191", "25422", "252", "41477", "52996", "40692", "55102", "8031", "3220", "1553", "21288", "54589", "51001", "9334", "17587", "26993", "14202", "54751", "15409", "3468", "23652", "5814", "26553", "25332", "45535", "33626", "5373", "22923", "29476", "41176", "24084", "20152", "47104", "9878", "42618", "56559", "1519", "51428", "7161", "28910", "43249", "45040", "18026", "30168", "2546", "23586", "28698", "26788", "15784", "55316", "28462", "24194", "19948", "59281", "35014", "39390", "45191", "5358", "12793", "24782", "9240", "56662", "9611", "27244", "15638", "18216", "4212", "10585", "21486", "15395", "21638", "47119", "42875", "56750", "8279", "30013", "29973", "54845", "25385", "18395", "58644", "35", "4534", "51812", "11727", "29514", "53855", "36049", "17535", "2063", "4855", "54278", "9759", "31672", "27272", "30261", "40670", "14809", "10986", "38959", "26051", "16290", "48365", "28327", "8029", "32527", "15895", "51098", "47778", "19248", "10590", "37723", "44305", "47331", "37601", "18189", "58823", "34152", "12972", "47831", "45519", "488", "52436", "12959", "5445", "10110", "2814", "14960", "47011", "10270", "6324", "5247", "58199", "6888", "42439", "13340", "28922", "36884", "25972", "14939", "55638", "41809", "44796", "12189", "58963", "22605", "18973", "30553", "15346", "33315", "47418", "23721", "27451", "55007", "55917", "4942", "5804", "494", "55971", "51490", "3079", "16461", "15019", "56281", "3742", "914", "58709", "19494", "35478", "6064", "40471", "27510", "1815", "14257", "59096", "22742", "42284", "29569", "39108", "12364", "40790", "37447", "27291", "15963", "58057", "588", "19380", "46554", "45562", "41727", "28276", "36321", "18134", "28554", "42289", "53596", "39643", "42932", "57864", "56443", "50313", "8131", "6174", "59223", "15493", "15221", "19760", "11393", "13680", "6472", "52178", "31234", "22116", "23922", "50892", "36367", "8942", "19955", "17515", "7908", "15908", "32821", "4618", "54670", "24288", "624", "21534", "55131", "16089", "54236", "23128", "13787", "41525", "52070", "47848", "9947", "55776", "42963", "10398", "57345", "47076", "43103", "24024", "58426", "10003", "35038", "5901", "54321", "59377", "15769", "16324", "30198", "13120", "37539", "45142", "36802", "59620", "50553", "38083", "32338", "37699", "32855", "864", "26984", "25406", "20636", "24560", "16753", "38645", "58094", "22061", "53969", "47519", "6401", "37964", "56667", "51540", "19872", "21175", "18816", "56125", "3746", "21289", "38933", "6890", "16575", "53031", "49554", "41135", "38689", "33126", "42445", "23965", "35589", "47298", "29047", "26531", "14632", "59490", "20104", "39745", "608", "24432", "6601", "36753", "15462", "12030", "18256", "47029", "57196", "1363", "7071", "27550", "11508", "15215", "5081", "35542", "55465", "22213", "33547", "14121", "41177", "36681", "17890", "29649", "54640", "32847", "57086", "32718", "42892", "37840", "37988", "44533", "41373", "43702", "20757", "15416", "15112", "40075", "53710", "6675", "15492", "49030", "2161", "25861", "51390", "10763", "34386", "24771", "218", "11312", "44734", "34347", "50373", "30871", "48395", "22856", "4398", "2421", "45548", "3491", "6808", "35647", "8007", "11802", "40350", "15657", "45826", "23626", "47066", "15229", "58946", "40392", "16385", "42729", "12821", "32815", "32900", "55961", "44974", "39665", "6155", "13921", "16546", "7515", "27693", "40878", "47892", "19648", "51559", "39594", "2175", "791", "25750", "52475", "45765", "34786", "25062", "58776", "29930", "37466", "45202", "13876", "45274", "30036", "57589", "38318", "26939", "39945", "29122", "2660", "48208", "18580", "20134", "14172", "47737", "56503", "58430", "13767", "25897", "43050", "9212", "34125", "22782", "39684", "30271", "1647", "15665", "10594", "50301", "45020", "17872", "59470", "9920", "51026", "23490", "56558", "48332", "3257", "7033", "15932", "58541", "49607", "27022", "37270", "16344", "5216", "44184", "26372", "33928", "44571", "14898", "48394", "59676", "23946", "57114", "30389", "51100", "41854", "26367", "28378", "52544", "12635", "40164", "13264", "5432", "50215", "47014", "18", "18963", "27765", "28991", "27796", "45", "40288", "6111", "45683", "10825", "1621", "25128", "28342", "39824", "23794", "34157", "3735", "29143", "54152", "53631", "19861", "27080", "43899", "8904", "43957", "13952", "8082", "32334", "54760", "19813", "57486", "49764", "55009", "30726", "34132", "13522", "45464", "55129", "16029", "9092", "23179", "16728", "49835", "16987", "11845", "42508", "15016", "9785", "9750", "36151", "44576", "18852", "36864", "35413", "24881", "4954", "3226", "44020", "17083", "10376", "40260", "51072", "32292", "8545", "3176", "44644", "14804", "41304", "8862", "37600", "14004", "33388", "54518", "23803", "27165", "53520", "12513", "33025", "34709", "14796", "59037", "39774", "50339", "1300", "42160", "41705", "28060", "42910", "33666", "35512", "31795", "34961", "7685", "55349", "23842", "10584", "45600", "33299", "10253", "51141", "47739", "32055", "21877", "31246", "50203", "23168", "22664", "25606", "6239", "57303", "47578", "4929", "49124", "49292", "1865", "17323", "54338", "57360", "3019", "27107", "51743", "28709", "14791", "58308", "22448", "28937", "44473", "24605", "46749", "2837", "4565", "34394", "9702", "49770", "52428", "17612", "31786", "52526", "1461", "56461", "4572", "8513", "5641", "47271", "35267", "57567", "7092", "8897", "30985", "15672", "40249", "14978", "9079", "17896", "27484", "30642", "59383", "55600", "29768", "16203", "34276", "39042", "7730", "24517", "17019", "15730", "12643", "6489", "8820", "10753", "27790", "52489", "58033", "3923", "13038", "20715", "31617", "39622", "22587", "41146", "58234", "12918", "1292", "31112", "54809", "25069", "18186", "32695", "56333", "47377", "33218", "30123", "9508", "53397", "33780", "51430", "18890", "7082", "47862", "51901", "59151", "527", "25236", "18817", "57293", "34797", "34018", "29156", "19246", "2149", "5821", "27726", "31477", "37822", "12540", "53269", "20298", "1134", "38717", "56358", "28259", "6497", "29046", "42520", "19641", "59103", "9258", "40631", "36164", "35803", "26462", "40948", "36232", "29557", "39104", "59904", "42803", "21342", "48697", "45120", "39534", "2521", "22173", "31810", "40964", "4559", "18520", "20469", "2811", "49498", "40665", "52679", "49260", "37500", "2797", "54577", "7939", "5583", "16378", "20568", "37693", "31500", "7013", "1452", "29744", "18252", "21468", "14238", "14088", "48747", "18380", "23838", "38323", "21204", "35551", "2953", "55079", "8535", "14034", "46422", "1706", "35997", "39462", "58633", "21267", "39542", "22250", "10183", "40320", "38197", "40323", "51065", "51373", "14194", "48048", "10147", "33522", "30025", "49037", "42514", "22345", "41677", "36012", "17511", "57267", "20939", "13631", "12919", "53328", "3987", "26139", "55782", "10712", "55801", "57553", "46555", "53094", "26041", "20388", "22454", "46001", "34878", "36610", "29648", "26725", "2750", "56592", "9410", "37881", "57309", "6410", "11852", "16884", "5130", "59873", "36612", "34679", "21087", "21250", "50409", "58794", "46867", "46910", "53135", "8295", "42974", "49009", "45756", "31536", "40255", "23519", "17325", "22111", "34093", "36556", "13201", "30216", "3907", "46275", "22943", "49414", "2116", "35406", "27895", "30715", "37488", "30846", "28419", "57287", "16785", "28456", "1075", "38806", "23727", "59959", "23614", "33843", "51770", "23248", "58203", "16674", "34227", "49253", "55436", "53277", "44641", "6656", "41623", "49043", "46639", "27543", "10723", "7846", "19800", "48736", "29551", "46338", "59627", "4314", "49314", "48529", "11285", "11782", "28657", "58188", "18652", "7618", "14574", "8093", "38668", "16680", "47636", "14089", "11223", "38625", "15557", "31200", "43583", "13770", "43501", "21463", "59949", "29473", "31911", "21326", "22340", "15173", "46438", "4824", "31286", "45990", "45863", "50177", "19592", "714", "26673", "30615", "4072", "40519", "37868", "44566", "19806", "41523", "8474", "1309", "49631", "57112", "21232", "5192", "45382", "22853", "44977", "21004", "55137", "44508", "53158", "33690", "39656", "29433", "57135", "36300", "26387", "14776", "23365", "22882", "663", "31965", "31833", "27111", "57875", "42547", "23648", "32204", "25498", "40427", "56231", "8460", "15240", "33717", "42095", "38285", "37438", "2933", "59568", "22531", "30600", "23588", "51464", "37321", "33545", "53960", "11265", "16014", "53850", "14443", "45720", "59388", "42691", "59025", "53056", "46297", "38188", "32771", "19963", "43389", "36048", "54727", "47934", "36292", "31128", "26948", "22065", "25607", "59121", "55688", "1510", "46456", "43297", "24136", "36539", "802", "53668", "25442", "19289", "43813", "10325", "45717", "18307", "54240", "37227", "26540", "28527", "5296", "49415", "5047", "55761", "38994", "57886", "2643", "6805", "22571", "48441", "22688", "3461", "37558", "38601", "18742", "42652", "9439", "5595", "18927", "47896", "6698", "35139", "44937", "29209", "58312", "27696", "7938", "11582", "51378", "15890", "49346", "13372", "47995", "31437", "17712", "35759", "47991", "37506", "35974", "36117", "45546", "8949", "11537", "45640", "58569", "46811", "31764", "53061", "37333", "50901", "53226", "49880", "37982", "11183", "54713", "38415", "57820", "40414", "19389", "27873", "50235", "8847", "36015", "54482", "20642", "55186", "24330", "43565", "53792", "17097", "13694", "7545", "34520", "35503", "17608", "38085", "32011", "7571", "30214", "38819", "30054", "3361", "43959", "22858", "41419", "58146", "54990", "23271", "9872", "55855", "14223", "32073", "5676", "49447", "48931", "54546", "41855", "55501", "34162", "40802", "22780", "55080", "42789", "37109", "15590", "40331", "15534", "8172", "31081", "8199", "37602", "29101", "9303", "52427", "31598", "31171", "5291", "29893", "49667", "32387", "31775", "4923", "17698", "35801", "34246", "25805", "27954", "28903", "36903", "59626", "39423", "51220", "22442", "49157", "13466", "5169", "50520", "27344", "41707", "49402", "8686", "34202", "46889", "22522", "171", "46852", "10619", "40259", "42539", "52798", "32252", "32629", "9602", "20790", "26032", "45513", "11540", "26323", "15634", "59757", "47936", "46090", "17321", "37023", "37292", "43467", "178", "38606", "8850", "22626", "9464", "11861", "3073", "27719", "13650", "29707", "30968", "52407", "58629", "13871", "11558", "7935", "43705", "25934", "47359", "1809", "34613", "54379", "12245", "53362", "56194", "52739", "24521", "29594", "4163", "36727", "24695", "10506", "22041", "31521", "52171", "20781", "31639", "632", "44624", "44317", "58550", "42913", "37901", "55900", "18521", "29385", "51089", "22568", "3244", "38038", "4731", "31434", "8431", "40822", "36737", "58482", "37400", "23898", "40361", "11818", "16610", "42785", "20822", "20540", "44666", "16757", "29321", "55450", "18019", "42553", "42576", "28360", "37882", "56562", "30995", "43160", "11697", "9389", "54567", "13705", "30180", "38078", "26796", "1673", "53463", "24493", "10629", "35928", "5141", "47946", "34572", "53118", "32964", "51224", "12882", "17838", "16696", "25962", "48733", "36478", "40583", "56671", "4282", "59950", "37810", "36393", "43580", "5171", "31110", "2396", "22799", "20557", "28909", "58843", "39231", "43627", "46148", "29688", "2222", "46496", "55040", "59944", "28184", "56641", "50504", "5954", "24942", "33925", "47199", "28804", "57092", "37892", "53994", "38613", "22507", "10303", "49803", "38256", "28426", "18972", "50459", "42213", "54586", "36805", "43169", "20423", "37656", "18836", "59869", "22172", "20886", "46358", "34771", "31328", "37815", "49275", "32955", "14757", "56437", "34168", "17364", "59196", "12359", "37218", "24318", "53786", "45767", "6804", "55241", "32232", "31623", "42076", "27381", "4096", "23499", "2657", "49531", "16669", "33188", "51016", "24133", "50939", "13181", "45795", "31684", "49069", "45494", "16196", "5511", "44404", "33493", "40602", "9016", "21611", "44630", "47012", "16446", "19604", "24029", "54942", "27237", "48054", "46762", "46710", "37046", "30549", "32677", "27658", "6056", "22160", "16477", "41233", "8880", "11885", "28158", "27976", "2884", "56976", "12336", "53946", "57326", "53474", "59687", "43555", "36444", "39326", "44001", "43925", "28592", "17522", "642", "21177", "35451", "48889", "17807", "15260", "27323", "17720", "10414", "57353", "9781", "21078", "35125", "13749", "675", "55508", "21998", "42420", "2427", "21020", "39323", "40651", "34288", "47471", "880", "59766", "29752", "45838", "35726", "56008", "55081", "7706", "50358", "19826", "34371", "18843", "38296", "32263", "36764", "50960", "53096", "52765", "35269", "15287", "26347", "10745", "5459", "3281", "42052", "35270", "7739", "50846", "36220", "50999", "8204", "2847", "24297", "33359", "16786", "1767", "69", "39803", "58313", "14543", "273", "53473", "50202", "25113", "23435", "50278", "22860", "39446", "44783", "47607", "39647", "42623", "49453", "42888", "59022", "30633", "10075", "14914", "43877", "4094", "22175", "51954", "7048", "15714", "50145", "54153", "24691", "54239", "51027", "29856", "18115", "21214", "17186", "11374", "6231", "45197", "49353", "38721", "30672", "32539", "14341", "45943", "19747", "21423", "38724", "55942", "14069", "43760", "51580", "16053", "49113", "3689", "38091", "34840", "37299", "20816", "43276", "42695", "30060", "21604", "3633", "57562", "8328", "15976", "2098", "19131", "8043", "4220", "44860", "4609", "7778", "51003", "16529", "47602", "50531", "50974", "23487", "45784", "47565", "8437", "38991", "34345", "915", "43432", "37946", "2218", "5646", "1250", "37153", "24943", "57961", "42832", "10089", "340", "12039", "46315", "28549", "58411", "13992", "57651", "19319", "6929", "41671", "33593", "20184", "48548", "58190", "35516", "27844", "29010", "23416", "39576", "8569", "8337", "43190", "4283", "59137", "8828", "21924", "27868", "31179", "3981", "10680", "1788", "18791", "28281", "3623", "11610", "9397", "28278", "55540", "32947", "30084", "47513", "6506", "32631", "58559", "38514", "44942", "9641", "41382", "13877", "52434", "41015", "34540", "18337", "41522", "26899", "5188", "54719", "1047", "13427", "30898", "14728", "48740", "5225", "33716", "23693", "12001", "19503", "40867", "37437", "50024", "51232", "25702", "31086", "32593", "34198", "6859", "3900", "28889", "15911", "51276", "5644", "49243", "27340", "51086", "50714", "41083", "52480", "33270", "36488", "11771", "59483", "27357", "32198", "36961", "20820", "13879", "21615", "56621", "11177", "33823", "35879", "21956", "55831", "29296", "58285", "48162", "12187", "54514", "31372", "47873", "39788", "16870", "58984", "43388", "2669", "49806", "3344", "20731", "51924", "42237", "2905", "49036", "38050", "45736", "52283", "49086", "29903", "27900", "17534", "57556", "53466", "35505", "9898", "12277", "58490", "48263", "15870", "8901", "53483", "401", "44786", "17342", "42736", "58875", "18364", "44377", "12687", "47866", "37096", "55042", "16980", "22663", "28972", "39493", "25707", "16955", "49362", "28558", "16028", "57729", "52576", "56285", "28142", "52995", "20407", "55753", "13245", "51434", "39085", "14766", "18865", "55909", "7840", "22967", "44653", "57198", "45396", "30006", "51533", "58867", "31646", "58167", "22857", "21035", "39340", "42262", "36377", "9815", "16726", "35394", "30324", "43280", "27362", "59135", "46313", "33613", "58245", "29902", "27304", "11296", "51004", "50398", "34110", "13888", "50660", "19737", "20428", "58492", "40711", "32257", "43153", "7563", "42603", "23970", "15532", "9605", "9193", "36093", "53747", "3772", "52667", "39058", "9358", "40662", "38348", "37963", "11063", "42662", "27409", "50931", "59404", "38551", "28870", "18935", "38762", "9460", "1259", "1437", "14290", "28229", "36359", "26523", "16298", "23003", "13175", "49456", "54765", "43259", "23990", "9286", "44155", "56085", "45992", "32378", "42900", "23222", "59424", "58562", "35996", "50984", "54306", "47310", "15559", "50249", "33125", "440", "41873", "58272", "32082", "36670", "1310", "59840", "24606", "14469", "4101", "4560", "34988", "41458", "26857", "12388", "40440", "48217", "2601", "24012", "43450", "25101", "21538", "24556", "55982", "4042", "23778", "24255", "50966", "11706", "2088", "28233", "51198", "58537", "3360", "30788", "58132", "25016", "36294", "20915", "48673", "15351", "38095", "35113", "3330", "49887", "21809", "27178", "33263", "44574", "16514", "8716", "41543", "9015", "23795", "48183", "25936", "4747", "2735", "50404", "43801", "20932", "10492", "56906", "12211", "1383", "52125", "47478", "19086", "36346", "4644", "30021", "17214", "3321", "54470", "18102", "34298", "36047", "28977", "28518", "8954", "35707", "38472", "59061", "8618", "51274", "51392", "48387", "20940", "47237", "42071", "30874", "10055", "5438", "12224", "39336", "12536", "39923", "23945", "22365", "7776", "22432", "17353", "15442", "33776", "51572", "32712", "54042", "9366", "48326", "42545", "49665", "46307", "33261", "371", "53726", "28646", "49926", "56644", "28595", "26216", "24405", "17632", "50333", "39074", "24177", "19373", "12393", "50224", "21696", "7332", "10682", "19275", "33745", "34867", "26422", "58781", "970", "20089", "33873", "19099", "57437", "18665", "26950", "5120", "28930", "35266", "38381", "58214", "9772", "5755", "7529", "19831", "47275", "11551", "1333", "42205", "50337", "39752", "16451", "17437", "164", "34473", "53540", "48277", "8073", "46314", "22446", "44564", "40076", "50967", "49181", "18126", "13142", "2276", "48748", "34938", "6691", "13128", "40389", "46481", "499", "54509", "22067", "53091", "46753", "21915", "17723", "2097", "47470", "16112", "3172", "37746", "13458", "16347", "28981", "18271", "17309", "23998", "47392", "30682", "20888", "33382", "33892", "53620", "43658", "43635", "10624", "49428", "33608", "34490", "34538", "55611", "30199", "14465", "18619", "20447", "5905", "12372", "46285", "18426", "23444", "34747", "21533", "48669", "2691", "52841", "4792", "3959", "18769", "45048", "2575", "22119", "48246", "47190", "18465", "48620", "24875", "23570", "21478", "42371", "11576", "50131", "41701", "37877", "37467", "21654", "39641", "17713", "13948", "48030", "5771", "52245", "10775", "7772", "6520", "35713", "38654", "12590", "47906", "26804", "16395", "11887", "57803", "54167", "48979", "52534", "12321", "14599", "40037", "16369", "13494", "13700", "29216", "18785", "25850", "46079", "54391", "12225", "52045", "13146", "50163", "33784", "17635", "2917", "49694", "54043", "30964", "18633", "47227", "21190", "41472", "50483", "28417", "34737", "26981", "56126", "56391", "12294", "29737", "39048", "36925", "45087", "33555", "28538", "2444", "40968", "51254", "20714", "57598", "15881", "32581", "22677", "8022", "50305", "28836", "57888", "24602", "47651", "4177", "50575", "23071", "58507", "863", "7066", "26187", "42990", "38478", "7431", "10193", "59272", "38076", "26659", "39278", "23825", "55520", "6503", "51160", "54041", "7937", "8160", "8187", "24701", "8613", "2944", "4232", "7017", "36135", "11373", "52784", "49690", "24378", "298", "31920", "17124", "12630", "25081", "44539", "9737", "14128", "37362", "47033", "39440", "10998", "47019", "26389", "32398", "45486", "40975", "43440", "20712", "40987", "44990", "26015", "56708", "11251", "52535", "6823", "15347", "5005", "2018", "57186", "4364", "38887", "26518", "14125", "11378", "12472", "26580", "58077", "19382", "54446", "176", "13726", "48681", "28723", "39122", "4051", "32910", "59407", "47758", "18498", "57535", "51826", "38006", "44171", "16997", "50794", "42067", "2585", "3947", "35919", "27786", "1965", "55191", "37617", "37025", "24685", "9949", "17868", "38831", "55832", "38156", "48618", "58803", "1983", "10114", "44898", "45596", "41590", "58897", "16282", "7714", "16636", "41720", "31884", "31808", "27288", "4853", "27557", "13703", "5674", "42096", "31634", "10651", "54049", "46797", "28163", "37302", "14979", "41973", "1805", "16950", "22179", "24472", "16274", "59364", "26807", "14605", "28337", "50618", "23580", "8511", "55513", "20914", "3109", "26174", "50705", "51130", "35640", "4809", "55851", "58302", "11799", "34376", "16847", "55374", "35646", "56510", "18308", "31122", "33210", "7039", "11654", "5251", "12960", "45684", "45009", "13386", "26376", "8713", "38048", "19073", "51323", "48750", "11906", "120", "58611", "11055", "9730", "33902", "15399", "45507", "20868", "48257", "32697", "27432", "10517", "4751", "13226", "56414", "33224", "29699", "50005", "8798", "36308", "33963", "4069", "41554", "8148", "4345", "58136", "23504", "59978", "13107", "34388", "55201", "8070", "48809", "23787", "21439", "25481", "48337", "44253", "33319", "53100", "36684", "45686", "51461", "32498", "8646", "2367", "35051", "55836", "876", "4081", "18345", "39399", "40429", "48224", "54970", "43531", "27135", "43645", "206", "23870", "43317", "38682", "7975", "8550", "11056", "24276", "54919", "7427", "13062", "6033", "36669", "33952", "53548", "575", "48007", "57710", "43778", "37463", "38637", "21413", "54270", "33198", "59180", "32138", "55777", "14430", "33014", "800", "45258", "21876", "36833", "39763", "5744", "14760", "56408", "33699", "53425", "42166", "51927", "11202", "51345", "37418", "1664", "32977", "42721", "54603", "2254", "55108", "30288", "49258", "17753", "46993", "18688", "44109", "52016", "38785", "50304", "55038", "16653", "46898", "49221", "16421", "46122", "59879", "10868", "33168", "7861", "32683", "3678", "41791", "22394", "23896", "4238", "21975", "32954", "47760", "15602", "32793", "27695", "35653", "31020", "48273", "20900", "13034", "36437", "10412", "52308", "22196", "13890", "47379", "10052", "43177", "20839", "46696", "51597", "51370", "45165", "75", "9645", "37911", "38460", "9778", "57853", "31428", "8261", "13761", "3310", "574", "24819", "3393", "33480", "2808", "57679", "17397", "45027", "25049", "7263", "34409", "7981", "11974", "40100", "30249", "25983", "6593", "24649", "22467", "14198", "52352", "37361", "53086", "32632", "29863", "41790", "51691", "20789", "37204", "4650", "40022", "13732", "18907", "52523", "4649", "5765", "9098", "48266", "32748", "46549", "59099", "44016", "50970", "31933", "4573", "13306", "48097", "6783", "58519", "52838", "9768", "55665", "32004", "58831", "32172", "10890", "39195", "59438", "37127", "4325", "2494", "40038", "54606", "3237", "51499", "54539", "21046", "8666", "36282", "54867", "16261", "10767", "18413", "29448", "44368", "54401", "52046", "11284", "18201", "168", "3609", "13267", "48700", "4224", "32535", "47856", "4833", "20919", "36900", "1635", "24054", "12120", "29957", "54790", "33053", "50332", "7583", "14933", "42953", "45192", "47734", "19724", "31417", "11141", "33387", "47330", "34533", "2283", "12787", "59849", "5961", "56155", "50940", "27794", "43426", "34212", "56844", "29605", "1365", "2298", "14287", "44923", "49931", "56247", "48118", "58661", "47352", "50708", "46002", "58198", "30737", "35231", "46348", "22655", "10882", "55274", "58623", "22808", "10939", "24666", "45365", "12108", "8150", "25524", "10421", "5634", "28801", "10111", "49168", "4861", "26206", "35479", "39728", "37660", "22622", "28605", "37875", "16738", "16864", "48538", "15677", "48168", "24672", "45579", "32362", "28220", "14219", "55213", "48186", "56850", "44831", "13608", "31997", "37055", "42817", "48646", "21123", "56447", "36909", "46334", "43429", "49417", "58376", "44772", "3692", "56557", "25868", "30209", "13708", "16930", "56960", "10871", "50602", "31766", "32260", "53306", "52909", "49434", "18784", "46214", "56131", "30762", "19088", "50491", "42750", "45243", "42659", "11332", "11110", "21867", "59238", "55121", "55049", "14965", "40664", "39985", "40488", "59679", "30924", "43923", "2457", "6870", "35330", "25859", "54013", "50300", "2237", "7744", "57720", "28089", "13872", "12990", "13239", "53228", "3575", "891", "13059", "33369", "55643", "49834", "54712", "31716", "59896", "59589", "50296", "4474", "18219", "26962", "57953", "4661", "21207", "49596", "56326", "882", "56774", "19693", "43633", "10440", "57106", "59797", "55297", "47610", "38028", "18654", "8836", "18698", "54833", "19905", "4703", "48940", "17028", "16427", "37805", "24901", "9650", "9894", "40959", "32916", "53041", "47057", "53905", "48543", "21657", "50715", "19667", "52311", "41478", "41607", "23164", "1374", "35036", "22456", "34022", "48569", "36061", "12007", "46038", "10940", "39071", "52472", "57766", "23109", "37971", "19518", "14302", "49332", "17718", "55684", "1360", "18378", "50954", "41345", "21514", "5727", "6069", "54421", "51337", "40658", "21280", "32766", "2303", "57900", "32767", "31058", "4209", "48716", "31447", "47137", "52359", "51258", "52641", "2742", "37584", "6244", "59506", "10819", "38174", "22246", "37544", "10588", "19519", "10103", "1889", "51808", "16479", "55660", "7860", "16130", "43594", "14284", "9149", "3296", "35834", "40655", "45916", "4710", "53326", "21909", "31874", "10575", "20753", "59941", "58674", "3040", "25837", "15719", "51650", "45469", "25543", "42582", "10093", "13204", "32550", "14941", "25119", "50233", "58660", "6745", "52319", "31418", "18309", "17592", "43834", "47773", "53914", "31980", "4547", "59712", "17630", "2120", "57273", "12646", "21808", "23978", "5311", "51175", "16090", "47419", "17833", "11856", "21726", "57016", "11170", "27817", "15650", "43064", "45500", "10252", "22498", "33428", "31348", "44275", "2622", "7298", "32660", "43045", "17356", "41173", "56460", "11027", "2053", "55134", "54565", "7445", "16643", "20947", "57906", "33500", "3835", "36019", "13693", "53320", "40337", "54985", "31773", "21567", "1417", "32491", "27602", "14706", "32480", "18122", "17598", "36158", "16044", "16115", "1377", "52253", "26312", "54289", "31138", "43831", "18827", "35334", "30341", "28406", "43002", "59239", "20808", "2779", "47626", "25963", "44931", "39581", "40435", "34850", "4268", "37364", "3411", "22197", "44101", "31505", "3113", "49848", "36154", "46864", "15319", "17649", "6169", "36490", "27566", "39028", "58105", "5320", "2832", "3687", "15062", "30616", "9846", "13018", "45537", "875", "31804", "52521", "28563", "45710", "11317", "5964", "50269", "21798", "57223", "49298", "52672", "7476", "44105", "53012", "12203", "10831", "24154", "16266", "13667", "36794", "11767", "45719", "6781", "26890", "15740", "7754", "8853", "42468", "7398", "40858", "44548", "2395", "51946", "8173", "55907", "44722", "34337", "44332", "40083", "2561", "34502", "19572", "36265", "16160", "19799", "54771", "31390", "44205", "21199", "49203", "2311", "51070", "478", "48213", "44843", "37793", "21094", "45178", "48953", "40094", "47685", "29188", "43030", "40876", "832", "10738", "56063", "41429", "2658", "14822", "23134", "38732", "1107", "29668", "45275", "25949", "9812", "21082", "4328", "9625", "19699", "25855", "55062", "40949", "10833", "40572", "27120", "10958", "11883", "37629", "47853", "5328", "2901", "22937", "8579", "44543", "5290", "59979", "11915", "45923", "32668", "55063", "42117", "58051", "1381", "8293", "16352", "5869", "37658", "21621", "22769", "42070", "19849", "27097", "2556", "7567", "35292", "12156", "29563", "11954", "50689", "57584", "55116", "53607", "2418", "697", "39607", "29974", "59532", "49305", "39282", "53629", "50920", "48221", "29077", "55716", "6197", "44224", "33363", "50716", "38647", "522", "3168", "23200", "48360", "30526", "25185", "59161", "55857", "14098", "27226", "5935", "56678", "5407", "37733", "55323", "49138", "39115", "5741", "46363", "43756", "7811", "42181", "43054", "41738", "19220", "24121", "52771", "56904", "33293", "35584", "31238", "22604", "51735", "32182", "33815", "22606", "18242", "45516", "27489", "49528", "49548", "31916", "48535", "10143", "27172", "51368", "57062", "24101", "49725", "54766", "18659", "28270", "6697", "29452", "23685", "34740", "18240", "9487", "24409", "1893", "43970", "51495", "53236", "1238", "51451", "25090", "47980", "36000", "5758", "2678", "52719", "52891", "57831", "25848", "14861", "2587", "1617", "177", "59203", "37024", "498", "1148", "11270", "3529", "48403", "29517", "17022", "46126", "15190", "6046", "29324", "39334", "3044", "10061", "49034", "13499", "49412", "2826", "15332", "16216", "19902", "45398", "41664", "29228", "15193", "56738", "20903", "31344", "33010", "21173", "54249", "51277", "2975", "5259", "32504", "7123", "44023", "54831", "25472", "7968", "7134", "14122", "13439", "35843", "32728", "18902", "54117", "42272", "44551", "13112", "51789", "58310", "33541", "58048", "9335", "16914", "25238", "52942", "44556", "23062", "54797", "28781", "39993", "18897", "20147", "24507", "33026", "24753", "11434", "10027", "26506", "58807", "59729", "37549", "7699", "36715", "56320", "11065", "59864", "19243", "29260", "50971", "54768", "27605", "51313", "51893", "28693", "54431", "58038", "57465", "44036", "44830", "41819", "4338", "41961", "59393", "10718", "19186", "22189", "17025", "35157", "14650", "33506", "55434", "33770", "23210", "4794", "14701", "16518", "52551", "47537", "27498", "54957", "46336", "59553", "36623", "11306", "22685", "53238", "4900", "38202", "38535", "7205", "20613", "26493", "5011", "42118", "15810", "46208", "17828", "54679", "33280", "12523", "40541", "15132", "7053", "6514", "11978", "6810", "24376", "34165", "23637", "54553", "47240", "1336", "47984", "14498", "5070", "2624", "12966", "48154", "40127", "25214", "43266", "41661", "16139", "46120", "52715", "40506", "9010", "50740", "43083", "45790", "57850", "16606", "10154", "58169", "18804", "15065", "28250", "20282", "36438", "17590", "39892", "36721", "3210", "2224", "10284", "40278", "27185", "21373", "44187", "57457", "55724", "23817", "41221", "21124", "57994", "54483", "9064", "25877", "34111", "24946", "56130", "22489", "9577", "38215", "25194", "47700", "7667", "33625", "51524", "14242", "28799", "41884", "36735", "28773", "33968", "26991", "3471", "30684", "28214", "36857", "39944", "44311", "19146", "26120", "12259", "16910", "45864", "54364", "46603", "7891", "5016", "38354", "30481", "24577", "12489", "6798", "55701", "55482", "22627", "56237", "37720", "42177", "55880", "26226", "39600", "36537", "33798", "20119", "21679", "8940", "31133", "6095", "10650", "54581", "33539", "55792", "8606", "52884", "25915", "26772", "46578", "31398", "369", "46985", "47094", "5860", "22083", "48898", "42218", "16250", "173", "18179", "46410", "12204", "44975", "36316", "11965", "22372", "42605", "37879", "50027", "11957", "27713", "34047", "37414", "48434", "7465", "26359", "34233", "40340", "2520", "24816", "35431", "58333", "11742", "33094", "12738", "18406", "1611", "18225", "13847", "21121", "53344", "29364", "42025", "39049", "38593", "43321", "16268", "5625", "59227", "28656", "11433", "10977", "23801", "9556", "23400", "14326", "11464", "20603", "29697", "47499", "21995", "41049", "5116", "7542", "36555", "18661", "31592", "58989", "38692", "25262", "42701", "42139", "36687", "54635", "34828", "6989", "23259", "55204", "19623", "56802", "44048", "27727", "16415", "48405", "2664", "15838", "13168", "30926", "25892", "27610", "56795", "44494", "22645", "29840", "7", "12942", "50733", "43835", "2988", "25129", "18496", "16113", "39293", "26783", "57010", "10109", "51014", "41338", "56339", "12253", "23762", "10250", "5608", "10019", "2852", "22270", "16202", "33904", "8306", "8162", "21345", "21701", "18288", "11935", "50047", "43665", "1186", "4828", "19414", "17292", "59452", "34058", "24396", "37563", "49616", "59390", "54389", "10681", "6778", "8396", "19309", "41246", "48486", "11535", "4926", "47503", "10487", "37785", "35140", "44582", "192", "19541", "30027", "46517", "11385", "51312", "59156", "973", "1743", "52322", "24512", "25989", "54185", "53716", "22517", "54228", "51816", "42849", "56135", "1252", "12631", "58154", "49445", "20274", "59045", "40571", "6433", "19539", "23572", "15992", "31989", "28301", "40360", "280", "39953", "56433", "44913", "33993", "55046", "15213", "23634", "20991", "7415", "4119", "23294", "45662", "21343", "30237", "20929", "31149", "27335", "36125", "23256", "27854", "48570", "39706", "29975", "17128", "55985", "36795", "52921", "89", "21443", "42443", "39579", "44086", "3546", "27777", "47476", "36197", "26096", "54571", "17154", "15955", "21754", "11488", "187", "44054", "37417", "45367", "42119", "44079", "33037", "6149", "59504", "27641", "28435", "52539", "14708", "27675", "53239", "21008", "26654", "24544", "18215", "21341", "24923", "7758", "45989", "8476", "32545", "27933", "20376", "29351", "7064", "10359", "13090", "24629", "49861", "47236", "57410", "11621", "11645", "32819", "13527", "22726", "58566", "16915", "1443", "19394", "29366", "27036", "18170", "18110", "45565", "29521", "15901", "32094", "6341", "4621", "49699", "10860", "58187", "5421", "52029", "19147", "59013", "32814", "51698", "27997", "49179", "5417", "46393", "38121", "58657", "14657", "11153", "54576", "43662", "3052", "3648", "20717", "19843", "12290", "19697", "51761", "19417", "20719", "56322", "3069", "48289", "46216", "27636", "21855", "32354", "32686", "35178", "32773", "42029", "8506", "5229", "46817", "17897", "39605", "12062", "4730", "50588", "22873", "16428", "24565", "47650", "12496", "11733", "38203", "32139", "20660", "49342", "56791", "19170", "21125", "46567", "50229", "9019", "19036", "14155", "49526", "4595", "37111", "22321", "55136", "26809", "49586", "9353", "3906", "50217", "9399", "38576", "24506", "9510", "23929", "54846", "4005", "43265", "52614", "46974", "51304", "18320", "2762", "41571", "33762", "28515", "4082", "57442", "6041", "506", "44122", "13790", "12243", "30742", "19291", "14477", "56388", "17655", "15192", "32276", "12244", "4612", "20474", "40456", "28125", "12592", "5780", "10240", "35681", "4031", "25021", "28613", "6017", "45734", "51059", "13374", "39919", "14334", "4907", "56250", "32248", "50826", "56399", "23077", "2895", "36881", "51133", "29182", "25669", "14024", "11741", "35389", "14250", "51111", "30293", "51801", "59539", "24593", "53572", "24126", "33204", "34116", "54724", "16956", "43080", "51771", "46782", "7915", "47733", "42201", "40577", "42145", "45373", "17457", "1369", "8446", "36793", "57140", "57544", "57507", "1924", "56957", "42683", "54686", "35059", "57533", "28535", "17645", "15797", "18020", "12676", "41387", "33237", "11330", "27249", "28110", "86", "31387", "13696", "2380", "19356", "29781", "13689", "31014", "34098", "45463", "21526", "23101", "2078", "48731", "28584", "58741", "562", "33029", "22995", "24682", "7352", "54206", "59690", "789", "50032", "46299", "50252", "8639", "30161", "56422", "4241", "19884", "11250", "34559", "53286", "24056", "21596", "33788", "35469", "50619", "29404", "24872", "30492", "16611", "58143", "27256", "6016", "35923", "26903", "57497", "5018", "14832", "46150", "35506", "9156", "36713", "41223", "58581", "22862", "51442", "4373", "30699", "47108", "6676", "49064", "43656", "12641", "22094", "20451", "47620", "36087", "15135", "48896", "27526", "26146", "48756", "28237", "23480", "16698", "40739", "57446", "32563", "39562", "13235", "42469", "47052", "18456", "23370", "11079", "5282", "4337", "44856", "37441", "37559", "58359", "11810", "22821", "19768", "9394", "24962", "40528", "1434", "52590", "51658", "29698", "49518", "22278", "58147", "50159", "4458", "58636", "8146", "15776", "30667", "35911", "29843", "33598", "42655", "7275", "34133", "47396", "24033", "31711", "24143", "20755", "3313", "44805", "36020", "28943", "11469", "59089", "29575", "4430", "44925", "36573", "58175", "9822", "27992", "15640", "59570", "54668", "31520", "59189", "30900", "33386", "56075", "42757", "51866", "22833", "19035", "274", "4544", "42144", "5491", "36818", "12065", "23349", "24192", "15183", "22411", "37490", "31789", "5920", "34812", "16873", "3550", "19531", "14309", "8341", "6183", "11360", "51922", "35473", "26697", "59358", "41609", "54631", "54120", "31759", "30487", "4416", "7680", "19981", "306", "37029", "42599", "54729", "20561", "44509", "25529", "6501", "4144", "16810", "9478", "21665", "30723", "58164", "33518", "14183", "21093", "38541", "44615", "45589", "32189", "39359", "27469", "2071", "1518", "32557", "55219", "29808", "57674", "10295", "40363", "21499", "6361", "37611", "21800", "2680", "1142", "45399", "18928", "51659", "8842", "58457", "3303", "23078", "23473", "29779", "24906", "55111", "19212", "37138", "10524", "29585", "8532", "51942", "13538", "42808", "57670", "20175", "51409", "38529", "11208", "59600", "1441", "32765", "36453", "55402", "5327", "35987", "50184", "33103", "49112", "57174", "2964", "56567", "43448", "31241", "24859", "28484", "50164", "56935", "29248", "51252", "5394", "53181", "10132", "16658", "10477", "55572", "48571", "49439", "35716", "33646", "16301", "34738", "8301", "38786", "11874", "363", "20148", "40445", "40179", "52468", "51592", "37022", "24634", "1811", "33227", "14060", "422", "35220", "52241", "25973", "30875", "16213", "6349", "51330", "29547", "23153", "35561", "52231", "31596", "9442", "9104", "45999", "36811", "372", "27124", "51768", "53175", "45530", "31085", "59018", "24059", "52288", "8158", "45487", "22263", "59901", "2894", "37769", "18524", "38534", "989", "3755", "4499", "53252", "28086", "41371", "36431", "42308", "12178", "30504", "136", "38584", "23532", "35958", "3639", "2972", "43456", "4771", "8155", "51417", "44774", "47530", "35594", "4057", "25751", "33445", "16536", "18365", "14097", "58026", "7412", "36226", "52454", "2504", "32460", "57165", "2748", "54548", "14514", "11596", "48202", "42028", "57228", "25430", "36780", "5147", "14345", "42891", "20859", "28894", "10059", "40005", "49743", "20000", "6", "50170", "17525", "6457", "31574", "5102", "37795", "39065", "59836", "19632", "34728", "42326", "9012", "43055", "5219", "22087", "5897", "44388", "32335", "22811", "7503", "38382", "49635", "29900", "8855", "3203", "23766", "41282", "26200", "7815", "24696", "34130", "215", "59456", "13643", "965", "25796", "46842", "42825", "33659", "16412", "21804", "1584", "59023", "59216", "35969", "55075", "1390", "35101", "48010", "480", "28242", "15308", "24468", "6210", "53573", "15795", "34609", "19040", "33000", "6629", "4008", "58074", "57374", "14447", "56733", "26082", "53719", "13126", "25724", "19276", "2773", "59983", "2321", "516", "39729", "54188", "18325", "6001", "10542", "270", "24874", "9250", "53204", "29367", "11753", "10196", "10911", "19497", "39925", "47350", "24919", "9391", "38034", "44618", "4742", "2874", "8640", "36718", "34226", "34677", "48116", "4799", "12299", "27025", "45779", "42112", "13326", "52720", "49281", "2611", "55677", "20873", "19242", "45778", "34995", "56053", "48872", "58367", "32999", "721", "27851", "53324", "47684", "38563", "4322", "14340", "18331", "11291", "1801", "53242", "34879", "13745", "24009", "3702", "15573", "40023", "41432", "8169", "38517", "21228", "9004", "5663", "43156", "57084", "31139", "35609", "49119", "4222", "54447", "26552", "14017", "44704", "279", "37017", "57385", "34129", "37149", "19262", "49347", "53507", "13584", "15120", "38799", "8065", "3387", "57727", "12420", "29838", "40185", "39470", "27181", "39057", "14732", "43106", "51500", "14957", "5046", "17855", "51238", "33156", "35721", "23291", "41603", "14355", "18776", "37425", "25620", "1858", "30691", "32365", "37090", "47479", "5665", "46602", "31703", "49873", "40053", "49984", "59044", "39846", "42406", "43945", "48274", "21209", "34348", "29604", "54125", "6370", "40721", "289", "15637", "43287", "18017", "8036", "56987", "55736", "16363", "5098", "57912", "22033", "4865", "32569", "41953", "52922", "42948", "21045", "14156", "36725", "53990", "38455", "47768", "21728", "42044", "18458", "37595", "22520", "42840", "32702", "43174", "51166", "44737", "47595", "20213", "34413", "48091", "25420", "37894", "45604", "18879", "58759", "7927", "37492", "46201", "10971", "38413", "2867", "56745", "11750", "48028", "32107", "1476", "6436", "25345", "9211", "59773", "31282", "4765", "47043", "7592", "14358", "15014", "45868", "26733", "32240", "13356", "56612", "19597", "41634", "13083", "58993", "30220", "42389", "10300", "50425", "37401", "21488", "30650", "59615", "54063", "22368", "18764", "59672", "42115", "47769", "19759", "41770", "38377", "1214", "11305", "17746", "35656", "16943", "33910", "54116", "44784", "47692", "37749", "50649", "42758", "5195", "56854", "6589", "1574", "35786", "37352", "58425", "39950", "53418", "10403", "36837", "28496", "45685", "26044", "44438", "14530", "5509", "2008", "59048", "20597", "32842", "11131", "53187", "48900", "13400", "34998", "59857", "39676", "18369", "16709", "14375", "30674", "52282", "3284", "56356", "45417", "53471", "6768", "11971", "13909", "4250", "33678", "31290", "8242", "386", "20598", "55491", "11580", "59131", "52371", "14509", "4758", "24491", "56119", "5458", "38738", "28522", "12594", "19710", "58070", "39947", "5969", "41923", "4784", "19433", "41357", "42611", "33266", "16229", "49477", "25364", "11543", "15832", "33075", "29686", "1364", "22894", "4616", "31029", "51328", "13969", "26620", "33589", "19553", "3737", "1436", "15904", "22118", "13737", "761", "16975", "21917", "8422", "24057", "48313", "14855", "59502", "50156", "41792", "1281", "50654", "9137", "27261", "52906", "1863", "2308", "17546", "13077", "50090", "49754", "51355", "5657", "26324", "28683", "14577", "7550", "44605", "170", "26110", "13426", "12724", "13850", "42898", "29377", "47907", "24348", "26994", "11783", "42775", "26355", "13158", "9197", "55034", "32340", "42915", "10372", "40033", "13984", "13382", "52824", "4230", "36473", "36499", "123", "20160", "38060", "21290", "39996", "18428", "23617", "3105", "55243", "54904", "24386", "29177", "29002", "4464", "22521", "27145", "8578", "17970", "30321", "26765", "48521", "57299", "21323", "42726", "42521", "12995", "40188", "16060", "38726", "23192", "46450", "20562", "7803", "25942", "15434", "20420", "42378", "14931", "13317", "27797", "13403", "36140", "18685", "8990", "3083", "37828", "48914", "38526", "34220", "43129", "29720", "45140", "48945", "56357", "43573", "45739", "34638", "49336", "8682", "31004", "57615", "52031", "16010", "24233", "29932", "54406", "29197", "1996", "59546", "58832", "58104", "11302", "11472", "35296", "45318", "16678", "42313", "12916", "15415", "23274", "34872", "30622", "30950", "3747", "40754", "1271", "34639", "40347", "17681", "30470", "48489", "47627", "47383", "39588", "16936", "31412", "39387", "55604", "8542", "19055", "28482", "59817", "345", "11189", "6325", "52729", "15243", "15002", "10034", "5803", "7341", "30590", "53541", "39253", "56411", "4914", "48051", "10769", "8734", "50091", "11350", "5419", "58418", "41687", "6806", "31307", "43053", "15458", "51550", "21474", "32604", "39508", "32838", "40660", "22155", "4787", "27302", "46364", "20399", "26694", "39975", "52708", "23733", "58124", "31206", "31903", "57213", "12356", "4026", "54009", "43634", "40618", "12926", "46303", "35597", "16313", "28731", "46862", "52117", "39370", "12405", "2485", "22223", "82", "45297", "21404", "28599", "18518", "40234", "13773", "2108", "22578", "31992", "58468", "40828", "56986", "31530", "26225", "41897", "3507", "32969", "7462", "46659", "10756", "35210", "59065", "52904", "56622", "13795", "36621", "12504", "15259", "26916", "3246", "52955", "32846", "15456", "26918", "55232", "35343", "50079", "48607", "23338", "57361", "3884", "4807", "49139", "36089", "32243", "15206", "37577", "11682", "6573", "20074", "47761", "8115", "42397", "59694", "24950", "24035", "51362", "46649", "48776", "20083", "35893", "56901", "48038", "43896", "25050", "46105", "3271", "28966", "59824", "5563", "10179", "38405", "47123", "8406", "24327", "18992", "16665", "13786", "44266", "8014", "42937", "59047", "49427", "10329", "57768", "18065", "10272", "43777", "52816", "9998", "21520", "57281", "5043", "108", "39328", "8650", "22889", "48879", "20187", "58160", "25378", "43188", "1544", "6342", "47126", "56748", "57408", "9557", "7833", "44281", "12604", "18280", "59778", "11877", "49757", "43409", "6513", "14102", "52027", "37522", "48415", "21891", "45129", "2085", "31192", "39601", "43132", "866", "38955", "58986", "24250", "44973", "50319", "34244", "28597", "48431", "9210", "51781", "45325", "11668", "55956", "30784", "42679", "57964", "23941", "47374", "1027", "34287", "58283", "55787", "1423", "26785", "10495", "56767", "14160", "15734", "43180", "31467", "24804", "19764", "1273", "34437", "46139", "28504", "8275", "1628", "55578", "6000", "59582", "20138", "23135", "4472", "21819", "21320", "7350", "25511", "57381", "30404", "56470", "37657", "3622", "28946", "32064", "19214", "7107", "56977", "22762", "36545", "51640", "20220", "2595", "53765", "2670", "12374", "14639", "48997", "44757", "49227", "46161", "5462", "40608", "25608", "42446", "57825", "24853", "48396", "28025", "44812", "41568", "35441", "19778", "21893", "39150", "45764", "53897", "6499", "42814", "26188", "16423", "16237", "48847", "44853", "21493", "7009", "8568", "21958", "5587", "42749", "16561", "20042", "2374", "1253", "12663", "26285", "34722", "27050", "38130", "32958", "36896", "50144", "55220", "4847", "51870", "52710", "45924", "34175", "17966", "56474", "47406", "22852", "1695", "24151", "328", "35249", "51625", "26926", "19317", "33144", "19056", "686", "18914", "21707", "48227", "13339", "8754", "12729", "14994", "58690", "16339", "56383", "48955", "2319", "3013", "53814", "44339", "19645", "48534", "57979", "50900", "43655", "46738", "32007", "13823", "11428", "3282", "20254", "42674", "48581", "4604", "2802", "30766", "55161", "57998", "2957", "29499", "32135", "6463", "17079", "39090", "1521", "34158", "46954", "8003", "53443", "6206", "35526", "774", "31774", "56415", "3232", "34262", "46596", "20323", "33684", "45747", "22797", "10894", "57125", "35687", "33852", "59989", "16260", "23171", "2631", "34384", "34644", "38396", "45663", "5948", "9766", "33840", "10528", "8702", "8762", "18474", "15718", "50567", "51631", "53851", "48193", "18790", "42574", "9552", "57869", "37259", "59478", "15272", "28533", "51493", "21753", "23065", "37412", "35075", "54420", "50739", "13174", "59998", "14893", "47356", "12899", "36397", "2602", "54986", "43571", "47250", "44729", "11033", "5877", "3950", "3251", "44807", "16478", "58874", "10294", "28548", "24226", "24488", "54385", "34963", "8801", "38047", "26568", "27829", "52594", "43076", "58544", "56983", "58010", "4497", "53307", "33988", "46544", "23981", "5501", "57626", "44069", "58905", "58062", "46853", "17879", "7535", "30659", "30652", "19844", "53644", "42874", "20738", "37527", "20694", "56353", "12424", "18980", "44567", "8844", "33740", "22759", "13224", "27246", "27311", "19690", "35840", "52588", "55451", "9763", "57225", "17343", "59218", "53830", "24735", "59324", "50816", "30440", "24527", "8851", "42323", "14671", "44983", "3325", "8667", "12994", "47845", "21435", "52618", "17276", "14266", "9061", "11430", "39713", "37612", "21406", "40925", "24229", "57955", "44659", "18693", "40407", "47280", "49259", "45850", "48373", "2006", "56170", "28942", "30008", "38500", "50685", "27147", "45175", "4085", "27303", "18645", "12395", "40313", "28785", "11671", "52075", "20093", "19835", "40893", "50453", "49890", "21215", "25522", "17685", "44584", "12841", "58109", "42110", "33461", "23176", "51352", "57787", "22737", "56132", "15494", "25067", "12439", "16966", "50645", "34903", "28587", "8750", "21440", "41724", "55420", "2580", "20814", "348", "30571", "57612", "4125", "51367", "48233", "35132", "32552", "46783", "32437", "45384", "24422", "5805", "49590", "15412", "24295", "26675", "19951", "37848", "51031", "42870", "37767", "52295", "3966", "38041", "8032", "27221", "34072", "15970", "9201", "8837", "36584", "8782", "2430", "8812", "46162", "49159", "17148", "34278", "4598", "29108", "35459", "9048", "25460", "10637", "58067", "2526", "36946", "32724", "24280", "51799", "23515", "57407", "35068", "39126", "14881", "33935", "23628", "4405", "9836", "46442", "200", "3339", "9663", "39416", "31006", "47547", "1445", "31354", "12653", "35704", "3503", "37296", "36014", "48401", "22397", "23253", "21563", "55056", "23295", "44504", "17087", "33371", "4997", "37253", "37865", "16323", "39013", "8103", "27220", "42250", "15527", "12569", "24713", "54868", "21025", "27852", "10439", "25053", "57375", "48863", "18453", "56560", "58485", "7042", "6376", "51406", "27085", "6224", "48098", "31998", "53776", "57837", "51985", "12298", "34449", "25873", "15849", "17423", "22351", "49825", "139", "41836", "14275", "53913", "44596", "2142", "6075", "3260", "23096", "14161", "36178", "42199", "44496", "29700", "32753", "46065", "49628", "39473", "36695", "17560", "12572", "28264", "30017", "31693", "43102", "56002", "48093", "11832", "45731", "10822", "9223", "29864", "53899", "24042", "2939", "55187", "9938", "54245", "53301", "6740", "10071", "59771", "26219", "7214", "36772", "2345", "55959", "22029", "46057", "57921", "51228", "8495", "1159", "29237", "52032", "47382", "47138", "41046", "29029", "6156", "3699", "59153", "52292", "38333", "16141", "34433", "9987", "11734", "42526", "45452", "17487", "39297", "42459", "54373", "23099", "26233", "9317", "9288", "24198", "5512", "54313", "9913", "51121", "41264", "56428", "56143", "9710", "27270", "27669", "8635", "26978", "29148", "45807", "57126", "32655", "51407", "37397", "35939", "33407", "23290", "24961", "42834", "44919", "497", "59353", "52056", "10416", "16522", "12627", "31194", "36882", "54163", "52772", "8540", "20201", "32438", "33285", "21102", "10425", "10997", "54033", "6389", "57364", "13561", "37159", "10190", "48393", "20583", "51073", "16468", "28893", "6262", "43092", "52356", "37313", "52939", "39135", "11686", "47238", "42002", "40396", "5598", "19543", "10168", "35720", "15333", "40518", "49975", "2286", "37475", "5849", "17977", "24210", "1887", "51526", "56137", "34253", "20862", "10887", "56704", "13604", "24253", "4592", "59208", "33635", "1748", "18014", "12811", "26638", "53876", "4820", "15624", "25323", "10015", "1912", "660", "54582", "47365", "5085", "20784", "12980", "13618", "14745", "39401", "21869", "25250", "45089", "53467", "54168", "24494", "56241", "18284", "58128", "11517", "27393", "39678", "54244", "43522", "53420", "57793", "21847", "17944", "33520", "56936", "39937", "19654", "32345", "46957", "37402", "55812", "10558", "58880", "18492", "44665", "54978", "51303", "256", "25019", "31324", "54941", "18285", "37309", "38868", "47823", "53567", "3887", "7226", "48637", "8119", "54832", "10852", "3087", "59384", "15771", "964", "41404", "2853", "22315", "43830", "40237", "51135", "18125", "1851", "56147", "25634", "35639", "15794", "26859", "41990", "13914", "54981", "57687", "14269", "5114", "3837", "31757", "1792", "29014", "26162", "23914", "52004", "23049", "29516", "38358", "9080", "17403", "7733", "12129", "16330", "39976", "7369", "2499", "35851", "51903", "21041", "20164", "59853", "27131", "3787", "40193", "20684", "47416", "33612", "15352", "56605", "3217", "12849", "20483", "26774", "54503", "32072", "55993", "58396", "55058", "37435", "49639", "28928", "913", "10561", "7903", "50136", "44030", "19809", "22581", "16039", "58145", "59115", "38753", "10056", "26888", "40418", "38575", "52499", "16540", "31852", "42824", "11964", "44050", "51242", "9979", "55728", "14448", "26304", "17731", "33207", "57707", "8600", "54945", "50885", "25462", "31885", "57166", "8783", "29727", "30387", "51745", "9741", "44637", "41211", "6452", "17193", "49588", "48343", "22589", "23320", "27627", "788", "46178", "17493", "3385", "40142", "24720", "29", "9251", "3898", "15603", "15162", "35956", "39780", "1306", "13523", "58628", "51970", "46932", "56625", "26621", "42402", "29511", "22325", "34712", "34805", "7241", "49265", "37223", "57711", "37485", "46653", "42233", "6161", "4983", "59304", "28395", "18423", "8505", "50465", "26589", "43072", "53481", "37806", "58330", "59399", "47003", "51099", "34444", "19546", "21425", "17340", "56350", "47756", "51291", "29023", "47504", "3014", "11192", "37067", "16569", "47833", "31770", "31100", "40768", "27143", "47596", "44903", "37372", "59188", "12256", "57592", "6060", "17888", "42416", "58735", "35181", "24995", "43335", "14427", "58968", "159", "47317", "56044", "45788", "55759", "41793", "20130", "22127", "23080", "33005", "54327", "24798", "12690", "1174", "38063", "57657", "46131", "50724", "58229", "54776", "52690", "22089", "18743", "45037", "49549", "25316", "35106", "53824", "2111", "12525", "46760", "26517", "47405", "19755", "13064", "51357", "26567", "22845", "45810", "9537", "457", "30157", "38826", "4422", "28770", "4777", "46349", "15574", "32511", "15258", "29325", "28481", "38677", "44414", "27", "36448", "35271", "4806", "14482", "49675", "3004", "23836", "30993", "46189", "49928", "10861", "29319", "15459", "21781", "38673", "59935", "20500", "52130", "15962", "59808", "21937", "49070", "9932", "47346", "9999", "15793", "19703", "3807", "47954", "36481", "40737", "29149", "43220", "23073", "10117", "30061", "21558", "29214", "53906", "927", "44292", "2274", "59529", "31539", "27630", "25001", "50514", "4129", "33574", "50806", "30046", "57871", "54229", "41171", "17605", "36708", "37996", "7295", "28384", "47764", "13563", "1315", "51152", "52237", "15804", "59514", "13474", "31824", "15056", "52624", "5045", "51452", "32410", "41104", "11513", "40796", "28686", "6877", "18356", "27931", "15783", "8776", "34492", "55561", "57949", "23852", "5820", "26300", "8967", "43595", "47704", "30805", "0", "23579", "56180", "31983", "49968", "3150", "36901", "38837", "37351", "55950", "5363", "47036", "53870", "32978", "41662", "22795", "8886", "15618", "6319", "53186", "36892", "42973", "59520", "9559", "9380", "28960", "14439", "46107", "45563", "34895", "40017", "1587", "43983", "45728", "29114", "46839", "1457", "10505", "966", "2572", "37902", "41936", "51998", "37619", "33979", "25380", "35789", "42946", "6512", "5400", "8035", "48309", "12827", "25630", "806", "10666", "7065", "44482", "12840", "25525", "1039", "32312", "4998", "13195", "36958", "37646", "47148", "35060", "26407", "24876", "23289", "34469", "58871", "203", "25549", "12987", "6417", "31855", "40710", "59441", "28601", "17143", "52989", "38503", "5049", "14991", "12144", "43579", "592", "12061", "47747", "33724", "12084", "18015", "51593", "44770", "22839", "1980", "2506", "41141", "17902", "15425", "10699", "59657", "1428", "26232", "45004", "52182", "25008", "45929", "8799", "29402", "4563", "18140", "23561", "6456", "16533", "16723", "57264", "27611", "32901", "41831", "36677", "12666", "6038", "29267", "15635", "25563", "13021", "28846", "27259", "43489", "43207", "42821", "48959", "37", "3357", "14998", "32729", "57054", "47811", "5799", "23377", "43533", "53834", "43131", "7393", "58028", "1326", "8675", "14709", "18123", "20615", "16821", "34699", "15758", "53299", "48145", "8827", "21732", "15041", "57908", "38254", "35308", "39835", "12893", "49489", "10230", "31746", "36373", "6725", "46695", "45712", "50666", "58671", "36775", "33439", "28088", "46521", "13531", "1211", "18362", "5153", "57173", "48297", "9806", "20026", "3145", "19552", "55769", "44318", "15699", "31330", "30638", "49899", "45622", "39302", "48767", "46808", "59002", "44252", "44403", "6243", "47677", "27943", "58112", "44067", "2830", "45076", "47717", "51334", "17958", "9490", "23631", "30534", "37163", "18163", "32145", "27218", "33131", "15737", "40778", "16191", "25177", "46478", "17005", "39110", "38111", "53593", "58451", "35448", "9723", "907", "44466", "2091", "53136", "11217", "13570", "26010", "45146", "8248", "28663", "46667", "46032", "19301", "41814", "33318", "36236", "57578", "2060", "29617", "50854", "13133", "34211", "46605", "50582", "59720", "52652", "20496", "44813", "39053", "47559", "44405", "13549", "9083", "38298", "31303", "50419", "32600", "243", "14229", "18447", "376", "53751", "1843", "59", "41493", "34816", "37162", "46019", "44417", "1656", "41802", "48206", "33631", "51719", "53365", "16542", "20488", "29553", "35701", "12984", "1562", "18825", "6986", "16767", "4426", "6532", "7176", "39634", "40935", "31395", "53427", "39353", "17588", "50957", "32599", "45553", "2683", "7307", "3110", "19421", "23510", "32508", "6050", "23902", "1579", "12537", "40184", "56966", "17567", "47090", "38945", "18028", "12946", "50814", "35797", "1410", "50598", "45014", "56729", "36985", "39726", "33665", "54418", "1566", "5832", "18439", "43727", "12252", "2114", "36109", "32358", "17711", "51394", "9976", "38482", "27019", "11646", "2649", "1328", "1196", "43672", "48627", "13087", "22475", "23313", "57416", "53037", "41538", "16958", "38135", "11337", "4915", "6012", "19030", "17313", "44836", "45816", "347", "6519", "18046", "3878", "3396", "23263", "43246", "1017", "44390", "50363", "48510", "57183", "5544", "28545", "35184", "23501", "2732", "52899", "56245", "54237", "39212", "35198", "52651", "13304", "12076", "52082", "13802", "18867", "50773", "25639", "51412", "41694", "37315", "27492", "49884", "8006", "7109", "48075", "38235", "54633", "46207", "13821", "19573", "23175", "11831", "53987", "45233", "31728", "40279", "40013", "41095", "13941", "56606", "18598", "10889", "36904", "11746", "36997", "45435", "23491", "56212", "16230", "37110", "26602", "4545", "31705", "8517", "39191", "39380", "31231", "51869", "8838", "48596", "53111", "24023", "9261", "9561", "34790", "4852", "2518", "8120", "52593", "18810", "29064", "35538", "6871", "9070", "47015", "46209", "52732", "4445", "29782", "4502", "53945", "47680", "54937", "54569", "28819", "52112", "34889", "10560", "53358", "10237", "12638", "50723", "26609", "34415", "36761", "44746", "42473", "13408", "38939", "5435", "21590", "5709", "11717", "38739", "11320", "30437", "34744", "45558", "15464", "28145", "34824", "33779", "29556", "51958", "3570", "38230", "33", "24557", "7653", "1847", "15095", "15528", "49310", "46758", "33498", "15343", "10503", "50787", "50897", "50092", "44842", "11227", "46222", "29927", "39351", "32892", "6130", "25652", "22876", "45030", "13864", "6935", "59909", "24554", "18197", "57764", "7747", "43343", "9196", "50405", "33580", "9224", "44866", "611", "11919", "29587", "10796", "52979", "43793", "35016", "42747", "53393", "20555", "58826", "29368", "36770", "45931", "27839", "52057", "53546", "5061", "34329", "3790", "52978", "23611", "28095", "1535", "20335", "44726", "27522", "26715", "16920", "20303", "667", "491", "10197", "25864", "10845", "21362", "15285", "55788", "54275", "39033", "35319", "58915", "47687", "11034", "9958", "35040", "28542", "6587", "670", "7399", "37228", "12730", "13227", "29716", "15676", "3495", "21504", "59862", "54277", "33279", "43459", "1817", "16177", "56331", "40230", "24085", "51081", "33378", "5013", "42590", "19535", "24831", "16094", "2134", "26562", "28924", "21645", "34581", "220", "31008", "14171", "48737", "11396", "48163", "11486", "5152", "46346", "8105", "25251", "39388", "239", "18144", "1289", "18259", "38176", "43110", "10643", "25705", "24471", "20385", "7902", "9129", "42442", "47384", "43789", "58973", "35921", "44557", "43115", "7212", "9407", "39858", "3127", "40222", "33800", "48536", "58008", "42004", "54647", "47347", "20311", "53116", "32218", "55685", "17606", "645", "24879", "34917", "24275", "39356", "43390", "6726", "17324", "11235", "33629", "52107", "28472", "15696", "37004", "13159", "738", "6309", "31170", "4073", "27883", "54081", "38599", "47956", "50382", "30845", "47894", "49438", "25649", "4538", "53517", "11226", "3861", "35306", "27424", "27192", "26782", "33890", "32339", "9599", "52550", "2058", "30", "22438", "22669", "50684", "8656", "4419", "13982", "22600", "51746", "45769", "43685", "4982", "17573", "48474", "34073", "44528", "43391", "26739", "35837", "14367", "7638", "29308", "50616", "59597", "40063", "21451", "14816", "16796", "46099", "22559", "9099", "14163", "49472", "39078", "40803", "52271", "3001", "3263", "44068", "44797", "31212", "18853", "38029", "4781", "31061", "58808", "59997", "52270", "52860", "4342", "25176", "30115", "51268", "39818", "55036", "26292", "23379", "5211", "49269", "51846", "49911", "53986", "10031", "1228", "51393", "34378", "26121", "48523", "29370", "18079", "26969", "59994", "24004", "46381", "15599", "34815", "33614", "43013", "3416", "12616", "39853", "10597", "19987", "36129", "39116", "44672", "9796", "39007", "4672", "29695", "39036", "49307", "56666", "21741", "26467", "29224", "12542", "57222", "8098", "27487", "47140", "35946", "11099", "36034", "5027", "21597", "14244", "41029", "28356", "1646", "13887", "53350", "57737", "37308", "54352", "18711", "5602", "13637", "49808", "56647", "25730", "39275", "52223", "51616", "25434", "35340", "18597", "51169", "36216", "38888", "12828", "56809", "8518", "49722", "39891", "41319", "34706", "24599", "7842", "13358", "32933", "6602", "6441", "53732", "38633", "8588", "41401", "15728", "26038", "36869", "27234", "48261", "4375", "14986", "45913", "51161", "50621", "16458", "19743", "28809", "6997", "1941", "6099", "51347", "50162", "49002", "55719", "10198", "410", "52363", "48854", "57418", "12697", "48367", "9915", "58868", "36905", "463", "53073", "38210", "37344", "12069", "18013", "14174", "32213", "29621", "46729", "341", "2817", "38790", "5543", "490", "52325", "48100", "25764", "26883", "17448", "18590", "24159", "21560", "24688", "2963", "53518", "37772", "8919", "12175", "51991", "24065", "28501", "24807", "41305", "42502", "35289", "58529", "38635", "50789", "22815", "58131", "46523", "41800", "1486", "5502", "52993", "50116", "41201", "40146", "33883", "21490", "25636", "37080", "21134", "13585", "51814", "18044", "23575", "20179", "26199", "46553", "45227", "24397", "36994", "16573", "33983", "4483", "27227", "59214", "28034", "36237", "55421", "52441", "57818", "33880", "58186", "22287", "45794", "10470", "28752", "58512", "48818", "30713", "33009", "54437", "58810", "52646", "12823", "22298", "16248", "34838", "56947", "21825", "17486", "16700", "7207", "18663", "23354", "40099", "2061", "51742", "11514", "10862", "46487", "6019", "52881", "58697", "3259", "52785", "25996", "54772", "41328", "36357", "3190", "47408", "39612", "15234", "21530", "7994", "27375", "17526", "45518", "286", "45221", "19025", "23281", "44592", "24161", "3988", "33601", "52787", "111", "21497", "38313", "18324", "21698", "1677", "3571", "4888", "44123", "8143", "35188", "20145", "8327", "28096", "3066", "45957", "18929", "53131", "57671", "17213", "50758", "8334", "46283", "27380", "21730", "11846", "12083", "26954", "59509", "6576", "1874", "42743", "2489", "13756", "7182", "14101", "16351", "15811", "50350", "29032", "29106", "6913", "14466", "9178", "37011", "39740", "19435", "32734", "16336", "45283", "39666", "10393", "23773", "58711", "49444", "5628", "5555", "8526", "49877", "40609", "56518", "38086", "53863", "49525", "48087", "9001", "35352", "38759", "43212", "47438", "47970", "36933", "36311", "14428", "14538", "52894", "39637", "52623", "26059", "43893", "57736", "12825", "37514", "55018", "19894", "14891", "22257", "9466", "295", "17950", "4240", "39427", "13156", "58015", "47269", "46533", "35418", "13288", "14178", "31289", "7351", "19049", "36835", "23930", "16066", "53447", "26129", "25093", "18565", "40880", "30584", "52459", "18009", "1395", "25811", "52257", "11695", "46233", "44190", "55048", "27002", "41407", "52315", "52962", "30416", "51670", "33761", "45104", "30774", "55196", "3779", "19783", "22", "18794", "3213", "42394", "14162", "7331", "2502", "41384", "52330", "12181", "23170", "22354", "10083", "7504", "12036", "51853", "46240", "38216", "15486", "49369", "13651", "14261", "55122", "15722", "51079", "14980", "44811", "45139", "30832", "6021", "25770", "12335", "24505", "30565", "18759", "20100", "40743", "26925", "28660", "17539", "3739", "44900", "10893", "39955", "2162", "36141", "10562", "32295", "8744", "7710", "56760", "56973", "38998", "44430", "45605", "5694", "49121", "3312", "15254", "57168", "38155", "18124", "8708", "15102", "20263", "29579", "17339", "37863", "56402", "4866", "41194", "13842", "29807", "44273", "16454", "11325", "45912", "33360", "3732", "49693", "45057", "2804", "49941", "51849", "3990", "25907", "29739", "16162", "3020", "13294", "32127", "9247", "19566", "22667", "44141", "29718", "53630", "43640", "45333", "420", "17248", "6704", "43537", "22126", "23573", "58242", "15157", "48835", "4127", "9160", "24631", "3911", "59611", "58356", "11796", "31150", "27076", "23935", "23964", "24842", "40054", "18860", "53621", "54136", "56672", "46097", "3973", "23091", "18832", "21018", "27006", "9413", "15475", "35688", "58443", "51968", "10430", "22934", "23512", "6920", "34362", "25259", "4173", "16426", "44526", "45648", "45643", "849", "59936", "42146", "49632", "27944", "10140", "45936", "4385", "15224", "32389", "13753", "57916", "18390", "26516", "50589", "16341", "32936", "19007", "55570", "20259", "4019", "32317", "30887", "49333", "21935", "52453", "39992", "42624", "56789", "25105", "11295", "58883", "56096", "22417", "15961", "22366", "11693", "3967", "21793", "54435", "54290", "59193", "14507", "30196", "36458", "56188", "42022", "56080", "47657", "15585", "16763", "30517", "3636", "38227", "21447", "48619", "39914", "3885", "48110", "17174", "4769", "3045", "44647", "5939", "52137", "48586", "36785", "19585", "28661", "8332", "7752", "23629", "36245", "29691", "1917", "43313", "46008", "6367", "24452", "10846", "3724", "17273", "59956", "36631", "57389", "15593", "44381", "41666", "17569", "2563", "49521", "34181", "4968", "30916", "14433", "44342", "36734", "3406", "9395", "18942", "54264", "31222", "42370", "1563", "30770", "53532", "33702", "45394", "21617", "42424", "55535", "23319", "25835", "7659", "57515", "36464", "12673", "37949", "32899", "20013", "19846", "59246", "50644", "6083", "20506", "18756", "50451", "47542", "6645", "21023", "34968", "31658", "36324", "57098", "39435", "29599", "31454", "57141", "28424", "34004", "24201", "58979", "48777", "44163", "13313", "20620", "31172", "42460", "5138", "32347", "39828", "31725", "40932", "51187", "22054", "10667", "15738", "31334", "44183", "39885", "43625", "43782", "21471", "46765", "56888", "38099", "14053", "1222", "56171", "20118", "24319", "32143", "8347", "29824", "49860", "4462", "54048", "45395", "19485", "18752", "46149", "40310", "13037", "33509", "29673", "41469", "2971", "27157", "41902", "1601", "36058", "53871", "26657", "25373", "28532", "30687", "40380", "44735", "48109", "11252", "36272", "53972", "1465", "20203", "4878", "53132", "35281", "53781", "9626", "10095", "29304", "36777", "57630", "36552", "6415", "40679", "6830", "42735", "25711", "55199", "42577", "30342", "1918", "55288", "25032", "17127", "50641", "43585", "707", "38283", "30450", "29652", "49606", "10692", "33414", "9110", "16586", "43258", "7390", "13281", "21453", "39489", "33296", "54882", "1967", "49213", "44448", "27152", "25188", "38075", "33209", "28292", "13962", "11363", "17915", "4626", "46282", "49249", "52856", "56166", "42853", "27040", "54951", "5941", "4106", "29174", "16306", "27285", "29119", "2112", "59346", "18075", "53044", "6443", "37371", "42224", "47703", "2190", "46380", "41518", "46320", "629", "25140", "9367", "35772", "55073", "38984", "58609", "15249", "27732", "10162", "42966", "8063", "38033", "34662", "27262", "34035", "22907", "21185", "51978", "54994", "35020", "51510", "20494", "22000", "53074", "1724", "17747", "46970", "53197", "46460", "48161", "7582", "39280", "48245", "6605", "52083", "38982", "26029", "27342", "53589", "38129", "10254", "6076", "31359", "13758", "20188", "8500", "20069", "59260", "518", "57913", "55954", "38929", "50302", "58045", "5541", "17579", "23767", "26564", "31495", "6340", "3003", "29310", "23851", "19430", "52461", "15063", "24142", "15053", "4471", "46294", "5452", "42403", "19349", "53587", "34666", "22274", "6008", "16839", "3489", "20906", "9507", "59670", "58322", "8005", "50348", "32555", "58922", "42466", "38398", "23489", "7455", "9588", "37430", "39433", "49082", "5934", "14177", "30953", "34707", "20398", "10459", "11047", "22755", "45295", "5914", "17064", "9198", "3114", "50392", "57959", "22026", "24652", "41880", "11929", "40673", "7418", "30139", "56800", "14028", "29358", "3256", "56909", "22609", "39214", "52457", "12068", "10953", "36564", "55325", "34660", "22294", "16794", "54114", "33050", "47540", "40498", "14975", "63", "55531", "1816", "43475", "29207", "33733", "44763", "22240", "54445", "57", "5093", "27190", "39568", "49654", "31699", "14947", "46355", "25181", "29415", "37683", "30268", "31084", "26880", "27862", "35475", "18952", "39032", "21348", "21957", "42299", "47179", "38870", "57493", "41730", "34774", "57202", "18003", "53611", "39352", "32592", "27173", "38678", "34501", "7604", "23492", "27437", "34148", "52961", "54644", "5599", "4697", "44855", "47615", "28641", "1212", "44366", "32085", "13413", "16588", "31039", "47881", "9793", "40580", "55118", "32395", "4599", "9884", "34947", "26891", "45360", "56203", "2265", "43017", "17994", "54738", "6403", "54426", "35691", "37374", "37263", "39863", "4105", "9844", "35732", "13619", "54554", "20049", "32719", "29480", "24309", "40969", "6168", "36679", "14360", "48080", "44214", "42746", "52783", "18874", "47134", "10373", "59440", "54743", "25588", "33149", "49799", "4264", "34481", "22409", "56821", "14789", "27625", "28173", "54529", "21068", "33638", "42295", "4305", "42208", "610", "45455", "31637", "34040", "7303", "46117", "15153", "16934", "43316", "58089", "33370", "25921", "5365", "3707", "46280", "50062", "40449", "47718", "56770", "6553", "31926", "33700", "15087", "31600", "42009", "6791", "32044", "52758", "5839", "15867", "5060", "58360", "9300", "51657", "40736", "26126", "39673", "56142", "19836", "53627", "25299", "29206", "36608", "7987", "49580", "15027", "4382", "52285", "43032", "49846", "24596", "20358", "8234", "39411", "17208", "56039", "46524", "45718", "10049", "54920", "34170", "23035", "48103", "54660", "24839", "253", "57139", "25424", "16469", "23969", "5433", "47162", "23705", "45323", "15685", "28741", "43328", "8375", "20337", "40892", "51960", "951", "13444", "16745", "13597", "54102", "12020", "11884", "45949", "52203", "45948", "1824", "13519", "4432", "45128", "5569", "4527", "22194", "54029", "31962", "4193", "9817", "51259", "40145", "7855", "10383", "56857", "24754", "35143", "50851", "26381", "25580", "23201", "17819", "49759", "19239", "44946", "46973", "2505", "50359", "57179", "16768", "5173", "27179", "41119", "32439", "16431", "33431", "11020", "11611", "32532", "58658", "27580", "33753", "54536", "40470", "48664", "34020", "13312", "28724", "55250", "32959", "25131", "49956", "40247", "51208", "12152", "30658", "59881", "38777", "55689", "3599", "27932", "27000", "54885", "14572", "39721", "52479", "45410", "2210", "35272", "27975", "27874", "1380", "34663", "59064", "58318", "56731", "19899", "20708", "21487", "35024", "34645", "58850", "7753", "8544", "55800", "40870", "25130", "33272", "14095", "46829", "666", "45206", "17803", "17180", "59549", "42708", "45381", "9005", "21623", "51123", "51934", "44689", "45278", "47109", "18887", "28304", "18275", "26050", "55914", "17421", "37628", "26238", "48712", "56781", "41327", "31281", "6026", "45219", "20975", "25212", "15282", "33862", "24176", "38623", "32397", "53694", "33763", "41815", "43372", "44453", "19681", "50833", "56154", "42338", "25679", "36519", "50756", "56161", "27754", "34464", "33923", "25266", "32941", "38671", "4065", "1593", "5906", "13598", "6116", "29279", "55528", "23740", "6245", "52341", "48780", "22141", "7482", "12134", "32991", "27065", "5756", "33432", "1399", "37542", "18789", "21263", "47120", "32701", "48551", "9739", "29515", "13251", "29914", "36626", "9717", "48981", "45761", "58526", "19808", "17508", "50688", "44654", "43004", "55251", "45787", "53005", "7884", "1409", "40477", "643", "49091", "46227", "15223", "32238", "47282", "15471", "24248", "18543", "51013", "41184", "22451", "57214", "19507", "37112", "52509", "33215", "57699", "40282", "15733", "36263", "2287", "27506", "5816", "50431", "51206", "43165", "30976", "44715", "38267", "17106", "5451", "32752", "10387", "12785", "11544", "37594", "53985", "35605", "36530", "30317", "8768", "30521", "26437", "16056", "19596", "51813", "15061", "44668", "9100", "15579", "51139", "24450", "39177", "313", "26100", "19143", "35979", "31566", "41710", "14109", "24671", "13066", "18162", "34450", "8189", "5316", "16762", "51919", "41185", "47431", "44476", "20071", "46405", "57259", "20416", "18480", "15725", "32927", "58754", "33561", "54439", "13041", "29469", "7195", "34331", "40307", "7416", "44320", "58217", "25441", "46288", "2615", "51795", "18323", "30973", "12410", "19392", "32972", "35321", "12155", "4903", "43941", "36527", "4349", "3039", "54911", "15619", "58130", "5715", "29428", "34139", "49967", "35338", "39847", "11411", "57313", "41931", "44697", "53378", "58487", "1963", "42369", "11039", "55691", "50709", "12510", "605", "29783", "42388", "36765", "34002", "17538", "36355", "48515", "25409", "39512", "15749", "10045", "23650", "49961", "16129", "23885", "56424", "34016", "39616", "50822", "1403", "11609", "51798", "44446", "17599", "12041", "58262", "4695", "41168", "57673", "1774", "47635", "20831", "22861", "360", "29274", "9712", "22813", "52893", "27241", "13640", "30623", "57131", "40393", "42436", "36918", "7253", "22040", "24153", "54655", "33820", "36204", "7388", "59511", "19429", "38852", "36699", "54691", "15315", "39870", "24477", "16667", "52565", "4463", "19205", "45681", "15417", "46855", "6531", "56610", "57392", "29099", "10504", "8376", "2977", "29488", "30902", "19920", "37078", "55157", "59719", "27373", "58228", "25324", "37334", "39111", "56832", "53196", "12963", "17196", "36112", "17908", "1732", "24655", "53848", "38560", "19726", "36356", "35785", "40110", "32490", "38791", "36142", "59402", "90", "3899", "58257", "5207", "56173", "11586", "55110", "13375", "53049", "5990", "16076", "10177", "7598", "30593", "9502", "42131", "51002", "11749", "56603", "30929", "30990", "23335", "36667", "53543", "52564", "47412", "19021", "32983", "50167", "35440", "37524", "41152", "6102", "14457", "35810", "53910", "59445", "31078", "17502", "8972", "21396", "31239", "51179", "6719", "44241", "26151", "31858", "36456", "20574", "35053", "5021", "25595", "8140", "26955", "12383", "7651"]], "valid": ["int", ["24863", "34154", "36167", "29011", "42656", "55748", "50959", "34092", "48759", "19370", "18570", "53922", "23093", "32944", "46430", "26142", "10471", "32625", "50228", "8377", "35868", "50778", "959", "41341", "42259", "29291", "34546", "1842", "58053", "30441", "4205", "1050", "6427", "7028", "17695", "48509", "23223", "14315", "39623", "45339", "25552", "18493", "55755", "21336", "8451", "33288", "21769", "976", "29082", "44437", "48194", "12530", "50600", "9852", "54046", "7318", "8973", "43364", "34553", "5306", "46464", "18674", "9101", "35790", "19228", "1925", "55705", "11164", "47410", "20779", "2186", "56508", "18531", "36999", "31082", "47398", "49816", "58465", "59293", "9902", "17210", "29528", "51736", "1243", "18023", "53563", "31739", "31228", "1691", "13298", "59717", "7643", "16579", "39942", "47302", "18886", "44411", "13415", "7766", "46504", "46068", "12954", "2908", "5529", "56836", "29702", "8436", "28305", "36445", "1896", "38849", "6700", "29347", "26236", "59662", "5891", "40926", "42583", "21424", "52681", "41971", "2411", "58774", "19388", "10091", "6267", "31749", "2431", "36337", "53436", "1068", "53784", "53639", "46670", "921", "37953", "11467", "29292", "51887", "320", "33180", "58261", "3723", "50876", "1179", "58027", "570", "49381", "45824", "30048", "6043", "58292", "2051", "53258", "433", "2217", "17289", "31184", "37008", "42548", "47660", "51399", "41174", "18437", "4384", "49773", "26360", "8657", "15488", "18204", "3931", "31687", "2876", "21253", "12034", "9452", "59059", "54443", "23228", "8670", "41921", "12236", "22496", "28074", "44674", "12313", "52621", "42791", "22086", "43729", "13150", "5725", "47521", "29466", "22499", "23158", "53154", "24894", "12099", "52683", "49736", "22469", "30932", "34597", "47053", "7405", "33773", "28967", "45504", "33618", "39957", "11497", "50968", "31455", "28153", "57322", "45170", "15393", "23356", "22035", "4429", "10907", "344", "48390", "37672", "50106", "21141", "46868", "36732", "15057", "52587", "5586", "51448", "14331", "19865", "27563", "2279", "7711", "18594", "47860", "53964", "53645", "3188", "35345", "23198", "33687", "52349", "45654", "5662", "12168", "54547", "10490", "34019", "42230", "41218", "39235", "30603", "8067", "837", "43880", "35529", "9024", "16270", "2702", "44484", "58499", "36110", "36912", "48022", "21254", "32475", "31240", "36096", "23960", "58466", "54560", "6423", "43348", "14966", "43135", "12975", "43706", "33409", "50622", "2728", "41302", "35614", "34274", "4136", "32253", "37316", "35377", "10903", "10335", "9733", "23463", "19773", "15668", "46937", "48385", "28779", "31051", "49776", "5773", "31981", "24549", "25338", "46436", "56009", "18913", "1968", "30051", "19182", "16741", "39425", "39769", "19015", "34201", "53858", "43223", "27453", "31197", "2313", "14754", "23204", "28820", "37757", "54662", "59451", "42567", "39146", "53337", "22629", "37860", "58614", "22112", "1734", "13717", "42602", "35397", "17318", "17957", "38913", "41110", "41293", "32524", "29685", "43496", "38103", "22487", "36605", "44657", "31560", "28654", "9027", "46426", "36571", "54710", "54864", "47401", "56530", "7297", "7875", "14404", "59049", "40415", "12492", "3554", "21103", "22900", "38160", "9408", "56930", "13389", "41840", "43218", "27893", "31432", "57436", "10452", "3395", "43096", "7820", "30112", "33660", "35315", "35166", "4780", "55903", "54180", "52869", "48801", "22036", "31201", "45241", "11003", "9681", "25990", "26622", "8312", "24829", "4333", "11572", "3578", "35933", "2833", "39281", "55481", "43703", "36685", "2878", "58846", "56913", "4654", "29349", "30444", "13003", "22046", "20448", "33632", "44538", "15424", "8136", "49627", "12519", "24822", "29844", "16289", "45554", "38260", "8811", "59108", "47413", "25985", "28883", "19271", "55783", "57549", "31615", "50852", "28823", "13030", "3626", "8318", "35030", "1592", "4327", "2344", "56548", "26655", "17077", "41491", "4840", "52742", "11351", "5989", "14157", "27915", "45993", "4696", "23953", "58553", "33712", "27680", "58269", "5319", "42404", "43881", "55474", "59415", "16063", "24455", "20675", "34302", "57947", "3398", "44711", "37766", "41335", "44521", "23759", "46741", "43394", "39291", "35536", "29608", "35160", "22443", "58818", "40696", "39364", "41048", "32848", "30782", "26345", "36260", "3809", "24815", "17837", "40720", "56611", "5824", "20600", "15584", "38520", "19180", "16686", "58093", "46037", "58275", "26160", "36758", "27275", "5164", "1200", "9435", "30883", "57524", "7863", "22174", "39345", "52985", "37287", "31860", "51260", "59703", "16848", "43886", "49411", "50998", "48524", "10658", "9644", "30824", "43379", "59832", "31565", "11869", "16263", "54915", "57289", "5406", "44167", "35298", "53403", "36299", "33739", "19181", "23055", "26148", "36885", "56616", "28189", "54200", "22308", "14997", "26003", "14618", "54173", "40570", "46036", "27808", "59802", "45254", "26601", "28664", "38394", "37155", "8986", "44413", "16486", "7732", "9910", "38180", "59430", "55835", "46887", "52197", "49198", "12776", "15354", "29254", "407", "28303", "8807", "10077", "42194", "37329", "35881", "16602", "34169", "32009", "5285", "29013", "6493", "33583", "40283", "4815", "47420", "13406", "7961", "35796", "37647", "39219", "13510", "25202", "49499", "8698", "41058", "10942", "42190", "13634", "13047", "48682", "18550", "10644", "48330", "35358", "19144", "18979", "42678", "22296", "19876", "46485", "36666", "13544", "54388", "27918", "29633", "57455", "4979", "40842", "145", "31793", "14429", "40533", "14491", "29730", "8244", "26738", "15368", "59621", "39447", "32786", "7456", "59903", "54141", "28485", "31751", "50075", "15984", "15466", "54059", "24731", "53237", "54658", "6887", "10948", "45752", "56165", "2743", "50486", "45111", "1770", "50699", "37283", "56091", "32826", "46017", "26279", "10060", "44249", "24060", "18589", "16437", "52864", "47579", "57573", "49123", "28097", "47708", "13843", "3516", "34932", "42036", "24120", "10595", "42786", "10510", "47031", "40327", "7380", "8271", "58365", "19538", "10632", "6816", "2084", "44606", "48827", "17207", "6286", "24918", "17140", "10976", "57258", "9687", "29139", "50918", "58521", "1032", "41131", "932", "11259", "48247", "34992", "22028", "22865", "1362", "48480", "57307", "57974", "1517", "32750", "13586", "38234", "13401", "38400", "32237", "16581", "39980", "14267", "15051", "54513", "30240", "10813", "36423", "3594", "53750", "29125", "51905", "19723", "2618", "25022", "55995", "22922", "58298", "43905", "28884", "31442", "46636", "43908", "57742", "25482", "50049", "59943", "46826", "30380", "58385", "17036", "5753", "15388", "29969", "26414", "36883", "2076", "3133", "32113", "42546", "12146", "41889", "41584", "53492", "36440", "46077", "10461", "1720", "27361", "7613", "21052", "45262", "47208", "33549", "51177", "40205", "54613", "19753", "4459", "9597", "20473", "48615", "13684", "43304", "5878", "34579", "53280", "14930", "47425", "28753", "2381", "1499", "38969", "25898", "4895", "57859", "55021", "12202", "4340", "57639", "9953", "7304", "18227", "7694", "24232", "53372", "22226", "32176", "25222", "33447", "33747", "47081", "45403", "6769", "21364", "50829", "31327", "12392", "26532", "37117", "46688", "15015", "28471", "43911", "55597", "13856", "13985", "1932", "52992", "30005", "36955", "1645", "33486", "4417", "28540", "12601", "44157", "7509", "58753", "54193", "18529", "24619", "44544", "8287", "20524", "50118", "56780", "49304", "55430", "42737", "39768", "36563", "41870", "44367", "5703", "44467", "21281", "15743", "25851", "19923", "39430", "55076", "48000", "54316", "53942", "3777", "35048", "57074", "1088", "58129", "49483", "49488", "5757", "35679", "43752", "15656", "54699", "43530", "7373", "40344", "34039", "460", "30582", "29714", "46803", "13269", "53993", "20740", "41736", "48198", "49766", "52299", "3302", "36441", "38981", "55203", "59776", "8180", "7805", "8636", "7627", "51710", "48088", "35263", "35009", "54103", "31366", "52044", "37502", "43158", "27389", "681", "40151", "13026", "29753", "1341", "20430", "41262", "28900", "48235", "45749", "13955", "53380", "39693", "25084", "37387", "50026", "11097", "25456", "9459", "58331", "36067", "41031", "15083", "1719", "7691", "38885", "39419", "59493", "8374", "43698", "40734", "20310", "37174", "41520", "56880", "21363", "16897", "3768", "28758", "38990", "25286", "20211", "38284", "36081", "46744", "32464", "20948", "17615", "59136", "8462", "8091", "784", "21724", "22575", "11389", "10746", "37276", "20554", "50061", "14687", "44206", "18111", "53963", "52572", "593", "22700", "43172", "8796", "35565", "6241", "4023", "30952", "48915", "51119", "1608", "43879", "14640", "50324", "2650", "43310", "11924", "12925", "45973", "21218", "32288", "31426", "35245", "30760", "26837", "10915", "38165", "52419", "56859", "4879", "52586", "9688", "29104", "29687", "21140", "43762", "24891", "41929", "44760", "22947", "3664", "52897", "33604", "35254", "1139", "49441", "16124", "48039", "27804", "3124", "43007", "23239", "1203", "56694", "59739", "57664", "49578", "55357", "35276", "5322", "15689", "13687", "47155", "3309", "55141", "15216", "29162", "22818", "19092", "45346", "11187", "31906", "41948", "14342", "43178", "20907", "1057", "31346", "4210", "2394", "48725", "45722", "49215", "24474", "42558", "18250", "9074", "50768", "5561", "34475", "18795", "55756", "14104", "40536", "22277", "5148", "58509", "16198", "15702", "46324", "7813", "54017", "42157", "36170", "39597", "14143", "32249", "23082", "1168", "6963", "39098", "3447", "45173", "33437", "47170", "14656", "58054", "40446", "34588", "8598", "11087", "54952", "18173", "42773", "31213", "1346", "42163", "48546", "23173", "30414", "43724", "11391", "45628", "7702", "41530", "49174", "31343", "32415", "4303", "36432", "38664", "25419", "45169", "29475", "36644", "44428", "21299", "40727", "14059", "26325", "55333", "29128", "26400", "24146", "1116", "51712", "18834", "31016", "50341", "52200", "22921", "22068", "9256", "50850", "33329", "42393", "51897", "36124", "1366", "59601", "4197", "30518", "48101", "2671", "44688", "22563", "53941", "32481", "57528", "7791", "12526", "3883", "13734", "17211", "53133", "6626", "9151", "16425", "48899", "36277", "7956", "11836", "20221", "35835", "57574", "3734", "48142", "51478", "14481", "28449", "9355", "38301", "26815", "34495", "27952", "2330", "10231", "33250", "49492", "4594", "16462", "12522", "58805", "52871", "9877", "38274", "41356", "13016", "54328", "54396", "55594", "51938", "16773", "59807", "25281", "51829", "27191", "48946", "3479", "24019", "44893", "33568", "54397", "37569", "28541", "31278", "55372", "40219", "2413", "3102", "20415", "49871", "59891", "53966", "38266", "29588", "39551", "59447", "4930", "33080", "38889", "21202", "18612", "26455", "28052", "23651", "31834", "27670", "2816", "32141", "56179", "17942", "27704", "9995", "19502", "242", "13419", "9899", "52873", "47688", "1275", "28760", "31318", "32129", "12554", "57295", "17985", "43860", "14195", "36399", "9715", "4038", "18135", "49279", "14390", "6525", "2999", "3983", "20476", "55083", "48892", "36613", "20205", "25823", "54906", "14616", "24087", "8563", "39799", "51436", "6132", "18988", "43696", "27922", "1583", "18508", "49537", "51810", "24537", "53674", "388", "46896", "26599", "7524", "41463", "24160", "59874", "8871", "25180", "1334", "19802", "5985", "39621", "31664", "5636", "59072", "51293", "39024", "26195", "48507", "20460", "39801", "8418", "11425", "12443", "20271", "13483", "41650", "21013", "3761", "37185", "27140", "29550", "22709", "44418", "57275", "42872", "13652", "13789", "24131", "36638", "6739", "22018", "23547", "37713", "32262", "4928", "52497", "31674", "39422", "14262", "27072", "27368", "30248", "41537", "2469", "18559", "49501", "4978", "39687", "19412", "19017", "25900", "12912", "2496", "53323", "13412", "13830", "29003", "4596", "30537", "18153", "56174", "29542", "24878", "27319", "52517", "17360", "25334", "44934", "51617", "17533", "21302", "11855", "57233", "4154", "1546", "15921", "28669", "55197", "36051", "27202", "8399", "2003", "27757", "4517", "46184", "20725", "33844", "7216", "22077", "48971", "22450", "49297", "2238", "35548", "57422", "27690", "47492", "28028", "361", "50996", "49238", "39766", "39158", "44014", "12813", "46273", "19946", "45694", "44955", "3661", "49751", "5919", "28048", "23881", "28082", "9617", "41066", "20517", "35890", "36689", "10086", "12734", "19425", "16777", "19673", "53733", "29715", "58582", "4739", "48757", "21194", "48412", "53511", "6486", "2271", "14124", "20867", "50860", "46747", "31610", "43312", "24286", "27060", "37457", "56259", "9528", "32799", "50477", "59689", "21304", "3319", "59184", "4213", "43346", "55901", "27470", "12040", "56664", "42760", "15289", "1021", "7737", "10139", "42562", "28446", "5581", "47457", "55446", "34519", "48603", "58005", "33921", "1264", "29886", "52731", "36021", "32938", "28556", "14307", "12262", "42185", "46751", "10135", "37839", "8979", "5388", "31124", "38554", "18133", "4848", "41488", "57234", "50925", "48252", "44583", "8154", "35004", "30227", "26482", "35322", "57337", "50234", "12603", "19175", "11685", "27582", "9871", "4727", "23775", "21960", "6923", "41949", "18636", "35878", "2440", "42356", "20425", "46220", "2961", "7522", "33021", "27771", "58887", "26069", "7036", "17927", "20941", "5309", "24767", "32360", "22374", "1791", "14080", "45732", "27838", "35604", "20373", "28989", "29789", "51477", "23106", "34576", "38172", "51229", "20767", "5187", "17039", "25633", "29295", "5307", "6524", "20507", "14297", "57456", "33731", "5578", "25068", "52843", "35755", "36401", "30512", "4188", "36717", "47289", "8266", "58489", "41879", "21764", "44215", "49090", "51036", "24207", "803", "15177", "23552", "18960", "12131", "51163", "28634", "34472", "22913", "9006", "34983", "38610", "57247", "22103", "28217", "42092", "38940", "23742", "9957", "34142", "42042", "41190", "57962", "16803", "43652", "4488", "29346", "29455", "57927", "3423", "20550", "47663", "291", "30002", "37682", "24475", "57637", "37203", "21879", "45854", "3829", "42979", "52210", "2334", "13058", "48987", "35757", "19922", "47622", "45356", "47056", "21636", "16883", "19446", "9186", "21613", "8631", "33442", "49015", "18490", "17219", "53742", "29012", "27621", "35472", "17860", "54000", "56591", "49818", "9448", "35749", "26692", "55770", "1095", "22642", "47712", "5174", "42924", "40785", "11614", "25070", "10750", "23443", "54392", "45844", "9513", "46592", "3306", "45602", "39626", "5331", "211", "47927", "31671", "14273", "16067", "8671", "20151", "9484", "19915", "47742", "47814", "39117", "28647", "3517", "45160", "43755", "27921", "2952", "35955", "54466", "31652", "22023", "40643", "41420", "3158", "29944", "4634", "7679", "58384", "19825", "34430", "14617", "5909", "15340", "26157", "29565", "43904", "39571", "49074", "20477", "11528", "58604", "33859", "20183", "52780", "19785", "39926", "44901", "48172", "23701", "16005", "20539", "15148", "1554", "4183", "80", "54856", "33457", "17921", "54219", "58276", "28667", "50923", "12970", "34789", "57856", "36767", "42522", "43596", "32136", "19855", "46434", "12607", "30280", "11380", "2029", "48788", "58917", "42361", "50114", "53653", "3276", "2155", "19982", "58914", "50821", "18504", "55002", "49016", "9946", "46625", "12352", "46129", "16532", "43901", "42778", "39629", "36064", "16055", "10160", "32477", "34874", "31636", "6621", "43171", "29911", "39075", "32456", "23060", "59310", "40967", "7005", "3104", "12042", "24930", "18866", "56221", "40610", "6337", "26866", "24266", "36179", "37300", "57262", "12640", "42587", "49027", "110", "23559", "532", "43569", "25865", "2272", "15043", "30796", "25839", "47919", "56272", "12004", "54182", "8182", "42783", "24245", "29087", "44558", "59337", "9051", "25256", "55223", "46228", "8176", "21845", "24043", "36069", "28572", "44656", "32965", "39176", "29938", "27459", "13910", "49727", "45223", "49990", "16006", "49465", "45927", "24124", "25087", "43670", "14487", "30469", "40704", "49505", "10050", "12396", "41473", "37450", "48832", "19769", "50539", "34945", "20786", "37278", "20544", "46362", "27966", "5570", "59234", "44686", "44393", "8534", "4655", "20098", "14631", "6990", "55516", "6945", "32967", "18643", "29601", "30277", "32461", "23719", "7942", "26319", "9244", "46025", "19903", "40491", "8673", "56829", "9989", "38764", "25193", "22358", "48663", "33805", "6915", "13480", "23805", "5377", "23472", "8574", "55694", "5565", "31196", "39517", "54475", "51108", "4943", "48867", "26031", "1627", "28391", "30197", "41998", "26834", "5212", "11784", "16193", "3327", "56635", "8456", "22646", "25112", "9115", "33588", "18218", "17634", "42868", "48686", "19378", "45231", "54165", "28793", "56926", "49398", "54002", "40248", "34857", "29873", "15869", "5142", "4367", "32266", "50627", "55897", "33962", "38147", "58100", "381", "23770", "15934", "3167", "45803", "16210", "43148", "55885", "17841", "81", "1201", "52009", "49396", "1132", "11835", "41982", "33419", "42706", "26341", "22775", "3862", "14540", "45244", "1111", "52773", "28798", "23475", "25108", "3298", "45502", "40941", "10931", "18316", "9853", "58921", "3965", "52738", "55945", "51784", "56232", "5718", "998", "11925", "1415", "12965", "52274", "28811", "31952", "50912", "39157", "2074", "31682", "47601", "24703", "16651", "32596", "18806", "23980", "6339", "35226", "3011", "22661", "48691", "853", "24625", "26841", "59279", "41442", "8834", "56853", "10641", "45277", "8026", "47911", "48754", "12233", "26081", "33254", "30855", "11451", "38640", "32384", "56365", "22676", "19918", "33535", "31178", "41195", "26912", "31890", "20537", "56874", "32144", "16912", "48496", "36160", "21951", "26315", "27559", "797", "54267", "15553", "6751", "18432", "34848", "59077", "30608", "24478", "1679", "47278", "24868", "32693", "53663", "2441", "18373", "48962", "6014", "38867", "31237", "26752", "20821", "20923", "23646", "34669", "47900", "32209", "12908", "512", "19307", "44460", "52123", "55740", "44552", "45488", "38252", "7215", "45755", "37480", "28904", "35235", "53493", "38521", "20225", "27037", "15032", "3354", "39298", "22725", "23156", "54449", "56104", "47783", "49965", "43967", "25614", "787", "1288", "45133", "17909", "32115", "2712", "40612", "52395", "22537", "22171", "22908", "41767", "59232", "40840", "42302", "44290", "38495", "9500", "16657", "52343", "8020", "1319", "35415", "221", "5492", "1370", "32902", "1548", "57045", "45892", "6421", "55248", "43792", "53919", "5885", "8749", "39628", "22938", "46982", "32070", "26024", "55311", "412", "16303", "14179", "46609", "24098", "6780", "16703", "4570", "13526", "50667", "5673", "3638", "28676", "48749", "47551", "39640", "41275", "53174", "8111", "3021", "23410", "1475", "54220", "12815", "39136", "42984", "15298", "29947", "39569", "24889", "30309", "16543", "34316", "32158", "15126", "52345", "6827", "51570", "4298", "50468", "22556", "47366", "17709", "13624", "38128", "47926", "51522", "11720", "23234", "55918", "52425", "12534", "18741", "18811", "32949", "57852", "59119", "57542", "44604", "20760", "53500", "34296", "25957", "28246", "37685", "31009", "53034", "24545", "30562", "8085", "22823", "21014", "26790", "58578", "27273", "36050", "22718", "19630", "18145", "7450", "20988", "5364", "34239", "51619", "1314", "59876", "49618", "43839", "55429", "24091", "21817", "16693", "45007", "31496", "53459", "37014", "46818", "12190", "2181", "34048", "50658", "51609", "6080", "40424", "54521", "41796", "30149", "39886", "25923", "45420", "28408", "48457", "45580", "9571", "47430", "36690", "7745", "22631", "29524", "43454", "38820", "41070", "55320", "16684", "50529", "21217", "26681", "8597", "57136", "41350", "37277", "735", "21830", "51764", "737", "8797", "32219", "37761", "45364", "25082", "16025", "46022", "1503", "34783", "4922", "36611", "59847", "59644", "8926", "36415", "23554", "58970", "26013", "3221", "42498", "58801", "49966", "21509", "40246", "7006", "24053", "45166", "29866", "36781", "23594", "13591", "22051", "20202", "35900", "14971", "43837", "10988", "14126", "34426", "530", "12338", "21815", "6603", "7543", "32454", "20394", "40295", "11747", "45338", "17816", "47849", "30511", "40021", "2965", "57922", "52284", "8004", "17657", "3467", "9868", "34557", "35355", "33528", "39999", "3025", "9504", "26203", "10592", "26936", "14455", "36965", "18807", "29555", "31329", "9294", "46988", "4032", "32185", "20754", "27649", "53847", "12047", "58583", "346", "48968", "43001", "40834", "30538", "22135", "21099", "24974", "26803", "31283", "15745", "23888", "11812", "16984", "35370", "44166", "39703", "50151", "53782", "13825", "29394", "6247", "3151", "50540", "33132", "46960", "46597", "16238", "36773", "39683", "26847", "19765", "9195", "50962", "25305", "54462", "31715", "37578", "28553", "58599", "45603", "52582", "15706", "58975", "50182", "31555", "7794", "30350", "4262", "22870", "19926", "9538", "52166", "34671", "28590", "17512", "32442", "9956", "39310", "11070", "27293", "22748", "34195", "19456", "51714", "6348", "46658", "8590", "13433", "27903", "48787", "25055", "6477", "24443", "56920", "13293", "43567", "11253", "58113", "39301", "14937", "49026", "649", "16259", "56292", "17490", "3649", "20107", "8033", "7091", "20626", "2843", "29129", "17969", "34176", "39563", "47211", "11630", "3436", "18718", "20842", "14135", "20340", "32157", "53064", "59921", "36043", "29552", "33557", "31790", "18899", "51911", "8240", "32918", "52213", "15935", "1348", "48742", "41806", "15129", "13093", "42216", "1145", "33135", "21307", "48985", "1484", "42057", "46631", "26881", "53229", "8997", "12182", "995", "400", "18704", "22917", "55156", "11267", "47870", "18176", "35850", "19919", "27473", "9770", "49511", "23185", "37596", "33855", "31361", "28851", "50446", "26154", "7104", "47672", "57015", "29610", "39741", "20062", "36766", "15922", "36723", "57305", "9792", "14476", "20046", "4124", "43673", "52849", "13027", "19342", "57423", "29523", "43988", "19225", "11160", "50469", "18473", "30082", "3769", "34275", "47665", "51333", "50181", "6718", "8135", "10549", "23976", "32816", "18187", "1836", "606", "3799", "24910", "59371", "59418", "5449", "44602", "45205", "47175", "31147", "56201", "44926", "21778", "10982", "59242", "32825", "48545", "25313", "25808", "25514", "42089", "54024", "13615", "3238", "11269", "2774", "20608", "33235", "8619", "18448", "39285", "40382", "52327", "49188", "11583", "11459", "19126", "31386", "29545", "51235", "24411", "30960", "31497", "47070", "34429", "57924", "41202", "30626", "16389", "21196", "7174", "14846", "22225", "32715", "35756", "58227", "36335", "11647", "39011", "53688", "59745", "26721", "40102", "38231", "12218", "39555", "33311", "30938", "30753", "53144", "1933", "22102", "39296", "47679", "1034", "36169", "15996", "19068", "25896", "30375", "11181", "29460", "24726", "46420", "3308", "208", "40699", "42261", "58220", "20545", "30519", "49468", "37907", "35828", "22447", "58397", "15270", "16034", "34064", "41050", "42588", "27860", "41097", "22276", "24714", "47589", "18118", "40753", "53531", "31627", "30447", "58948", "52136", "4440", "45080", "11113", "22410", "12344", "12229", "57157", "36931", "39556", "23036", "57545", "2406", "9420", "11804", "13334", "35690", "49454", "47375", "12376", "2172", "28076", "44663", "57516", "26656", "30247", "39107", "21032", "34054", "47918", "17316", "55030", "58806", "28559", "51265", "19984", "25347", "33980", "35560", "25063", "5690", "32903", "53043", "6663", "43910", "34845", "59697", "5448", "36310", "38895", "46908", "16832", "29130", "3603", "41695", "13316", "40243", "18782", "45345", "10013", "12141", "3264", "39881", "2600", "11421", "8356", "39164", "15250", "49831", "4528", "9777", "36707", "25555", "32096", "56718", "45304", "39939", "32890", "17098", "55817", "25668", "55805", "16499", "12564", "3314", "22993", "30142", "6207", "55020", "12022", "28195", "25872", "23505", "37054", "28273", "4003", "24034", "51356", "28737", "28038", "49901", "11797", "27476", "23017", "48961", "4581", "48902", "17610", "42435", "1063", "36645", "23831", "29403", "8746", "34045", "53550", "55352", "35247", "51956", "36465", "41088", "43943", "41364", "21912", "5687", "53568", "53469", "53739", "27677", "7707", "26087", "24869", "1345", "45110", "55389", "12322", "23029", "37752", "35108", "6723", "23056", "26149", "36476", "48795", "11825", "40693", "44513", "30252", "39906", "17155", "10634", "22010", "30442", "52143", "28174", "55618", "24970", "19453", "21659", "41894", "8821", "58216", "39421", "9997", "48751", "54594", "41320", "28566", "51038", "18082", "34551", "7137", "24690", "36119", "49660", "28418", "15020", "21586", "14026", "12867", "10400", "58957", "24170", "15863", "23916", "7720", "44359", "47297", "28429", "12452", "7198", "9677", "3750", "21656", "53002", "21884", "38588", "52548", "30400", "41008", "55525", "11122", "51838", "37824", "13814", "50819", "47353", "43651", "16816", "52244", "22716", "41199", "23314", "29441", "52378", "15906", "38368", "24799", "2012", "21291", "7268", "17111", "19751", "18634", "57907", "6070", "11657", "6351", "6891", "30224", "31662", "310", "53217", "34697", "49065", "14733", "30187", "46633", "53121", "49585", "35293", "13951", "43020", "11862", "59243", "1474", "1530", "37006", "12722", "48056", "22577", "11970", "1072", "21773", "21899", "3916", "31472", "31469", "8747", "25803", "25464", "21377", "57370", "7931", "24604", "32430", "19748", "25454", "29203", "40691", "57041", "23177", "44782", "58804", "53693", "52929", "29398", "53076", "50482", "57792", "16328", "49738", "4482", "56390", "38196", "46821", "53575", "58622", "21856", "43043", "29926", "22357", "28854", "7177", "23950", "6092", "38360", "6549", "10589", "21086", "32076", "56689", "29171", "23352", "15209", "40368", "6459", "32656", "31165", "16927", "34374", "56348", "42413", "28834", "51912", "17363", "22926", "50581", "15798", "19247", "2569", "35037", "17603", "46433", "10875", "49291", "43324", "45564", "37239", "23673", "38073", "48020", "19461", "15829", "53743", "23901", "46322", "14379", "56243", "58952", "26635", "7105", "23057", "302", "17875", "10920", "11578", "3584", "46215", "50951", "3165", "23730", "25938", "40504", "7004", "27279", "3415", "53387", "9661", "37275", "11444", "56375", "14856", "51295", "20812", "22114", "35774", "7506", "47553", "42835", "979", "27499", "7140", "56193", "39618", "17149", "48329", "25953", "5056", "18203", "2687", "22904", "54753", "1962", "26971", "38499", "14834", "43145", "6632", "58630", "36383", "42047", "41768", "909", "19533", "28297", "20284", "24338", "21172", "38349", "45290", "20616", "16079", "12507", "22544", "50878", "24188", "971", "39901", "43593", "48468", "34085", "24740", "46232", "36977", "19022", "16591", "55277", "10353", "50369", "57546", "2974", "14527", "32636", "56563", "56308", "3665", "21953", "21988", "23275", "50807", "54750", "41907", "53951", "54308", "45852", "4962", "21944", "2788", "57138", "37842", "8051", "33571", "15929", "25478", "36082", "45631", "6866", "22281", "21412", "54605", "23396", "30514", "25710", "27823", "37484", "31351", "46909", "52018", "8887", "9132", "20612", "35751", "13381", "25122", "8378", "14742", "59705", "58065", "12791", "24998", "57242", "41094", "59833", "28784", "17046", "10668", "18001", "49779", "4246", "8130", "37452", "46329", "57291", "37689", "59350", "10438", "45594", "38457", "16440", "20883", "43622", "48764", "31823", "57306", "26167", "35766", "17935", "47822", "55967", "1114", "26150", "48897", "40515", "34514", "48146", "31736", "45109", "38233", "15746", "58943", "30096", "52464", "44053", "1494", "6112", "5125", "427", "9324", "59892", "41034", "12193", "3659", "37691", "8124", "29766", "24586", "31803", "43380", "26595", "15660", "7822", "57874", "40060", "40373", "39318", "41124", "45177", "50813", "32642", "21733", "53384", "18372", "50006", "4882", "10535", "2880", "26271", "33694", "50692", "35596", "21450", "54708", "55533", "55087", "33177", "58012", "50077", "11570", "46919", "54779", "10974", "33333", "6369", "16430", "13841", "37671", "29501", "30914", "33689", "17328", "35278", "55331", "19925", "45065", "15402", "52422", "17037", "59561", "20002", "16594", "18658", "38569", "462", "21842", "26197", "29100", "28098", "56109", "26161", "52314", "14294", "25816", "41347", "12695", "32980", "7941", "32241", "11585", "59338", "27731", "38533", "17781", "31605", "6363", "17568", "55096", "28341", "18737", "8787", "42670", "31494", "8467", "13123", "51850", "29679", "45215", "54535", "22301", "31087", "4939", "29185", "49303", "39167", "20525", "28367", "36239", "29670", "51149", "48541", "6073", "9166", "35578", "36654", "1505", "23816", "43548", "31019", "14426", "47550", "9637", "19123", "7533", "24058", "31901", "43503", "5330", "2708", "47156", "53330", "59631", "20312", "3077", "991", "17115", "17631", "4259", "56016", "44415", "41032", "27549", "18231", "30087", "13806", "34979", "42152", "16453", "53361", "12082", "38985", "12418", "25682", "26898", "2641", "32309", "3688", "29043", "49494", "59170", "58982", "12330", "54052", "547", "10687", "30493", "32154", "42863", "27199", "44257", "50008", "19655", "19014", "59325", "19295", "16603", "27539", "18606", "55853", "49823", "48339", "49184", "35674", "21515", "39880", "11078", "5784", "50634", "25592", "12843", "33797", "59831", "20956", "44994", "56105", "10622", "50196", "4245", "46777", "42728", "48611", "28399", "52249", "25298", "34235", "21371", "2672", "1416", "7375", "40593", "35574", "20736", "18347", "57740", "44685", "20132", "19716", "15127", "39464", "3106", "22820", "1524", "51599", "25321", "49196", "23535", "29612", "44956", "22753", "55461", "26343", "11673", "28282", "25727", "45049", "14815", "53737", "53662", "58415", "52280", "29261", "3368", "36720", "57081", "52256", "14602", "23988", "42179", "26077", "43691", "45584", "34584", "54062", "7151", "56818", "56021", "48136", "58612", "3223", "45321", "16735", "53731", "57799", "15371", "476", "34883", "24344", "19195", "9690", "12297", "49870", "27067", "8191", "14293", "45084", "13429", "58927", "13532", "43954", "58371", "53386", "38275", "43872", "51196", "40289", "13840", "59686", "49943", "42000", "3422", "15385", "19766", "26306", "25510", "47611", "49210", "50280", "12285", "25258", "47349", "46587", "7044", "5305", "26756", "11429", "50859", "39313", "28682", "26251", "22982", "46906", "54681", "25780", "38828", "40561", "55090", "15314", "14529", "35818", "13646", "31548", "52571", "1523", "36743", "34808", "24503", "10810", "41122", "36700", "20818", "50652", "48676", "5701", "21880", "42236", "44989", "20414", "23683", "27289", "15982", "16918", "17164", "47364", "40098", "24075", "12943", "23655", "19679", "26192", "33599", "10341", "53193", "21985", "51227", "39526", "31979", "49995", "31701", "25333", "22733", "1418", "56468", "58066", "48249", "9584", "57503", "49029", "16265", "28103", "33839", "10949", "5749", "58595", "17521", "21811", "34685", "27011", "30941", "13704", "1687", "13225", "39457", "19997", "10765", "25058", "50148", "37274", "1412", "52435", "10500", "30075", "35849", "49982", "47522", "23768", "11945", "9045", "33774", "56695", "50112", "12196", "41617", "1776", "54023", "17851", "12486", "40750", "35268", "34373", "45934", "24061", "40716", "35313", "299", "17782", "55841", "11555", "53335", "13180", "49594", "26702", "19163", "6538", "35286", "10367", "29219", "1533", "10608", "44469", "44322", "18396", "7112", "10191", "1704", "54423", "16589", "21865", "31681", "5500", "48315", "13253", "23085", "4318", "25651", "16375", "24305", "35912", "899", "19629", "1868", "25369", "9838", "10660", "2055", "24969", "58040", "49080", "52275", "52415", "27277", "56975", "43508", "23037", "20588", "3869", "24937", "11013", "36624", "45064", "13024", "32332", "22675", "36866", "59015", "10282", "48125", "35134", "20685", "39436", "57440", "34057", "31336", "45210", "49750", "4719", "5826", "59314", "8342", "28826", "36627", "6994", "44370", "10025", "48204", "3155", "2682", "12732", "33186", "26235", "46384", "27970", "58909", "25216", "46823", "48355", "16433", "4436", "53327", "37332", "20943", "35627", "50191", "10838", "33581", "34698", "37760", "44724", "21927", "21770", "30664", "25778", "14085", "59829", "33109", "54221", "39432", "4567", "6696", "55071", "9208", "54805", "41170", "36752", "1055", "57069", "41254", "42477", "18156", "9854", "11199", "2909", "39742", "44764", "58646", "16528", "38012", "25924", "55004", "42730", "42523", "31734", "51548", "25091", "22056", "23642", "29885", "27570", "22902", "52642", "12727", "6104", "31294", "11983", "30071", "24934", "56073", "27413", "21633", "7246", "41551", "35610", "25594", "24018", "13155", "51132", "16030", "41863", "1287", "48651", "42698", "26280", "20068", "56814", "26982", "12904", "19793", "43337", "22689", "51324", "53727", "58140", "38948", "38660", "23972", "53127", "22682", "24677", "56631", "24922", "45611", "23706", "25841", "42862", "23600", "24626", "53488", "19970", "47723", "55970", "46992", "21357", "40109", "29749", "30050", "19081", "37614", "51058", "34957", "35300", "34956", "34668", "9799", "21229", "17251", "24813", "11952", "57185", "6844", "119", "3174", "44282", "59753", "13394", "31826", "45025", "47270", "34977", "42635", "23503", "52516", "54983", "49720", "46807", "11309", "16311", "32746", "6996", "51272", "58150", "25587", "47796", "23871", "17736", "41599", "36002", "20724", "59987", "57846", "22275", "57053", "27420", "30840", "776", "32361", "14869", "40220", "43764", "35539", "8849", "621", "50025", "53168", "19458", "31546", "41975", "12835", "40857", "43428", "58163", "49235", "27208", "18607", "58255", "19682", "41244", "15983", "16765", "58144", "18641", "24031", "9549", "31374", "5779", "45229", "53192", "2542", "16331", "17913", "55035", "5117", "48409", "58233", "20097", "38310", "24195", "59043", "27312", "55496", "14674", "26615", "46425", "53891", "26133", "5833", "3454", "39", "2855", "55826", "59347", "43807", "32645", "6581", "21939", "11002", "34711", "31798", "38070", "1501", "41556", "2044", "41263", "3097", "43671", "52696", "14456", "32851", "38345", "49677", "34924", "1760", "14012", "27458", "40715", "28689", "53166", "46998", "14585", "25060", "27475", "18788", "3969", "56420", "50943", "19962", "36663", "47244", "57271", "19264", "25660", "55094", "130", "51233", "15097", "56454", "57660", "33152", "10058", "51008", "51382", "16132", "13828", "10996", "43617", "7458", "30307", "41904", "18808", "13535", "18127", "27879", "38444", "20215", "59395", "35252", "25073", "18894", "20052", "30452", "26202", "21552", "16228", "52279", "23864", "54118", "6619", "57876", "23741", "6461", "53417", "48592", "20809", "42885", "34442", "12652", "49833", "2766", "38845", "43836", "19803", "21595", "34575", "55043", "45560", "16018", "49125", "24208", "43657", "14557", "46616", "36671", "24147", "8627", "25325", "34161", "32271", "51889", "21961", "53877", "16991", "1468", "43946", "27946", "26633", "4086", "52494", "40394", "55296", "5335", "17960", "57891", "38158", "8649", "49841", "52627", "24836", "56688", "53861", "17926", "51977", "28756", "51785", "8058", "29696", "34804", "47069", "43374", "31788", "44227", "8714", "28591", "29979", "52438", "13446", "52377", "33269", "15975", "16743", "26001", "24076", "49110", "20131", "12778", "56843", "19222", "11052", "19851", "42559", "39479", "46755", "36897", "22379", "52911", "19738", "16721", "29483", "59636", "28626", "19544", "47935", "6834", "4281", "53544", "28202", "28483", "46279", "41093", "52000", "37717", "1604", "8028", "5084", "40225", "56639", "25884", "49768", "53869", "30677", "16717", "20503", "1751", "54672", "22927", "143", "5769", "58481", "5170", "8925", "18760", "46620", "52984", "25041", "24438", "6399", "1350", "52759", "23090", "29611", "42060", "38387", "41772", "7834", "37013", "20237", "3741", "40657", "29980", "8712", "31848", "58719", "16798", "46096", "11166", "19607", "6848", "15947", "39480", "9031", "44294", "52650", "3295", "2056", "59327", "55729", "5369", "44629", "20079", "6072", "19064", "48363", "50222", "48779", "7124", "17761", "40409", "2176", "25048", "45698", "23702", "41628", "11247", "41957", "26556", "30114", "21661", "52135", "24742", "36074", "19822", "8978", "35299", "4", "43438", "20854", "45678", "7165", "38126", "30872", "46589", "23867", "30164", "43897", "40847", "43995", "43208", "11934", "27403", "56785", "7879", "31037", "27352", "44031", "14280", "43888", "45218", "29659", "22633", "479", "50757", "16262", "4288", "23227", "48027", "9765", "41422", "27546", "19156", "49283", "21758", "49730", "13782", "15413", "59577", "43898", "5053", "25542", "55055", "50596", "53646", "48842", "12701", "35663", "10773", "21393", "20285", "21085", "9017", "754", "4313", "45078", "25374", "55258", "32333", "18844", "59309", "31485", "23979", "14548", "12052", "54251", "17542", "31832", "39646", "44845", "56807", "2302", "1565", "19359", "25394", "44244", "7789", "576", "56483", "57865", "39614", "16258", "3650", "14918", "17524", "47986", "14081", "42099", "56020", "17216", "44487", "19779", "41315", "13743", "42687", "9057", "27666", "53975", "27233", "56238", "20317", "6946", "32165", "31888", "45222", "30425", "55968", "19004", "21031", "22997", "201", "50390", "22635", "28172", "34764", "11920", "17491", "21074", "51538", "32133", "53597", "1276", "10989", "42505", "23010", "14385", "56720", "30508", "35658", "46091", "16359", "12171", "50314", "31740", "35087", "17871", "40793", "29454", "5510", "24772", "26988", "49844", "50583", "20852", "36650", "3059", "32675", "3515", "21711", "28109", "16255", "5589", "11670", "49680", "20910", "45450", "6362", "11496", "894", "59376", "33954", "57019", "45835", "10574", "23500", "40903", "23680", "19888", "17467", "42335", "12282", "29796", "8956", "25824", "31186", "5963", "38391", "32791", "58393", "32608", "4293", "15764", "26731", "55394", "14761", "38874", "34177", "17907", "16343", "20817", "46866", "59291", "2946", "3424", "16088", "44887", "42198", "47388", "15716", "35466", "20319", "2113", "3919", "18635", "48057", "51532", "11743", "54156", "4243", "55037", "11939", "31258", "2938", "1749", "44456", "55741", "26744", "16834", "58375", "1814", "1102", "31285", "9396", "36318", "49809", "42411", "13291", "12642", "34107", "3078", "41139", "34523", "31461", "51071", "40881", "49671", "50782", "41494", "13722", "44373", "6979", "49798", "29828", "30709", "23264", "57726", "36989", "8081", "52916", "11575", "12689", "30382", "41610", "53943", "9418", "38963", "20247", "27155", "16318", "13335", "13832", "49709", "31512", "46375", "31489", "43309", "44617", "18814", "32382", "37608", "33228", "19610", "28975", "29901", "50533", "30972", "43857", "3192", "20443", "13211", "39524", "33391", "3148", "38728", "16117", "13103", "22027", "47024", "16519", "9377", "47130", "3229", "45660", "26034", "26538", "38956", "52418", "6221", "53796", "42140", "19395", "28986", "32177", "44127", "40198", "24993", "5430", "42573", "47827", "19475", "38661", "24367", "32224", "34794", "31968", "4850", "11775", "58388", "23467", "19418", "46213", "45439", "6641", "31421", "23004", "36405", "14693", "20769", "27702", "10809", "47393", "42226", "4590", "54527", "58978", "4937", "2423", "59026", "29598", "39234", "38059", "25566", "2871", "19628", "19443", "45508", "7230", "22838", "22939", "31223", "50548", "26882", "9544", "43383", "11518", "21251", "53881", "31224", "28794", "32134", "48794", "44338", "4953", "2668", "20419", "27414", "36139", "36574", "52812", "3026", "52490", "12909", "41377", "57231", "17786", "50610", "58222", "45220", "17591", "1929", "55242", "5960", "42699", "32780", "26064", "8724", "16204", "1007", "11212", "20728", "1236", "28765", "44541", "15021", "41256", "2584", "22286", "12378", "56124", "2282", "13509", "13503", "12385", "9887", "17739", "59405", "1876", "20917", "5381", "15701", "20329", "4814", "18603", "51844", "17238", "14393", "53813", "37789", "47157", "18539", "3971", "42997", "43572", "55839", "27523", "44929", "23712", "40974", "28721", "15312", "34397", "18374", "16332", "49045", "10175", "53370", "20981", "45212", "10092", "32214", "15626", "48911", "10896", "2993", "39943", "54698", "15847", "26741", "5233", "7474", "23305", "30047", "11461", "12466", "51552", "17514", "7802", "45186", "21858", "24791", "53265", "38598", "36217", "37381", "16907", "41856", "35085", "15805", "59584", "39903", "33116", "54363", "47084", "14919", "30083", "23193", "57901", "19335", "2997", "28920", "1202", "8048", "22002", "15732", "12915", "9891", "35171", "23933", "11468", "33043", "17602", "33291", "3876", "55509", "55207", "22745", "4165", "28067", "54419", "29876", "34477", "15034", "28137", "2621", "14306", "45932", "24025", "10728", "40271", "50223", "4713", "10273", "24882", "32705", "8053", "59883", "51440", "45937", "35886", "28014", "23195", "55819", "28985", "16392", "15918", "3621", "15628", "7337", "36535", "7500", "16656", "40068", "9437", "7132", "17832", "21347", "12192", "51391", "26295", "30876", "4252", "5446", "46961", "390", "52704", "29672", "7353", "58858", "30351", "17054", "38672", "33310", "7696", "9020", "10308", "23669", "19256", "46766", "43707", "8914", "58170", "49467", "15172", "38250", "39668", "42455", "3752", "35581", "34985", "42594", "58392", "43011", "22915", "19029", "49879", "50351", "8829", "16046", "35496", "10645", "40069", "30259", "13397", "26508", "53656", "31002", "42995", "49571", "9744", "33710", "55763", "40128", "56181", "47835", "55557", "12548", "55934", "37482", "51463", "43093", "44203", "28596", "7675", "43903", "10979", "18443", "23237", "2760", "46535", "27491", "6193", "32610", "59713", "52068", "40598", "24351", "24139", "3283", "19223", "43985", "44510", "51528", "13858", "11536", "29908", "22318", "42869", "13791", "51285", "40468", "625", "10028", "59516", "34994", "24370", "36283", "41561", "22312", "47798", "40284", "3815", "54618", "36812", "4312", "55964", "8711", "48242", "17082", "18155", "7984", "34183", "38505", "33231", "46164", "862", "4450", "16520", "2579", "34389", "25295", "11866", "46389", "24451", "26018", "28401", "52999", "26333", "15164", "33429", "5401", "20351", "13575", "13609", "43875", "45627", "33526", "37928", "13367", "49272", "38307", "12085", "59186", "23351", "12672", "19167", "8785", "58866", "6164", "49005", "3487", "21763", "21176", "39047", "52344", "14084", "9266", "31005", "23792", "53542", "58669", "33443", "32822", "74", "25357", "21133", "17466", "27562", "14325", "51747", "47071", "54930", "24002", "15897", "28368", "50781", "56892", "54876", "34748", "53923", "30951", "48609", "23556", "17964", "53247", "898", "54787", "58265", "45633", "11307", "24887", "41517", "43891", "45935", "44353", "12222", "41737", "40004", "36059", "22668", "53819", "37613", "58785", "53103", "24883", "35980", "11968", "13867", "41864", "22952", "9686", "19074", "48229", "12362", "49962", "42712", "34860", "17413", "38662", "21378", "34914", "59143", "8394", "25255", "17861", "36091", "19016", "25616", "19208", "53189", "22872", "9388", "4269", "27265", "10530", "43843", "26390", "24711", "6774", "12636", "7909", "25708", "167", "12402", "10708", "37507", "59240", "46927", "55955", "30649", "47514", "42685", "48938", "38787", "24347", "58417", "30407", "6103", "54402", "13796", "56944", "52010", "36858", "26255", "47561", "54115", "10138", "48798", "21871", "27929", "45138", "25741", "59754", "31102", "39667", "50175", "28", "19120", "57675", "487", "55746", "36768", "53559", "1285", "20989", "48963", "350", "29178", "29562", "31660", "17389", "8995", "47469", "44278", "6382", "51776", "20019", "53211", "3402", "32337", "5575", "49696", "12448", "1284", "24140", "49099", "11392", "12935", "19797", "5834", "12907", "38098", "49670", "21057", "19722", "48904", "32931", "16829", "19625", "50411", "37068", "59114", "188", "14747", "43586", "21249", "53503", "27836", "57193", "12037", "31253", "7038", "35546", "58617", "50527", "41307", "41011", "36199", "21242", "49998", "2887", "5597", "1707", "1138", "14880", "8166", "29717", "30654", "22470", "16223", "10872", "15616", "42650", "4157", "50906", "24237", "51365", "7025", "55039", "36668", "14348", "22150", "45672", "22393", "51297", "27740", "31371", "54999", "43823", "27068", "26112", "8985", "33343", "38148", "49328", "50471", "6625", "21546", "1873", "45591", "10137", "37852", "5269", "57518", "6599", "34474", "55894", "39132", "15294", "11158", "44471", "23958", "3960", "50869", "24884", "20251", "20011", "22091", "27890", "38813", "1031", "56600", "17469", "23927", "21153", "33729", "3660", "5713", "56599", "40175", "1133", "39418", "36790", "25701", "41138", "5968", "16475", "49105", "19917", "27653", "34984", "34950", "39917", "51239", "55408", "12602", "54564", "8179", "35781", "19160", "17416", "40788", "48658", "20228", "9802", "43240", "10178", "45117", "46440", "3199", "35750", "41125", "53875", "59587", "55651", "7077", "19122", "45094", "54608", "29170", "40472", "53864", "52522", "36878", "7724", "14192", "51351", "46013", "10066", "35991", "29988", "56962", "25817", "44485", "12221", "10766", "56113", "34083", "47580", "45056", "20735", "54830", "44256", "54464", "8009", "27269", "19463", "18368", "6933", "13422", "19651", "34730", "57169", "3317", "29163", "18953", "45158", "22191", "8893", "48331", "16442", "17456", "59633", "53692", "21239", "42796", "27074", "30305", "31897", "41466", "45350", "14237", "39010", "6290", "53618", "35484", "33851", "29496", "43828", "19609", "12091", "8410", "46541", "14625", "55269", "50993", "4062", "14567", "35613", "10074", "16372", "20099", "43375", "8086", "28091", "30066", "58734", "54585", "22082", "8159", "25219", "8602", "35416", "9610", "57754", "38074", "32446", "5279", "16312", "44176", "23806", "29709", "29803", "41165", "26629", "50734", "36757", "47049", "35005", "53591", "42977", "4821", "533", "57340", "27478", "4059", "14896", "26136", "46506", "41092", "43150", "18535", "45620", "24921", "21408", "47630", "23074", "51074", "39357", "54296", "16621", "39394", "3703", "5726", "27795", "53617", "17016", "9468", "44111", "43682", "18171", "6331", "13820", "7585", "18008", "45593", "5573", "38730", "26108", "57517", "25121", "32205", "49035", "40299", "39339", "31581", "46946", "16523", "7117", "46154", "57721", "52128", "51818", "46708", "24014", "22152", "55889", "11519", "12947", "32075", "25519", "54494", "34516", "28574", "9843", "22598", "12006", "19442", "8453", "7849", "41415", "38498", "52839", "39338", "39708", "16811", "36532", "384", "25171", "27379", "16613", "59317", "53521", "55879", "23734", "34425", "7314", "2594", "37092", "2613", "28783", "22971", "35915", "37994", "32155", "29450", "38123", "55398", "40191", "35809", "36505", "27531", "17652", "38162", "58635", "31924", "43040", "47009", "14006", "33919", "24326", "15326", "15375", "56961", "44329", "35568", "56937", "9996", "21275", "54128", "19125", "25231", "33592", "23962", "29198", "56824", "35680", "34875", "59207", "11667", "49288", "51906", "5122", "12128", "4017", "53102", "57863", "33856", "36692", "50378", "23613", "15369", "16023", "59696", "8935", "59474", "11238", "52810", "18677", "49994", "54532", "1286", "56816", "3909", "45303", "16979", "54304", "13752", "42234", "23737", "55569", "47763", "10", "33675", "59828", "6380", "15928", "22309", "40229", "45782", "1160", "15115", "2278", "19574", "12686", "15461", "49753", "14752", "3345", "10042", "21296", "13862", "2868", "45041", "12183", "32351", "47961", "8731", "51288", "24801", "46188", "7767", "8069", "19462", "15048", "37788", "33674", "53499", "46250", "14920", "50762", "7715", "12215", "30869", "15524", "10676", "40733", "2567", "51976", "50407", "7837", "26555", "14810", "41288", "29199", "17441", "28888", "57680", "59191", "30146", "15381", "26662", "1644", "34819", "52361", "7116", "47914", "46060", "43130", "24050", "1849", "11364", "41484", "51162", "36850", "3474", "2028", "447", "31612", "54233", "51686", "30398", "22380", "45789", "20871", "25645", "19385", "51122", "26480", "40273", "37050", "38058", "48930", "14239", "37419", "48924", "7999", "4562", "39813", "55975", "21283", "10244", "54091", "59236", "22731", "39898", "15231", "46690", "179", "53017", "20231", "19314", "20056", "52947", "58637", "12066", "16487", "58525", "55195", "2700", "48752", "34607", "10395", "54962", "37178", "47428", "14521", "6754", "327", "46266", "3902", "11662", "57413", "318", "87", "7254", "48711", "16826", "11151", "25296", "8616", "25336", "25777", "56458", "27578", "21628", "23007", "24744", "56487", "41913", "2571", "5348", "10334", "40315", "19869", "42866", "57495", "24306", "2127", "31865", "59198", "23246", "5928", "18293", "29785", "15189", "7426", "32893", "16485", "58707", "54395", "25054", "23934", "16072", "39068", "55173", "48031", "43694", "58433", "415", "22265", "21506", "24489", "20364", "28503", "18278", "55829", "6970", "19278", "12749", "57724", "29141", "9412", "9638", "15884", "3134", "18228", "27329", "5440", "55998", "19139", "17335", "49455", "28974", "45137", "28607", "6747", "48220", "38866", "46118", "32162", "54329", "2688", "28725", "37605", "11808", "23030", "26436", "42173", "54650", "20771", "25995", "55984", "54459", "671", "59110", "58364", "11847", "6368", "59897", "51595", "53169", "4753", "25769", "12844", "29137", "15851", "28505", "11413", "3714", "31007", "23021", "43288", "19989", "59085", "43912", "9734", "21860", "54661", "50048", "28199", "54053", "22878", "12998", "27322", "50561", "12067", "43486", "38912", "8304", "33098", "14556", "33556", "3538", "7106", "48144", "34763", "26291", "20566", "2617", "7329", "20639", "41535", "2379", "46793", "1736", "43507", "6162", "9220", "14270", "48882", "33736", "46624", "51319", "39367", "1813", "52930", "18295", "15787", "32939", "28810", "57300", "34422", "18355", "12320", "26816", "7997", "19784", "39796", "37687", "31649", "18967", "16856", "34760", "52076", "11622", "1446", "24165", "49148", "39210", "33039", "50076", "31648", "16510", "55228", "57230", "59125", "40223", "30338", "22159", "53686", "34396", "53059", "41771", "51236", "12515", "34370", "56209", "15501", "48446", "44947", "50509", "44767", "47784", "41306", "42291", "48108", "23551", "5350", "46106", "56425", "25012", "6874", "50679", "40333", "13383", "53947", "29318", "7061", "49832", "58379", "17552", "19705", "48817", "14683", "22623", "19499", "57571", "9111", "25197", "35519", "42981", "21154", "6803", "57794", "34155", "25440", "29015", "32684", "8992", "50399", "3819", "56778", "9281", "16209", "2858", "17530", "50276", "40066", "7769", "33322", "26473", "1188", "10350", "56451", "43344", "12941", "36494", "54192", "17688", "13657", "12625", "48391", "6537", "59868", "19302", "40448", "46756", "56593", "58548", "59365", "49792", "30744", "55437", "24763", "30332", "2448", "13775", "35182", "16962", "35743", "8108", "34423", "55640", "4512", "23549", "51411", "22557", "25009", "38972", "8514", "54949", "36329", "46530", "29051", "22848", "31450", "44573", "21853", "4389", "35903", "17129", "11564", "57541", "24135", "4321", "35460", "37157", "51290", "58410", "34941", "46484", "5315", "35325", "23672", "24871", "5036", "56841", "31275", "2510", "59623", "18371", "27998", "40972", "17406", "25104", "45097", "31948", "30978", "41344", "1208", "15707", "15473", "36980", "24106", "32364", "12852", "49495", "57789", "25610", "59820", "48134", "7510", "38436", "2550", "3822", "18116", "32473", "41713", "6856", "24340", "14566", "50682", "31654", "9425", "22096", "11339", "42920", "35390", "20965", "44423", "37305", "12716", "1845", "3478", "50180", "18412", "32663", "18692", "40623", "5089", "35815", "25179", "31743", "34847", "23939", "17998", "11123", "18717", "41296", "10789", "37279", "45792", "42083", "24262", "12744", "7988", "47902", "21525", "44114", "3243", "59550", "21442", "41903", "18924", "50938", "56517", "42066", "22182", "14711", "24683", "21349", "44091", "35111", "13573", "4568", "21977", "5787", "7588", "12546", "25596", "18002", "26848", "25174", "30320", "36630", "51555", "44835", "11563", "2189", "3177", "46239", "33997", "3346", "54587", "34010", "2358", "35056", "31686", "59722", "15506", "4525", "21220", "40809", "10626", "38453", "37487", "26998", "13246", "34119", "9753", "1738", "26445", "43631", "57752", "22510", "38595", "33444", "38294", "3202", "30856", "15339", "23865", "40332", "24456", "43167", "52144", "52845", "52335", "51107", "18112", "27376", "15762", "56618", "18591", "36936", "2477", "12731", "17565", "42815", "36233", "48372", "27166", "54082", "37867", "38104", "15119", "35861", "48416", "2174", "1602", "52198", "31120", "28755", "1357", "49245", "48828", "51579", "50614", "21475", "28156", "36676", "53687", "11283", "48572", "4645", "43765", "11432", "41541", "7976", "25363", "46994", "26550", "35934", "57529", "44718", "36507", "36175", "44534", "55644", "29841", "37472", "59536", "34184", "10442", "51494", "40202", "47152", "33876", "7189", "38253", "8788", "28200", "50187", "29742", "27842", "47576", "29305", "49083", "34404", "21612", "28306", "32928", "51091", "36046", "39198", "43427", "32121", "36947", "29680", "14002", "53297", "41989", "51360", "10306", "56114", "16159", "21908", "5594", "48602", "50625", "9794", "41252", "34071", "46873", "44840", "13430", "39633", "17931", "29050", "38324", "7188", "44619", "43283", "39531", "49997", "59890", "23369", "48042", "6901", "32804", "14022", "49012", "32470", "17330", "2685", "38716", "4335", "8044", "43709", "43242", "39522", "55696", "7430", "2620", "1648", "56753", "19934", "18624", "45858", "26973", "22749", "40211", "42585", "34570", "48631", "40430", "42266", "8696", "5115", "10124", "49948", "10865", "51615", "41890", "50842", "8351", "46287", "36436", "13420", "18113", "48260", "23172", "27092", "29806", "11422", "55890", "5681", "59993", "12247", "13299", "40217", "29397", "23708", "13102", "23286", "49538", "7556", "31047", "33951", "24676", "52399", "27853", "6766", "40496", "24949", "22163", "10227", "41134", "37942", "43570", "49449", "43191", "39936", "15766", "36978", "40632", "16840", "9179", "55881", "19037", "25684", "49577", "54804", "11081", "26147", "25099", "11737", "2407", "46327", "9518", "22336", "16849", "57744", "12739", "1198", "19570", "34829", "47332", "29589", "22516", "12502", "13956", "30179", "42584", "54540", "46718", "46236", "6419", "16970", "23557", "9219", "7284", "59292", "46545", "56176", "25092", "5622", "11234", "34558", "44825", "44522", "28670", "47018", "15802", "49413", "29243", "49075", "38145", "1451", "16206", "31072", "35817", "55962", "39261", "10883", "50738", "28936", "46204", "42959", "7128", "35486", "6852", "36289", "10408", "59639", "49386", "52039", "36319", "5359", "5915", "32001", "4045", "54377", "12792", "27923", "21687", "56304", "21735", "20792", "7729", "25150", "23729", "28011", "20668", "27223", "2823", "28284", "29269", "33142", "37972", "31182", "35846", "54559", "56198", "52675", "51193", "27947", "39776", "18653", "36127", "12573", "22219", "19594", "20573", "34970", "33906", "56752", "31744", "26442", "424", "4800", "4615", "46913", "46685", "15697", "6875", "9304", "11273", "55613", "1568", "21697", "15322", "46618", "6644", "40831", "7043", "21768", "18564", "54011", "43264", "34087", "14950", "59805", "30003", "53336", "22955", "58096", "47206", "36213", "7658", "51015", "31620", "26917", "17126", "26261", "11821", "41412", "32406", "15871", "11612", "44948", "9684", "52803", "25640", "37073", "27682", "36029", "3559", "34751", "18885", "10808", "7231", "35084", "4911", "57690", "10618", "322", "35975", "41111", "50604", "32303", "46654", "11911", "52670", "5522", "631", "6692", "744", "2648", "54287", "18959", "43407", "26413", "12150", "12779", "24945", "51604", "14633", "26469", "29350", "4018", "6608", "10068", "43351", "20896", "26084", "38214", "36939", "50129", "56347", "38839", "22136", "33705", "38144", "49775", "44962", "41901", "14224", "59190", "23760", "20124", "481", "19780", "33905", "46848", "6269", "33089", "17173", "39497", "6200", "16806", "14077", "50761", "38218", "19416", "22471", "18317", "48319", "31076", "4287", "54490", "40081", "13860", "41151", "3233", "17095", "48715", "57272", "59625", "57161", "25278", "42432", "53989", "23403", "37624", "8483", "11209", "50985", "21589", "53647", "59149", "38088", "50370", "27515", "56336", "29115", "1439", "38493", "35712", "807", "38317", "21792", "37849", "25966", "43768", "20040", "31462", "51668", "17451", "20095", "41033", "32396", "44391", "5304", "18106", "3771", "36126", "32611", "545", "55332", "41060", "59080", "5962", "40605", "237", "47858", "33727", "12766", "54143", "12160", "43211", "22401", "9260", "49644", "20791", "19166", "27415", "53062", "31473", "37638", "40573", "33826", "59453", "40131", "617", "555", "33722", "13958", "19098", "21021", "31553", "5907", "19477", "21672", "40137", "4775", "50674", "16309", "45669", "56980", "30700", "43803", "38178", "35764", "7323", "30291", "9539", "34248", "4009", "18676", "51828", "44010", "13060", "23674", "42986", "20764", "46502", "20893", "58352", "31384", "1630", "34960", "54124", "46845", "47899", "24017", "36225", "23353", "18094", "15625", "24958", "7417", "7046", "41719", "22074", "20765", "29374", "14336", "1405", "12206", "33017", "28703", "58650", "50565", "38772", "51080", "10546", "13343", "30310", "36944", "36945", "38570", "36568", "17497", "13493", "23893", "20700", "38109", "32714", "19619", "5719", "16995", "44160", "16809", "4132", "54934", "23357", "25739", "55265", "5143", "28750", "38976", "5414", "32704", "26180", "25264", "54740", "17043", "810", "52559", "5514", "49452", "42535", "33651", "19003", "19424", "11032", "40210", "12752", "801", "20592", "10355", "43884", "54216", "30337", "22579", "17962", "24564", "17793", "39644", "1162", "1110", "26290", "38829", "18684", "43780", "36763", "59141", "47541", "8419", "21369", "50033", "18705", "51090", "12114", "24638", "21303", "5868", "31066", "19240", "32118", "20092", "48825", "12024", "41811", "49603", "8265", "14672", "22215", "19936", "40492", "11703", "36270", "52747", "46559", "9808", "22624", "34028", "6179", "6272", "20584", "6694", "33657", "15711", "28695", "46128", "36840", "33240", "11941", "56779", "50720", "28268", "49937", "56282", "21550", "25647", "50719", "17883", "18656", "20559", "4949", "42385", "40745", "40639", "5916", "40078", "36749", "9664", "5057", "31079", "49177", "40998", "33022", "30844", "30955", "12045", "57464", "43832", "51854", "27039", "48264", "22644", "39063", "36023", "58445", "58777", "43300", "4827", "53140", "41207", "16972", "49151", "50198", "23819", "9619", "9818", "33693", "39897", "45476", "43198", "8077", "21193", "5631", "45016", "11045", "58305", "51361", "7305", "27461", "36349", "12112", "11448", "45763", "48182", "32393", "10210", "20367", "6864", "55642", "8927", "41569", "51113", "54980", "27545", "6055", "29502", "35775", "8230", "52448", "18900", "32388", "885", "50271", "28412", "27408", "30787", "16640", "28688", "41331", "40253", "6188", "17683", "58589", "16846", "13913", "58237", "21751", "7131", "32108", "40813", "31870", "40869", "9755", "13350", "52760", "28745", "37248", "34245", "18224", "54761", "9647", "42487", "16746", "13192", "59783", "29751", "6306", "27401", "47358", "46727", "10760", "3964", "5574", "21395", "30789", "44017", "42444", "1109", "25322", "19627", "39203", "11665", "49040", "13698", "24066", "25956", "58316", "10107", "43964", "44064", "39258", "29235", "56368", "56432", "2233", "52093", "2928", "36987", "20795", "43162", "23001", "16011", "15405", "38452", "19102", "48773", "37237", "11456", "46378", "11345", "53413", "39022", "7619", "36709", "18878", "27951", "58108", "29215", "21614", "38508", "59247", "23808", "40846", "25480", "24244", "32990", "48135", "48852", "57685", "54074", "57331", "37010", "1654", "11061", "17483", "58626", "50914", "7111", "41287", "41089", "47220", "22859", "2299", "14112", "15093", "30651", "58006", "23940", "48858", "56423", "56551", "35055", "48043", "37743", "27502", "19621", "1175", "28201", "26417", "31012", "7070", "40045", "51377", "22057", "52206", "20080", "45408", "32641", "39757", "49704", "23033", "15336", "51266", "14601", "999", "58343", "24535", "57701", "41932", "17461", "21588", "31629", "56234", "25719", "2280", "33271", "52277", "40732", "5488", "34935", "12694", "48989", "28437", "21179", "12612", "4636", "12382", "4576", "11985", "14606", "47952", "35798", "18883", "55706", "50875", "35693", "12758", "9457", "24334", "23807", "29533", "53432", "32883", "46158", "13971", "43215", "12000", "6873", "8591", "6682", "10694", "27540", "36649", "17930", "50193", "26294", "4331", "39064", "12092", "10432", "38380", "10000", "12341", "29030", "32558", "36419", "6665", "45862", "24047", "49060", "3336", "13557", "9522", "22430", "16506", "8246", "53592", "44817", "51250", "20339", "43090", "47806", "56529", "36971", "38054", "14753", "28032", "2487", "51483", "22610", "8034", "27098", "12227", "51408", "7990", "6623", "45184", "27661", "11666", "32016", "56395", "13005", "8572", "49316", "4869", "27235", "45964", "24081", "10481", "55981", "15297", "19229", "11180", "2081", "36941", "27161", "21066", "9447", "23296", "2209", "1619", "14552", "47216", "42565", "12053", "58328", "58309", "44756", "48181", "23217", "51308", "47603", "2724", "36916", "8737", "27716", "43077", "1971", "45069", "4265", "51511", "8891", "3668", "20652", "11277", "56747", "20800", "12605", "52469", "30676", "47669", "18630", "16963", "17146", "32522", "37087", "32869", "40731", "6373", "11096", "47252", "18042", "5933", "43909", "56172", "7276", "2860", "21376", "29365", "21710", "5447", "29994", "31959", "38602", "18969", "9666", "9736", "54628", "16978", "58607", "20551", "36565", "1187", "59168", "34809", "20976", "56486", "28350", "22692", "14974", "4958", "3624", "50213", "41040", "31315", "11904", "53667", "17584", "57839", "40475", "38480", "47743", "53016", "48887", "14795", "26719", "43430", "45444", "23830", "27551", "59398", "36554", "1561", "59575", "40713", "53741", "24078", "53218", "1515", "41024", "17350", "16550", "11271", "47606", "41575", "24858", "19091", "760", "54087", "5082", "4856", "30258", "54089", "40391", "48998", "26070", "59928", "23129", "2043", "53099", "51102", "26737", "39088", "12407", "10550", "59934", "27351", "35401", "23725", "3814", "32298", "23785", "47025", "43114", "16087", "36172", "28512", "12863", "41056", "342", "2414", "8866", "50976", "31561", "15230", "7471", "17209", "56692", "55555", "5484", "53308", "21301", "26042", "17721", "443", "35523", "49423", "24395", "7290", "32806", "28123", "23956", "22491", "564", "30262", "49152", "24541", "20761", "37456", "49951", "10672", "50694", "59362", "40362", "193", "8809", "50435", "40770", "50591", "45288", "40616", "39591", "50605", "55864", "1708", "28441", "30730", "40815", "26989", "25909", "51097", "48803", "13318", "51167", "10263", "28653", "16322", "52191", "43653", "9255", "36443", "38943", "41460", "17856", "41962", "57417", "9283", "20334", "321", "34263", "21120", "36868", "22032", "37330", "47246", "7979", "8699", "9364", "4901", "27020", "40853", "26270", "54243", "21208", "25355", "13109", "17509", "41619", "37346", "43128", "57715", "37730", "58640", "417", "53976", "2371", "24303", "29836", "51231", "5347", "39775", "10245", "7548", "36986", "9573", "39807", "41378", "13811", "21642", "57284", "38269", "12185", "56627", "49569", "19658", "32726", "49496", "39854", "24722", "26999", "44570", "26997", "15846", "14714", "39831", "35463", "14609", "38440", "6651", "49096", "48770", "36782", "16267", "47139", "56266", "40647", "56163", "36760", "59622", "31180", "53679", "58686", "12742", "42006", "25457", "8302", "52970", "32876", "25599", "56267", "25558", "7018", "22122", "28935", "21092", "27084", "734", "4677", "31410", "3557", "15107", "54487", "636", "45702", "3632", "29512", "40930", "19986", "3480", "39437", "57817", "9249", "29008", "49497", "24854", "23698", "22760", "44211", "52026", "29266", "25136", "20487", "11054", "11315", "38617", "1305", "43368", "25977", "26460", "20877", "11823", "58347", "19012", "58259", "58554", "42342", "26386", "52333", "6615", "39746", "29425", "18921", "38007", "41483", "47872", "58670", "49195", "27438", "59363", "26784", "13233", "33219", "25766", "56798", "5921", "21317", "18366", "26350", "52041", "58662", "31453", "48430", "19752", "26573", "21380", "54874", "3760", "48849", "4620", "40089", "46590", "1543", "14777", "6546", "53677", "59179", "34648", "52814", "34910", "12955", "55627", "23974", "52637", "53392", "6238", "48678", "2562", "19829", "25237", "11557", "22681", "37589", "34402", "11837", "13989", "25659", "35661", "43396", "55649", "26855", "34471", "49057", "11341", "39810", "30395", "41470", "4060", "31712", "22555", "7179", "44803", "53888", "35021", "29155", "4660", "47062", "46859", "54626", "9118", "24846", "15303", "6638", "25232", "56815", "6123", "41112", "16998", "28155", "32147", "30354", "57184", "1993", "39817", "20438", "1926", "2821", "22620", "27388", "12958", "27972", "47242", "41627", "31198", "57842", "51386", "30254", "22100", "32099", "42287", "34258", "42345", "2652", "12846", "14072", "12475", "2426", "53799", "45690", "52600", "34237", "4186", "48834", "45389", "51460", "27525", "9745", "17846", "24832", "47826", "5740", "59174", "20085", "42222", "33680", "41878", "58235", "41769", "7890", "55271", "16668", "34587", "55164", "52085", "15262", "1177", "55778", "31516", "43113", "7481", "41540", "32041", "39106", "35579", "3245", "25733", "14027", "20244", "5058", "12682", "2305", "53574", "6280", "1779", "42054", "32231", "56102", "34802", "34346", "8723", "8519", "50256", "28294", "54866", "37615", "35211", "56392", "44308", "12639", "11316", "1622", "4272", "41789", "25656", "16133", "55279", "45581", "16293", "34813", "44128", "6562", "50928", "41582", "45236", "16357", "15096", "46398", "39981", "40816", "45509", "3332", "27318", "19927", "6048", "49932", "28170", "22316", "58608", "41502", "46519", "34091", "43261", "1689", "11299", "32849", "48978", "36515", "13903", "27619", "32952", "14676", "42882", "28696", "37141", "48743", "46869", "55551", "1012", "49558", "28611", "40169", "36447", "32760", "22107", "17822", "51915", "42228", "28629", "56397", "35072", "1154", "5303", "25800", "47912", "53460", "41605", "8151", "8721", "11463", "56153", "36210", "17609", "13541", "3407", "43770", "49711", "16739", "38157", "28182", "16671", "59916", "9124", "56561", "52532", "36244", "49613", "15253", "461", "51410", "31415", "24575", "19260", "21338", "30557", "444", "11135", "54703", "53009", "38531", "21911", "10800", "48913", "21054", "3577", "33035", "18862", "32571", "56200", "31132", "27153", "39377", "44058", "8900", "23197", "58717", "58895", "43157", "25473", "42074", "27445", "36398", "55847", "25346", "17554", "6119", "7443", "40276", "46493", "9689", "34658", "251", "5728", "10484", "33606", "43376", "47293", "51730", "3520", "16397", "14094", "39958", "43485", "35449", "8231", "40452", "52934", "50772", "27629", "12497", "20519", "32171", "58021", "25683", "39277", "2752", "3932", "10710", "56074", "51637", "31342", "43455", "7966", "53069", "10776", "55852", "41242", "37773", "12487", "57277", "29184", "12789", "25439", "11182", "35153", "1137", "9554", "31349", "22936", "46810", "9422", "42107", "29357", "44150", "29153", "10261", "56384", "42116", "57395", "2405", "6542", "11397", "35583", "13028", "26169", "25468", "19576", "13414", "2800", "11263", "10386", "768", "48761", "31611", "15007", "54938", "46527", "35883", "7775", "52512", "44590", "33663", "12911", "30885", "48398", "895", "34755", "20625", "54224", "48011", "368", "6967", "48053", "29546", "39348", "56413", "47143", "7877", "50893", "11210", "48137", "6213", "37724", "34025", "4113", "6216", "41663", "32427", "59132", "30912", "12553", "53854", "34138", "49126", "38695", "14740", "44462", "58460", "10950", "31050", "8208", "28763", "7110", "18481", "41144", "32622", "19293", "56997", "10844", "21933", "3032", "31063", "39391", "44623", "11372", "46243", "1542", "50802", "38615", "44565", "8648", "43037", "11050", "49648", "7243", "57432", "56929", "57605", "42238", "55930", "37085", "54227", "4174", "32754", "4352", "24007", "3365", "59592", "17069", "56582", "53584", "54357", "18824", "14907", "6758", "39495", "59911", "28817", "59482", "42771", "2017", "54088", "12979", "55722", "3107", "59357", "28751", "34146", "17672", "55407", "39232", "35684", "49143", "1176", "55338", "19897", "4836", "57102", "52721", "33160", "20933", "47122", "19337", "19767", "24113", "59385", "54757", "1008", "4611", "17900", "49004", "51819", "3693", "41376", "30411", "6588", "44995", "54020", "29518", "13421", "39700", "26058", "59822", "22529", "34555", "19132", "6294", "50366", "21591", "4481", "20459", "2483", "57082", "6938", "20495", "29984", "44072", "13498", "18181", "59751", "34524", "20718", "14153", "27752", "9066", "40339", "6936", "59152", "18099", "19000", "26530", "45299", "38379", "40677", "20840", "39865", "33313", "13118", "54005", "26648", "44237", "48588", "18954", "48649", "22437", "26911", "5248", "17285", "46623", "15081", "29654", "50903", "23609", "57897", "56465", "30133", "6109", "15447", "43737", "18964", "861", "10864", "51972", "10343", "7126", "48675", "37809", "58767", "10621", "54274", "50030", "46185", "11525", "52546", "14441", "28521", "34976", "45549", "11152", "49003", "13658", "18511", "24721", "36242", "4004", "23748", "49769", "40859", "23428", "31232", "13927", "51567", "52859", "9616", "18738", "32634", "1221", "47039", "41054", "48275", "27700", "50919", "26316", "44364", "2539", "56743", "38676", "16919", "3037", "7717", "24241", "10036", "51904", "18211", "37989", "40907", "25947", "1826", "37719", "355", "57388", "53744", "29301", "4817", "44701", "52312", "11242", "29142", "47231", "51636", "2781", "15426", "34079", "4551", "59598", "39682", "8662", "8280", "5865", "59454", "4658", "46529", "36930", "48123", "41183", "56839", "21498", "46768", "22625", "2225", "23590", "57558", "51525", "12199", "25183", "9086", "26273", "14137", "7391", "53758", "45830", "5892", "51824", "11595", "48722", "44646", "37599", "8628", "34574", "51426", "48040", "52106", "25475", "30725", "44179", "58561", "54991", "13307", "38644", "53707", "50221", "5521", "19924", "44599", "1254", "8664", "49772", "26408", "48377", "27088", "40501", "42784", "31854", "31675", "59922", "10791", "25944", "12561", "40125", "47509", "30897", "1332", "15583", "29201", "38037", "46594", "1527", "18966", "4714", "28280", "57012", "6418", "38331", "58742", "13325", "11895", "23824", "53113", "20930", "6710", "53333", "34788", "16616", "33721", "52431", "50916", "48885", "49066", "43958", "10258", "48035", "44992", "15693", "51899", "4108", "59157", "7265", "6917", "49111", "18011", "55455", "46824", "20697", "31161", "51809", "30106", "28976", "5149", "43630", "23189", "20624", "6129", "12280", "4885", "34029", "49218", "38932", "34569", "27902", "55799", "10242", "17073", "45072", "31465", "52878", "33742", "28133", "7449", "52896", "10048", "45621", "17262", "45652", "28247", "46790", "6379", "21213", "8156", "53604", "32831", "54211", "32772", "39663", "45647", "13554", "11431", "47770", "42197", "32201", "58899", "55027", "25163", "18177", "28754", "30962", "47612", "25398", "14758", "25098", "38027", "5850", "49308", "58118", "59730", "52412", "16711", "30306", "7169", "40966", "37281", "37473", "56311", "54541", "27811", "30579", "39415", "35882", "23339", "27055", "10870", "37486", "31835", "40663", "43480", "10345", "42212", "51947", "51591", "11088", "26488", "28573", "35395", "21670", "8339", "52506", "17702", "17571", "59204", "53552", "54302", "688", "45588", "10975", "20006", "17495", "29884", "48927", "40301", "8652", "5656", "22868", "31021", "550", "33239", "54018", "19740", "24254", "14119", "36181", "31935", "9292", "48958", "13746", "14134", "19691", "9489", "32862", "24307", "15537", "47506", "23834", "41585", "44851", "54158", "24603", "20240", "41679", "683", "52977", "31131", "17218", "53438", "22930", "40910", "47218", "46379", "22603", "43716", "37616", "25806", "33931", "13537", "6971", "41091", "15325", "53232", "52485", "23667", "44529", "43416", "58083", "5516", "15037", "13402", "3561", "2549", "36525", "18868", "3143", "26964", "56027", "13137", "17284", "54683", "58530", "42310", "6356", "42156", "56999", "4323", "59200", "45492", "20248", "12524", "47974", "55091", "3511", "718", "5503", "15327", "56788", "6395", "24752", "37593", "22674", "35162", "20384", "44779", "20265", "43355", "13260", "5661", "51948", "17277", "28159", "49591", "4846", "743", "45484", "19320", "34868", "23094", "17159", "14233", "1435", "1723", "57743", "34467", "4674", "28488", "423", "45861", "9173", "46759", "28107", "33430", "8178", "36822", "3828", "33282", "27448", "44702", "53390", "55424", "57415", "54501", "2995", "55529", "26412", "29474", "35729", "53048", "56298", "53295", "46366", "8226", "53038", "38501", "29997", "55256", "47055", "23005", "43233", "21467", "49910", "5472", "51195", "56310", "42494", "20848", "52520", "2288", "59972", "24924", "13991", "50168", "36113", "20344", "17633", "42114", "35131", "41255", "44500", "34147", "22306", "44100", "47171", "16338", "5031", "58323", "36578", "21197", "30253", "42474", "50372", "8088", "2927", "18206", "21739", "48922", "44968", "43920", "50230", "4456", "20698", "26759", "35838", "42628", "22415", "21472", "49756", "31685", "6498", "37957", "38461", "22707", "4191", "11116", "57023", "49509", "18190", "34266", "42541", "53582", "39592", "11950", "10456", "50083", "10311", "28050", "5154", "33796", "41616", "1004", "22656", "35361", "28161", "48278", "58381", "54095", "14955", "46767", "49530", "22472", "39675", "54696", "39843", "51513", "42306", "20912", "17", "52405", "31089", "34159", "10952", "36384", "59841", "27188", "54948", "46499", "58007", "12027", "16040", "44874", "12671", "1897", "23157", "9721", "39598", "34478", "2025", "19488", "37115", "38462", "18747", "47488", "8733", "13738", "9657", "46968", "47117", "55687", "42857", "35742", "26691", "30958", "25220", "17923", "5692", "39240", "2926", "54871", "6648", "19901", "30697", "13100", "31999", "28238", "33178", "11619", "33854", "36194", "36614", "48308", "24172", "22076", "26919", "17870", "30399", "16131", "15251", "55824", "7677", "1615", "27034", "32227", "27789", "24820", "15633", "17660", "33766", "45315", "18870", "6787", "44407", "12188", "18262", "31921", "58446", "37081", "1361", "21897", "51186", "59522", "31220", "1074", "37336", "41813", "57672", "9181", "21432", "45294", "356", "29815", "44748", "18908", "47272", "21655", "14739", "3627", "1900", "50282", "33533", "52741", "30937", "29091", "12996", "4706", "54457", "28628", "25987", "28160", "5680", "29500", "32377", "33681", "39325", "19153", "34781", "51531", "28618", "38361", "1119", "5738", "48374", "8270", "24178", "50948", "27534", "6218", "23576", "27465", "42364", "31373", "27377", "10892", "50597", "42246", "24723", "11157", "19002", "43524", "53968", "13521", "27624", "33531", "7928", "5293", "43799", "59012", "20727", "35369", "13994", "17674", "57480", "24728", "13392", "13017", "30158", "56979", "56594", "14944", "25303", "36778", "9676", "6618", "49746", "19468", "10511", "45029", "37857", "20369", "31229", "51084", "40956", "7757", "28933", "23439", "34462", "24430", "38277", "190", "22918", "49268", "9519", "7172", "6962", "17666", "20387", "31067", "15396", "44650", "12814", "50320", "17597", "48583", "34855", "56968", "59183", "43123", "8041", "29061", "26730", "33647", "13463", "36838", "18335", "17794", "11888", "7551", "19297", "29059", "45379", "17817", "4675", "6814", "49206", "56677", "9722", "19053", "3824", "20938", "43940", "33294", "6956", "18841", "54643", "26537", "37561", "189", "50338", "50289", "14213", "5426", "36344", "45411", "19742", "49737", "34104", "12992", "32994", "31152", "29259", "25372", "40950", "32635", "19587", "55324", "50461", "21776", "4566", "42665", "17707", "18101", "34778", "7923", "3182", "57314", "5734", "48674", "45978", "58680", "9252", "38798", "18263", "13332", "52053", "43981", "9649", "56751", "11511", "16456", "49555", "57182", "43117", "8089", "2390", "772", "31308", "38814", "36509", "58315", "48874", "52782", "31896", "43184", "41456", "21669", "58142", "27121", "10149", "32323", "47440", "58642", "27524", "46439", "20308", "35209", "4121", "41702", "30459", "12812", "15350", "45446", "13167", "45528", "15703", "31737", "37698", "9067", "28921", "22007", "34693", "32054", "34105", "42938", "59392", "46305", "6728", "45897", "37579", "15526", "32327", "53266", "19505", "46483", "4801", "33175", "57997", "9174", "50502", "26361", "16705", "13443", "540", "45940", "14928", "1855", "37125", "58733", "51381", "24907", "6699", "21298", "39208", "25312", "23598", "22890", "13330", "31256", "14532", "14890", "16707", "8824", "7302", "9849", "23917", "54036", "50260", "39222", "42733", "49365", "26750", "54950", "10664", "49059", "21183", "46854", "53426", "13416", "58747", "38800", "58522", "37606", "14199", "44410", "54642", "15611", "44291", "55282", "53250", "48090", "36418", "18815", "22329", "8061", "56990", "41807", "46387", "58260", "50264", "25173", "40898", "53884", "36267", "9773", "9669", "4947", "6568", "37229", "55940", "39475", "32717", "20983", "18563", "25880", "56544", "911", "5538", "44959", "59988", "57498", "40897", "6402", "24983", "50805", "46317", "41722", "31607", "8498", "11", "42290", "31093", "16584", "25477", "53419", "32828", "43751", "25201", "9964", "15577", "59419", "30009", "37451", "13935", "33709", "53303", "12862", "47869", "48469", "30645", "45090", "11701", "916", "59688", "28348", "42926", "3860", "8826", "44396", "58616", "22747", "59093", "49922", "7925", "24936", "54964", "31119", "13963", "59630", "11185", "14615", "45536", "1717", "52540", "22735", "24421", "22479", "45512", "39182", "53334", "14686", "50611", "937", "54454", "30116", "10155", "54756", "52519", "18305", "818", "3855", "50736", "19405", "51661", "24994", "6248", "16715", "14976", "30281", "1227", "43136", "36304", "20601", "16472", "47667", "13719", "22140", "15568", "35078", "6387", "51792", "55838", "38564", "36208", "47539", "902", "50165", "33673", "37498", "49623", "56307", "48520", "33384", "27064", "40177", "28090", "38620", "6170", "37468", "51181", "46598", "59197", "58756", "5637", "40147", "3525", "9339", "37823", "58610", "3300", "49500", "5190", "21626", "59458", "28258", "18402", "54737", "25277", "16940", "57976", "50911", "816", "17903", "31504", "33316", "19568", "15295", "40119", "58394", "42970", "22590", "53640", "29809", "38648", "27287", "59046", "7945", "55084", "46795", "23977", "59567", "9994", "56118", "46040", "17130", "8000", "1653", "1825", "1580", "48366", "8965", "38690", "47705", "29444", "59559", "49397", "55180", "26454", "22458", "57756", "26806", "28262", "56220", "7779", "46371", "16554", "40767", "56150", "52664", "39613", "56701", "56472", "28958", "20942", "8392", "13244", "45682", "7171", "26632", "16464", "49162", "5357", "17162", "30739", "33226", "17308", "22324", "37256", "11600", "45167", "54878", "56540", "13966", "33757", "15497", "26559", "58565", "30585", "36641", "27023", "1939", "17163", "31127", "8786", "45496", "58256", "3212", "41757", "33036", "27736", "40464", "9492", "31864", "17370", "28068", "23991", "50570", "22132", "42429", "49", "3833", "34651", "47432", "26802", "1384", "12629", "42682", "47516", "5113", "49641", "12159", "5866", "51462", "2976", "16480", "33292", "52557", "19811", "50248", "51926", "38056", "287", "25531", "12389", "58354", "15013", "2196", "15518", "17843", "28882", "15293", "37436", "52957", "59968", "8106", "33401", "56284", "16855", "19486", "48576", "27347", "45977", "19670", "982", "2864", "40490", "45675", "13453", "58407", "49476", "38314", "51957", "8489", "30954", "25421", "30032", "35453", "12660", "56505", "21982", "49474", "22481", "51908", "49933", "43182", "54342", "32817", "45839", "37349", "29019", "13054", "4237", "28594", "6662", "20043", "39173", "7683", "57459", "9347", "33374", "20530", "559", "34343", "38980", "49572", "13548", "9441", "35323", "45888", "40235", "45273", "56087", "6176", "40763", "56341", "26644", "54656", "22552", "12509", "22420", "36262", "11951", "52484", "27021", "37356", "37107", "20144", "3204", "26265", "55138", "23692", "45456", "3612", "54517", "17679", "6490", "38418", "50505", "21019", "19028", "52190", "42769", "51925", "15104", "24182", "38423", "35187", "30560", "14528", "47878", "48707", "53270", "23520", "22081", "52545", "27260", "42400", "58913", "39635", "16944", "21834", "11450", "24223", "28309", "34891", "59760", "15275", "27398", "7478", "45429", "5461", "17703", "32202", "40526", "4351", "42759", "5028", "38272", "16896", "58821", "14142", "18848", "41896", "57731", "40854", "57902", "58440", "9161", "23360", "16817", "2856", "33630", "41164", "19001", "35812", "49155", "26318", "40019", "25910", "1790", "21069", "3444", "4607", "49158", "29921", "41012", "55041", "24952", "22486", "37986", "51805", "10856", "35902", "27425", "16180", "15136", "26014", "28971", "3324", "58736", "38327", "55335", "35740", "49797", "49276", "35349", "19005", "59563", "2409", "22283", "8538", "1035", "38882", "25208", "53453", "28445", "45503", "30896", "32992", "55029", "58455", "6560", "11147", "22338", "32909", "17409", "39928", "10377", "41245", "53928", "20471", "46437", "3998", "30052", "39371", "26669", "8725", "20391", "52017", "1246", "36643", "48369", "4972", "2408", "28965", "47573", "3892", "56801", "49517", "11142", "33267", "1509", "59079", "28787", "48527", "31621", "37827", "9473", "31889", "19437", "41413", "47460", "37712", "43549", "24212", "4399", "23814", "10326", "21374", "29540", "58714", "38578", "52599", "39648", "11684", "44773", "47965", "21524", "22562", "8814", "56452", "19496", "30099", "40298", "30228", "36688", "28169", "11409", "59382", "39185", "36339", "27775", "11828", "51110", "18522", "33887", "36874", "20882", "6173", "52951", "32110", "4994", "40565", "42625", "34528", "14168", "1681", "19634", "55053", "52234", "13568", "22104", "15450", "31542", "10938", "41426", "24746", "5767", "29907", "41572", "41385", "32637", "23912", "57093", "1097", "19374", "3235", "2857", "55902", "33546", "55362", "51046", "37209", "27407", "7274", "55483", "46726", "51349", "27468", "20267", "6018", "42030", "42158", "21868", "11311", "35952", "12377", "59818", "34830", "22854", "10215", "18105", "56690", "13327", "45995", "56988", "14412", "21922", "8319", "31375", "58817", "47903", "21203", "28331", "43462", "33065", "52562", "18329", "16521", "42447", "58974", "59664", "26685", "53789", "23068", "15945", "3410", "51514", "22405", "33895", "37903", "35259", "40895", "43085", "29937", "17169", "21680", "24940", "49220", "14817", "24036", "11387", "6507", "47354", "31230", "5482", "34515", "23258", "3456", "34392", "55698", "52728", "3770", "33350", "34603", "21510", "19613", "19832", "5480", "53241", "51923", "37363", "50239", "26554", "59021", "39204", "11718", "51750", "32538", "25822", "47937", "26647", "35246", "877", "35643", "25103", "52645", "12296", "37131", "13996", "23542", "68", "16308", "45988", "29048", "15663", "28257", "9909", "31277", "7146", "29645", "11186", "10689", "11479", "53539", "56714", "7143", "39245", "11560", "7032", "37404", "45211", "50763", "44152", "16445", "30108", "17125", "40653", "10752", "14248", "29875", "3242", "52348", "22331", "10636", "41277", "22698", "59728", "12949", "47629", "3730", "25698", "48248", "53124", "35932", "9190", "48511", "8905", "16168", "59205", "11914", "8434", "8740", "45194", "53837", "10246", "18912", "19084", "15141", "27941", "16214", "49252", "6468", "23397", "50815", "53720", "16407", "17769", "22948", "42082", "49442", "45798", "55629", "16626", "5548", "35967", "34044", "44993", "11899", "30863", "16600", "36371", "1220", "18799", "22881", "34332", "6424", "7488", "34325", "36661", "26666", "23044", "25273", "20885", "18896", "40303", "8257", "38590", "23034", "51678", "47836", "2373", "43667", "26557", "51011", "54271", "8207", "22164", "6604", "2258", "32296", "20073", "13359", "57758", "57668", "39369", "9795", "54958", "21166", "39841", "19529", "42754", "35461", "49063", "30194", "49192", "20547", "27744", "15200", "18534", "49187", "33913", "14604", "852", "8095", "11125", "17060", "22421", "50345", "29088", "15821", "51251", "52541", "44297", "48420", "29626", "20410", "27481", "38044", "25124", "4905", "42215", "2439", "35165", "20651", "52952", "40791", "45457", "37925", "35458", "9152", "30443", "49470", "44287", "52589", "24808", "21386", "42538", "4140", "30393", "36276", "25248", "5775", "54977", "29892", "50323", "25541", "55842", "35141", "30498", "40088", "16964", "28388", "56940", "19584", "50579", "16673", "55511", "26821", "26569", "14459", "38741", "8139", "25799", "50246", "19725", "21011", "20269", "20238", "3628", "4963", "37061", "26143", "10960", "8703", "5979", "37787", "26274", "49988", "21201", "39319", "44447", "9040", "37242", "22686", "1744", "22824", "15629", "55478", "39589", "43955", "34456", "354", "31211", "27599", "24312", "357", "15410", "44173", "27735", "3091", "54292", "8684", "1331", "41234", "45748", "42518", "13301", "11604", "51997", "26086", "32591", "24393", "39988", "37915", "15572", "25643", "44350", "47916", "22177", "25005", "45915", "6151", "51935", "45302", "5384", "58020", "25654", "41428", "34928", "19850", "30239", "48806", "10073", "31183", "7232", "8720", "25589", "10978", "45250", "657", "40978", "26434", "12740", "3062", "2251", "6922", "28671", "4355", "43919", "24578", "35017", "37665", "39273", "18917", "49821", "28296", "20360", "31821", "5069", "15563", "22694", "32391", "44652", "54215", "20266", "29741", "49782", "47643", "4876", "10174", "46252", "3532", "28135", "28375", "5362", "10772", "38946", "47609", "14783", "57351", "10332", "32222", "44117", "4556", "52997", "46394", "38427", "27779", "41660", "53089", "44878", "41021", "46900", "43614", "938", "36921", "58627", "31024", "9176", "58097", "33686", "10522", "47776", "16457", "14227", "33114", "58061", "6454", "4410", "59965", "56213", "34721", "22554", "54747", "47749", "33967", "41408", "4762", "52052", "11552", "3924", "43708", "32493", "35556", "11658", "47529", "17444", "39657", "11829", "53958", "52229", "57654", "37303", "37170", "1149", "29622", "37427", "35628", "25819", "9205", "12552", "7206", "58387", "12303", "4448", "14870", "57649", "30675", "43690", "35547", "41830", "52394", "2776", "36769", "13838", "4147", "53821", "25575", "10519", "33038", "22349", "21831", "25718", "7448", "37093", "40256", "47497", "2329", "17768", "13513", "3279", "7591", "57040", "1347", "29083", "37413", "16720", "6142", "43332", "22110", "13013", "16822", "23092", "12015", "10886", "17000", "53244", "34643", "6892", "55754", "6702", "37158", "22565", "9021", "30653", "2554", "47247", "45348", "36625", "46102", "26522", "13073", "25714", "48871", "21795", "45015", "52766", "3662", "53513", "39041", "35703", "43256", "43606", "535", "36740", "18727", "54718", "30030", "41399", "43149", "26243", "47886", "3614", "46033", "54675", "36923", "9472", "7319", "48321", "17267", "27561", "6927", "34561", "14934", "46673", "1081", "25088", "41316", "8185", "39515", "49300", "42600", "54238", "24349", "25574", "22424", "44616", "38356", "30042", "13874", "23802", "42426", "43291", "40218", "51209", "50374", "44871", "37100", "18772", "7647", "31092", "11036", "15075", "56995", "29578", "48023", "56197", "34765", "4955", "47851", "34097", "52149", "33413", "8060", "5698", "47517", "27104", "38322", "38470", "2204", "5112", "30295", "23466", "23644", "3856", "49686", "58327", "52663", "32414", "5577", "47463", "46408", "49020", "26664", "46061", "57983", "39288", "41229", "6767", "3149", "2185", "58947", "37388", "42727", "49978", "11457", "22784", "27875", "28022", "39311", "55470", "43822", "33091", "48890", "36455", "11848", "33814", "58162", "49058", "34252", "31827", "191", "32037", "50801", "17206", "6931", "2442", "12230", "47230", "51542", "18090", "713", "50045", "6250", "16587", "3438", "50490", "54511", "38949", "6071", "25169", "23548", "24127", "58267", "6296", "16989", "32654", "28330", "2771", "47652", "19979", "36875", "24574", "14497", "9274", "53901", "24109", "1301", "50747", "44789", "35137", "35368", "2632", "6681", "34219", "28719", "14973", "29802", "22706", "34943", "30795", "52296", "12784", "10006", "24914", "7366", "3031", "43209", "9289", "14479", "40457", "12310", "25360", "12431", "34216", "21271", "16888", "429", "42221", "16394", "29447", "20468", "20691", "9293", "37020", "8884", "39686", "35158", "15875", "7807", "13990", "31838", "16436", "9237", "35197", "29071", "19450", "34702", "40901", "7313", "33617", "52404", "10211", "15269", "54319", "33214", "22389", "9914", "1956", "54428", "7444", "22809", "24251", "29075", "50544", "23199", "22271", "20990", "40953", "7824", "14952", "43861", "54895", "32836", "23287", "4289", "54058", "48984", "43437", "18671", "42820", "18076", "6485", "9960", "31839", "28017", "55486", "7661", "11106", "16185", "15150", "4515", "44004", "15852", "51076", "8478", "22170", "42273", "13183", "4752", "1164", "25961", "7527", "28204", "51445", "47973", "54151", "58999", "54495", "38035", "12055", "56769", "53075", "52207", "10600", "3683", "58699", "35692", "22268", "6113", "26446", "23097", "6226", "149", "56319", "17507", "39620", "56393", "22779", "11756", "48346", "3569", "8198", "46692", "10599", "27663", "23754", "19510", "10513", "22038", "56065", "43119", "13928", "38988", "29959", "31362", "26836", "12481", "142", "38187", "59192", "42072", "448", "3358", "52585", "25767", "26056", "56467", "18551", "31003", "46042", "13925", "9268", "26651", "7494", "53779", "3816", "23415", "34899", "37027", "33764", "6732", "43460", "36274", "22772", "36295", "57210", "54065", "41728", "34701", "56077", "13052", "4407", "50147", "43138", "21505", "9432", "29443", "4671", "17429", "38687", "51693", "45783", "14535", "25842", "33001", "841", "44848", "58408", "34357", "35130", "32921", "19688", "10122", "23063", "24051", "55400", "4996", "57855", "53173", "23662", "46734", "33426", "46010", "35332", "24702", "44260", "19965", "29093", "46088", "32061", "42227", "43112", "41363", "48621", "79", "58043", "879", "15594", "58994", "41611", "14864", "19605", "25224", "13908", "57614", "51613", "1606", "41492", "54314", "19967", "17563", "49356", "31468", "52301", "33220", "48152", "33155", "59768", "42532", "55615", "59455", "45623", "7751", "4770", "2228", "46302", "33748", "57249", "20766", "49917", "10368", "52833", "42841", "35003", "39724", "11754", "11417", "24258", "32411", "26426", "17885", "48230", "41755", "43740", "36268", "42232", "4582", "12700", "40540", "54341", "43094", "24809", "10358", "23345", "17059", "12234", "13460", "19031", "20596", "35630", "34530", "45308", "37708", "29193", "55059", "31026", "17901", "46787", "58786", "29580", "37541", "48071", "44944", "52098", "2205", "52776", "36576", "28525", "50855", "6988", "50391", "32404", "27061", "10392", "21892", "128", "35677", "51690", "38097", "18681", "6032", "41909", "27622", "8432", "39134", "16026", "4418", "29508", "53933", "36041", "26113", "21523", "49927", "26026", "33730", "29724", "47668", "25712", "34806", "23322", "9276", "6680", "21694", "34946", "49551", "9123", "116", "36252", "23238", "47005", "20599", "36504", "10348", "3600", "724", "2451", "42852", "16748", "36836", "31976", "18463", "7673", "47391", "31572", "31236", "3574", "35253", "9595", "6300", "34076", "7446", "23595", "51534", "59991", "43855", "44949", "16388", "53505", "41534", "28198", "2005", "23973", "20498", "29297", "48047", "27770", "48800", "11085", "34459", "46458", "42211", "23304", "13434", "40698", "12578", "44246", "6709", "46153", "31420", "34112", "614", "53680", "33916", "48525", "5556", "8066", "40356", "8100", "11082", "17408", "21334", "46168", "29211", "1741", "53085", "28552", "2931", "6688", "29537", "25742", "19398", "40887", "5263", "59057", "26576", "51937", "43107", "30704", "45491", "35233", "49658", "42571", "41391", "13114", "48311", "34627", "10849", "20665", "58090", "24530", "36190", "23837", "42956", "23880", "9095", "10312", "56917", "29790", "23845", "24712", "57338", "55130", "18258", "19890", "51307", "5453", "13363", "6097", "37049", "31040", "6338", "34134", "45185", "50072", "32744", "21421", "51864", "26048", "20799", "21403", "46990", "40296", "57532", "4682", "44049", "12684", "19828", "7149", "51598", "39143", "59124", "10795", "50924", "32727", "12581", "43686", "38510", "11650", "1784", "44967", "21857", "40294", "54489", "3533", "31043", "22440", "254", "28581", "33249", "38228", "859", "12113", "12375", "40730", "34661", "55291", "13514", "56648", "36538", "28332", "838", "25612", "45341", "5424", "39650", "58416", "37919", "19586", "32549", "5250", "2846", "52121", "52387", "23039", "59366", "43852", "19364", "58696", "7982", "52644", "30434", "16234", "37826", "58809", "46221", "48636", "38357", "19311", "23161", "45665", "44344", "44236", "20853", "19149", "51508", "58606", "52808", "55521", "103", "7735", "39566", "57805", "3773", "4988", "20194", "58459", "7347", "52067", "25754", "46193", "11473", "50791", "11366", "52570", "52025", "2892", "6321", "14274", "1714", "23016", "4643", "3867", "26712", "8769", "57926", "18268", "18720", "21750", "3377", "25030", "20954", "36326", "59940", "47112", "38907", "55397", "43046", "4746", "41704", "45774", "4774", "10823", "46074", "6789", "27371", "9896", "13095", "33128", "2547", "39719", "51861", "25925", "38567", "40329", "59884", "349", "25089", "23125", "59406", "43761", "10946", "40996", "29412", "31760", "47429", "36832", "6191", "6504", "38370", "39890", "44158", "36404", "51852", "10701", "35052", "9551", "17109", "5506", "6465", "10212", "42203", "58201", "1528", "40061", "31145", "38467", "42707", "34521", "45441", "47060", "37483", "52717", "42690", "58244", "2667", "31297", "3939", "25315", "28939", "50864", "47766", "19231", "41600", "51327", "14866", "57525", "46637", "3617", "2070", "42497", "49055", "23527", "17113", "27129", "52088", "59374", "10229", "50746", "2144", "453", "24187", "43604", "14268", "49859", "31849", "58724", "56653", "39550", "1018", "20502", "20277", "35641", "58787", "36908", "19471", "35018", "33603", "19680", "13311", "18016", "54282", "51965", "32942", "14710", "20165", "7559", "26256", "21260", "54850", "43195", "57965", "21212", "50413", "37194", "19579", "8529", "50599", "16724", "42767", "54218", "27729", "18618", "42644", "12311", "52684", "5499", "25986", "10514", "13020", "4980", "49380", "15741", "32485", "26728", "25775", "32175", "10255", "57935", "24826", "35482", "20226", "50796", "16937", "55428", "35603", "38190", "26008", "8074", "42878", "397", "7883", "31527", "34901", "49194", "6585", "15684", "21", "17472", "34954", "7040", "12805", "30681", "11659", "32723", "48439", "5159", "2062", "24257", "58678", "46058", "13036", "40133", "16775", "18489", "23083", "35683", "31235", "56409", "3615", "1927", "34768", "57004", "41042", "57572", "18209", "16371", "19059", "43015", "14149", "7333", "20281", "49602", "44591", "51278", "16556", "40212", "9127", "4216", "48074", "12528", "57491", "1295", "43378", "48977", "37625", "50449", "28100", "1130", "59019", "51603", "32482", "2360", "14445", "23790", "33159", "47738", "33578", "28371", "57209", "50474", "59599", "46681", "24439", "26237", "4468", "31986", "46513", "29270", "32244", "31914", "18977", "35994", "6108", "43447", "4363", "56742", "34964", "26330", "58597", "30878", "18904", "5368", "39056", "37598", "26672", "6288", "17229", "54359", "52530", "43185", "52033", "52706", "14806", "56449", "29085", "50927", "23687", "47507", "7411", "6035", "21483", "8381", "27200", "5277", "25657", "23779", "48037", "31524", "18146", "13212", "11643", "44535", "16552", "22988", "50965", "12830", "1768", "54984", "39973", "32090", "48223", "16568", "53491", "30283", "12876", "54285", "48975", "2126", "4126", "24579", "21469", "5607", "5048", "55794", "13268", "39094", "18387", "25887", "39905", "58377", "57035", "30221", "16780", "56673", "30304", "25715", "7655", "42141", "5695", "54822", "22433", "20401", "28403", "39405", "32647", "47614", "57683", "21848", "16938", "53660", "4168", "34230", "54892", "12163", "25187", "4271", "4039", "6140", "5302", "38911", "32390", "30406", "12050", "19999", "6332", "41398", "12164", "49547", "1745", "59461", "14283", "24528", "16666", "11264", "49793", "5487", "24289", "32626", "16697", "19487", "6078", "16199", "59895", "53149", "45532", "41594", "50564", "13425", "27436", "21576", "23326", "3897", "42632", "45552", "32922", "30182", "17767", "551", "38751", "44478", "29405", "55257", "49976", "9622", "15279", "49103", "14593", "125", "49745", "6574", "36219", "44347", "2522", "37246", "4219", "9601", "17042", "15422", "8907", "32294", "25149", "58555", "49117", "47779", "46656", "32184", "17748", "52071", "3598", "43676", "32448", "11419", "23932", "20010", "30550", "32463", "52156", "25933", "4602", "9131", "13938", "2653", "35534", "10873", "42657", "19077", "19326", "18230", "1355", "54543", "20987", "9138", "351", "15218", "53828", "52817", "43588", "38691", "35831", "30894", "28726", "57594", "6429", "51140", "39837", "32623", "15509", "6734", "55589", "33476", "57929", "19057", "44081", "28469", "17074", "32280", "48424", "29609", "24954", "7495", "23429", "10170", "12355", "1249", "27443", "53878", "11752", "10573", "41482", "59609", "9308", "1711", "39671", "29056", "40416", "30127", "27646", "40016", "54012", "46742", "53798", "48378", "43305", "29982", "4160", "59547", "40564", "48299", "25282", "8363", "37398", "57593", "37918", "36967", "59129", "43228", "32607", "47202", "27017", "792", "56649", "38864", "58859", "33222", "43749", "37582", "44852", "45674", "25540", "4662", "49651", "50306", "35404", "30070", "11705", "34927", "33948", "36088", "25544", "37431", "26240", "53383", "45376", "11057", "35257", "39005", "263", "33803", "56959", "12029", "41232", "30345", "12327", "34354", "33051", "43424", "29236", "31661", "45841", "21489", "55318", "2158", "9547", "32479", "16104", "32546", "33756", "48918", "21644", "37904", "45820", "23457", "33106", "33041", "35636", "22851", "20168", "16893", "13629", "32528", "1338", "29017", "33824", "25015", "45908", "57677", "9109", "54821", "41079", "48705", "14343", "26653", "27845", "27662", "17312", "16823", "29729", "30594", "17928", "50179", "24980", "30455", "27174", "29383", "49350", "49169", "16333", "23497", "49676", "41105", "43511", "32212", "59593", "48837", "18214", "36704", "25433", "42879", "51032", "11789", "10352", "15160", "12305", "34225", "47785", "37250", "52089", "41443", "31490", "4068", "39732", "35382", "27810", "46155", "27957", "11550", "34082", "33379", "17474", "39964", "30200", "36830", "15571", "1440", "20770", "18701", "12488", "1466", "47020", "13275", "1964", "48292", "6065", "25010", "43423", "36396", "14285", "2243", "43273", "44426", "30233", "42320", "48258", "13369", "10857", "14169", "21695", "35107", "48005", "19116", "14478", "19413", "43230", "49862", "25117", "25401", "44140", "51473", "25411", "42409", "50721", "314", "56086", "16138", "16593", "19859", "35054", "31700", "54922", "26810", "32424", "39054", "44634", "5679", "57119", "26175", "55749", "32597", "49318", "35366", "43854", "27137", "12896", "45081", "49366", "602", "29074", "25003", "1003", "5743", "57759", "39216", "55966", "260", "25635", "37033", "40411", "13605", "15578", "1859", "56722", "22190", "23857", "48410", "44459", "35145", "13869", "52935", "9517", "52556", "24103", "45547", "45801", "55100", "31564", "46501", "6707", "50192", "38446", "31966", "25600", "41529", "11520", "33896", "15589", "56719", "24855", "17297", "54943", "16804", "32878", "16169", "13848", "32386", "37009", "16294", "20193", "39956", "53202", "25405", "48236", "10559", "6263", "55661", "4803", "5268", "59880", "35255", "32467", "44444", "30683", "9041", "57782", "33334", "21118", "36954", "40619", "28651", "20634", "51706", "4344", "28143", "29073", "4102", "41193", "58176", "56309", "46319", "44501", "24150", "48808", "42319", "43419", "26827", "19657", "22951", "42304", "1207", "48352", "42968", "26335", "55464", "48490", "15074", "20536", "30969", "5988", "42651", "4263", "52159", "21278", "36148", "9746", "16496", "56441", "24569", "22754", "39731", "29573", "16390", "8477", "30666", "12130", "47038", "7922", "16015", "20841", "47546", "18787", "25432", "37858", "42646", "35102", "38704", "7150", "37651", "27132", "45352", "3583", "21631", "58862", "118", "17283", "39161", "28856", "10274", "10346", "2770", "57918", "35002", "14927", "34821", "10167", "21575", "35023", "13536", "30035", "42876", "37791", "26196", "30903", "6208", "22030", "44589", "16123", "38201", "13612", "35310", "15641", "9984", "31141", "59894", "25922", "25537", "37255", "50141", "32370", "45028", "51607", "17770", "32259", "14951", "27103", "33484", "42525", "57746", "3002", "39101", "51342", "29487", "10943", "17141", "41109", "44296", "55739", "6005", "40746", "32887", "15575", "36966", "29369", "13602", "36389", "18762", "39538", "32148", "46392", "38853", "13515", "21743", "58659", "59548", "15508", "538", "56573", "56006", "20091", "13530", "27554", "56749", "24747", "19267", "15397", "1084", "5486", "36979", "24645", "13171", "20828", "3918", "40989", "31115", "3694", "34811", "39747", "720", "16895", "10016", "4257", "2859", "37282", "53152", "3367", "55916", "8772", "50909", "22480", "38036", "12070", "3791", "11010", "38585", "52603", "2325", "31792", "33849", "36211", "55489", "5367", "50843", "46661", "37257", "16036", "3428", "59612", "33078", "16737", "8604", "15980", "30775", "28913", "16116", "669", "26005", "572", "7629", "32986", "27284", "34144", "8617", "58133", "35668", "45900", "45136", "59139", "552", "22879", "44787", "5612", "44265", "50563", "12174", "19974", "45216", "21354", "58251", "31476", "49740", "28800", "6458", "2429", "30409", "30821", "40482", "53448", "45188", "23495", "18273", "29618", "58202", "7401", "42890", "24944", "57401", "29776", "32871", "22898", "11601", "39617", "37406", "21789", "42019", "20629", "38302", "20191", "39162", "24213", "10926", "45891", "59738", "27168", "17048", "46731", "7406", "17142", "16761", "29332", "27709", "53305", "22461", "2148", "15180", "22145", "6528", "756", "8845", "20117", "56978", "7566", "14494", "39402", "54246", "38212", "2638", "19457", "49905", "47967", "10705", "21578", "56426", "9444", "49426", "46904", "44243", "16201", "52971", "5766", "23661", "8501", "3804", "1103", "38127", "23376", "952", "12033", "22492", "12877", "25096", "40341", "35146", "44527", "38782", "40194", "48033", "11546", "44681", "44238", "19914", "20395", "31155", "42373", "16108", "41545", "9141", "28389", "20022", "47368", "3206", "36799", "44234", "31780", "10096", "12927", "40268", "41249", "45461", "37736", "56537", "16858", "14504", "57843", "48846", "35909", "48471", "25720", "50401", "28564", "6302", "52662", "35804", "21171", "22523", "42957", "39100", "14332", "41013", "54800", "12531", "11005", "11781", "54469", "55388", "17293", "27417", "11352", "6794", "2557", "33720", "17099", "14884", "42766", "49533", "58814", "33514", "55896", "27650", "38494", "24873", "50753", "5349", "8359", "41574", "6690", "44170", "42895", "10990", "29643", "17400", "18238", "22016", "59660", "4842", "13482", "15644", "6101", "54993", "35176", "47752", "25689", "37886", "31355", "155", "1910", "10239", "13116", "12680", "28453", "37905", "9972", "14322", "18660", "54456", "18301", "45856", "19083", "47570", "29200", "47699", "19087", "35593", "16058", "44458", "30793", "49062", "30532", "18642", "52994", "42151", "8424", "49484", "58419", "48982", "56070", "7242", "53746", "42418", "11400", "34556", "20664", "39488", "54174", "36469", "26491", "56669", "56609", "12704", "43087", "53790", "23715", "37423", "55452", "24492", "28404", "56385", "38422", "8211", "42493", "50546", "49345", "30379", "53347", "17564", "57779", "32616", "503", "29395", "28938", "3075", "11937", "52155", "51772", "11435", "13919", "59907", "29411", "15544", "11923", "45005", "2666", "24547", "39461", "100", "40866", "272", "20569", "43581", "56416", "20087", "8054", "25788", "8888", "20234", "26745", "778", "15502", "40554", "51213", "47286", "6314", "39784", "32368", "42955", "53396", "54375", "44226", "41787", "42031", "36828", "52515", "38385", "1610", "7552", "45896", "15358", "42354", "54332", "24984", "10744", "47459", "12798", "40465", "45343", "28524", "48910", "39823", "11367", "50518", "46194", "6235", "32547", "207", "22221", "15165", "36426", "35517", "49097", "53619", "20223", "49872", "22785", "2943", "57203", "26992", "31756", "47166", "29946", "14744", "17706", "49552", "16144", "13778", "14255", "722", "48178", "15806", "10496", "25182", "9277", "38486", "9789", "15995", "25485", "33653", "12051", "1848", "9706", "9540", "4043", "56108", "12274", "36331", "44651", "48936", "44714", "24689", "33658", "1263", "33621", "40626", "20412", "5610", "29375", "6775", "35870", "14821", "16845", "691", "56230", "9082", "11454", "15391", "29192", "16400", "7486", "20366", "40835", "39172", "43699", "46526", "38891", "40855", "30063", "32401", "55813", "14717", "28827", "18820", "57990", "11675", "53926", "48662", "19580", "16807", "23796", "988", "28778", "43915", "30402", "23445", "32458", "48188", "29360", "12706", "11138", "8637", "58700", "38492", "35734", "24989", "46979", "15617", "53200", "51212", "17953", "5617", "30129", "28104", "708", "26812", "13739", "9375", "59102", "30661", "17639", "34898", "5976", "30510", "32628", "9134", "30596", "17436", "19717", "36620", "8795", "17009", "52625", "745", "1150", "43987", "53816", "59222", "27819", "46972", "20375", "41115", "2608", "19972", "27395", "5184", "55990", "28619", "52443", "29591", "17246", "53377", "40130", "39192", "41656", "38367", "29830", "33822", "30759", "58624", "10040", "38471", "33255", "5469", "5647", "14191", "1193", "44610", "1686", "55360", "59525", "2484", "26585", "47774", "59523", "52196", "1298", "20998", "22146", "24724", "57066", "34987", "12217", "38122", "5706", "49544", "48130", "18970", "45971", "25763", "54191", "3065", "10992", "41601", "17426", "40865", "2914", "56773", "58588", "24093", "8707", "26871", "19960", "54887", "4114", "18180", "56942", "17924", "49239", "48508", "7397", "44461", "4178", "32797", "3658", "897", "35304", "43285", "6965", "12788", "11038", "50065", "47386", "7957", "13176", "11577", "59142", "59865", "2541", "16045", "18696", "14265", "50349", "18089", "38096", "3743", "22300", "40585", "51964", "2655", "11495", "8062", "50071", "40531", "1830", "45096", "31571", "19210", "5938", "58813", "4365", "19858", "33297", "19857", "41039", "51920", "26494", "35491", "22834", "20400", "57466", "41261", "43124", "17458", "26006", "55658", "47985", "52358", "22768", "9758", "13760", "49707", "6526", "33349", "7442", "38118", "50706", "57399", "4830", "7191", "40487", "14475", "55292", "56852", "6025", "16256", "53400", "22761", "6664", "44314", "4838", "45024", "46179", "50932", "57984", "48484", "28983", "30326", "3913", "12316", "9797", "40917", "31501", "20031", "7889", "55734", "6122", "48864", "21572", "42551", "26439", "55690", "3811", "57485", "38908", "54409", "13191", "12422", "12116", "46132", "3338", "14317", "41324", "15220", "12173", "18195", "20233", "15419", "50022", "26528", "2553", "31713", "57613", "28479", "15105", "13404", "20751", "21874", "10899", "9168", "10780", "25115", "41882", "57945", "56164", "56463", "14296", "46711", "61", "17372", "49853", "37960", "12586", "17337", "56705", "7365", "24283", "22034", "17947", "18592", "51806", "1666", "34088", "21545", "54354", "44899", "58337", "40620", "19150", "16772", "41188", "39983", "8223", "2955", "52238", "45330", "46396", "54499", "14774", "45363", "39734", "34844", "31985", "7760", "40097", "52887", "9126", "57958", "38374", "42053", "44047", "23861", "50450", "45246", "4685", "32570", "42483", "24766", "24635", "1258", "8283", "22161", "32950", "31750", "57208", "53112", "27979", "47888", "18191", "48076", "43818", "20711", "22334", "52503", "44395", "3490", "26088", "7521", "53441", "1870", "32167", "17576", "11458", "42358", "14704", "55905", "24424", "25153", "20075", "50388", "49264", "4266", "42619", "11636", "35177", "48226", "55652", "10082", "12824", "55168", "40114", "38680", "23755", "52173", "39346", "48574", "23067", "38616", "31551", "18655", "10468", "16493", "5671", "21216", "48708", "3115", "24497", "36166", "40928", "46632", "18514", "40025", "51606", "45855", "28588", "41577", "46612", "17876", "46083", "984", "55519", "7202", "5700", "7995", "3195", "37557", "15071", "46103", "29676", "18261", "37411", "20546", "27999", "24164", "11071", "15441", "14467", "48992", "2741", "44916", "4347", "51576", "52649", "27101", "10642", "26596", "35981", "23675", "55344", "53666", "32036", "21330", "18599", "57347", "5945", "3290", "16498", "13105", "5239", "8582", "35515", "13721", "4052", "615", "13147", "14353", "33452", "12165", "26626", "39037", "59084", "54037", "47744", "31919", "25863", "1972", "1006", "31776", "20609", "51283", "9114", "12161", "32672", "56938", "6150", "55192", "12988", "8840", "43336", "9893", "16951", "2437", "15796", "54186", "34350", "5040", "34742", "52832", "19694", "53289", "19045", "43827", "58326", "34604", "24431", "20178", "18236", "10757", "28390", "723", "25965", "18755", "25609", "50043", "37366", "21492", "48290", "1092", "21907", "13366", "45688", "39952", "20959", "96", "12917", "49546", "51681", "26628", "48340", "35039", "2270", "2100", "4933", "25885", "9314", "48786", "46991", "43360", "27541", "23107", "23447", "41729", "42462", "5643", "18157", "27572", "13266", "34270", "49373", "22705", "7296", "21355", "4651", "8874", "18093", "35120", "1735", "5127", "21335", "56019", "9011", "43849", "35552", "46939", "18513", "50978", "49664", "3466", "46872", "45291", "20037", "23311", "27521", "33466", "42980", "22435", "48192", "57463", "9449", "30905", "48726", "40082", "59795", "11369", "30580", "49717", "20835", "18691", "16538", "36343", "32562", "8552", "18352", "11281", "22273", "29771", "10136", "13380", "6260", "44279", "39851", "57467", "53528", "55313", "14303", "58842", "58638", "29390", "15111", "49705", "24272", "32730", "22786", "51741", "22526", "45709", "26229", "17830", "23401", "38487", "50639", "59489", "20316", "53900", "39315", "10677", "14218", "51339", "4927", "27110", "40156", "38015", "19949", "16851", "10125", "52533", "22227", "59212", "6826", "48878", "52500", "12555", "3038", "11664", "52714", "16373", "28493", "33095", "26929", "43932", "38355", "10391", "50023", "45311", "47411", "24090", "9009", "11076", "6894", "15359", "30430", "46585", "33286", "42268", "19913", "48996", "37041", "49273", "52876", "38468", "33949", "24810", "42367", "54910", "9253", "41409", "30922", "57492", "47174", "36150", "46981", "41101", "19966", "10616", "21882", "13129", "11423", "15827", "12438", "45404", "59558", "10582", "26191", "54320", "50070", "12910", "45910", "36595", "36230", "24928", "26605", "51880", "25352", "57263", "48018", "25166", "56382", "42711", "20747", "25407", "56941", "41690", "52958", "451", "36159", "42964", "36736", "29719", "42419", "2559", "52254", "1919", "15125", "14078", "5086", "11216", "54262", "42043", "24823", "8149", "34261", "37341", "37245", "29671", "14705", "59875", "58876", "26975", "7208", "50059", "57294", "23424", "17659", "56146", "13454", "8197", "19361", "5068", "2304", "53510", "56691", "54574", "48107", "19095", "13237", "46964", "47844", "47989", "30039", "49460", "27576", "45592", "26661", "15999", "1322", "50204", "4493", "11909", "53637", "56970", "42782", "14993", "13197", "19272", "19269", "25675", "32529", "24453", "22978", "10555", "58042", "29486", "58401", "28194", "4829", "3328", "1551", "6237", "17147", "58703", "25759", "23417", "40712", "50293", "19839", "33494", "34993", "30602", "23315", "11278", "24932", "41369", "39408", "58980", "11656", "21152", "57645", "18771", "15123", "38408", "38516", "48894", "13104", "35880", "57932", "804", "37931", "57937", "10732", "22972", "13074", "46424", "7410", "50636", "21010", "33169", "23483", "28421", "42931", "8094", "492", "14935", "2686", "4117", "452", "47480", "49330", "367", "41576", "15449", "22931", "2295", "22059", "31940", "58185", "37533", "12281", "13496", "15828", "32286", "49781", "7122", "1789", "55001", "3731", "34531", "26773", "45689", "7771", "47099", "19814", "15943", "30509", "55958", "45799", "25773", "39080", "29992", "57349", "16564", "41227", "46012", "27004", "21230", "27325", "38003", "948", "10064", "42003", "45204", "23809", "21050", "11043", "28949", "39627", "19449", "41743", "56127", "38340", "33491", "33208", "50402", "7281", "26263", "14442", "32762", "41258", "42093", "51573", "26393", "14989", "5375", "38031", "33217", "33670", "6475", "49274", "52357", "58171", "37819", "15966", "28420", "14662", "55892", "16178", "1009", "21238", "44738", "10602", "40419", "35186", "47724", "12839", "42865", "3825", "26331", "13736", "50774", "24445", "6185", "54524", "12309", "34896", "33817", "46684", "58041", "35770", "30110", "47078", "5334", "34294", "44283", "16208", "40871", "9319", "35091", "29762", "2881", "22230", "56905", "23544", "40633", "28513", "6143", "5185", "2978", "35007", "58156", "36132", "17192", "3453", "53890", "26288", "49212", "52242", "25672", "4961", "1948", "6654", "49958", "19944", "7453", "20242", "14730", "49838", "40330", "9454", "24183", "46671", "59683", "11838", "4733", "29435", "55416", "52964", "57152", "16601", "45034", "13022", "27203", "16175", "55663", "37921", "53712", "49678", "39006", "5660", "15712", "53087", "13691", "4159", "29176", "56912", "24401", "48059", "59513", "24125", "46332", "27600", "39849", "45745", "10446", "52339", "7858", "55375", "47467", "10614", "455", "32973", "5454", "6435", "55054", "37386", "54936", "5471", "42192", "22503", "6139", "48640", "3241", "5352", "53160", "54309", "13289", "873", "41846", "11534", "18376", "3979", "45877", "5473", "19062", "12148", "50524", "34494", "37652", "52043", "58533", "47454", "38090", "30694", "23133", "38989", "30331", "35876", "25350", "18482", "57191", "4689", "6320", "32523", "51914", "30057", "11129", "51144", "52116", "23395", "47161", "33664", "45568", "32245", "16733", "7665", "38906", "42647", "31320", "40189", "15782", "58034", "36433", "36038", "7640", "49014", "18516", "17245", "6541", "39557", "43384", "53995", "8107", "42529", "29996", "59756", "53484", "27087", "49338", "24032", "15905", "9354", "59789", "29531", "59927", "49225", "46890", "57342", "13718", "16645", "56251", "58286", "34017", "9135", "1071", "55873", "22804", "28492", "14485", "47571", "11616", "46830", "58855", "47158", "10054", "37916", "41983", "45772", "50837", "53340", "33619", "13254", "30591", "22987", "46289", "46249", "53275", "39120", "24880", "58750", "15026", "7688", "27806", "13898", "6240", "4843", "13889", "25505", "4844", "17001", "29272", "43551", "9655", "14872", "2340", "44706", "12884", "8760", "25151", "15631", "1101", "38638", "40775", "26396", "6384", "16988", "23798", "2133", "16513", "18130", "25902", "41340", "235", "45269", "59368", "42903", "42780", "32086", "13809", "12304", "26927", "13574", "45501", "7359", "22535", "26944", "14536", "12831", "23420", "23187", "31443", "10206", "58324", "34005", "57632", "32474", "50908", "54355", "30010", "26698", "42366", "39260", "9630", "35666", "24163", "36364", "51230", "59078", "5076", "21521", "10188", "44099", "17315", "21981", "41121", "9587", "29810", "41927", "14735", "48830", "29522", "51506", "21775", "30124", "41217", "33213", "44554", "904", "58517", "10318", "53433", "33609", "32664", "1059", "1400", "48999", "11660", "29095", "6548", "53007", "9116", "19396", "5940", "47443", "31796", "2157", "31217", "5611", "24117", "7825", "41394", "27668", "8212", "16337", "1578", "16304", "21022", "24219", "2042", "17806", "21653", "16064", "26040", "50856", "56370", "44145", "25593", "42018", "48004", "49309", "45996", "53981", "27579", "55772", "21234", "1603", "14139", "54933", "3675", "49421", "15727", "16624", "17694", "41640", "12665", "10115", "18294", "8473", "34466", "40408", "49650", "15860", "57078", "43869", "5745", "30988", "43842", "39548", "59663", "4743", "58474", "43979", "14990", "7721", "31216", "29027", "35336", "24184", "57363", "45199", "15304", "57588", "10741", "54780", "48964", "22584", "35351", "45890", "11574", "19745", "8164", "10758", "19854", "23254", "52397", "22154", "23180", "37338", "15435", "20701", "13792", "41817", "11231", "38974", "51825", "56588", "4541", "26747", "1564", "49731", "17979", "1994", "39694", "31154", "46177", "48458", "56229", "19105", "46834", "4156", "26448", "37783", "10855", "13", "33981", "32018", "35414", "19968", "23301", "9572", "46813", "54909", "22280", "16965", "21757", "58084", "8959", "38401", "53912", "59607", "58749", "41510", "51331", "42563", "57124", "22169", "45051", "4904", "18847", "13481", "29954", "32015", "17044", "29467", "40306", "14581", "59977", "39171", "3653", "19772", "25687", "9150", "56317", "18051", "9870", "24388", "31054", "44383", "44988", "53696", "944", "49847", "27268", "25836", "22540", "40588", "36696", "15785", "25974", "5416", "19608", "59682", "6652", "33024", "19995", "12060", "13777", "47670", "16675", "33644", "45098", "54599", "39186", "28843", "41009", "25348", "31689", "45426", "20054", "11992", "44512", "6500", "4479", "25906", "21268", "4635", "46291", "47113", "43386", "18766", "12412", "56536", "35632", "31851", "52024", "16508", "2699", "26713", "33113", "24899", "7972", "12339", "23882", "9184", "59871", "3854", "56484", "43074", "37378", "23383", "9338", "36831", "34416", "47857", "8299", "2369", "45668", "11018", "16687", "17017", "6215", "25268", "52611", "28444", "29067", "5659", "47604", "26224", "38393", "17139", "48421", "31055", "51594", "29657", "20333", "58731", "40648", "4138", "20239", "1547", "21854", "15808", "13194", "14619", "17738", "36387", "4484", "55786", "47837", "51316", "40722", "5063", "54105", "51225", "37763", "21513", "52118", "16120", "30913", "26144", "24403", "8621", "47538", "41116", "42087", "2951", "5925", "2798", "26017", "23812", "37959", "55820", "22313", "37817", "22003", "54072", "29600", "22508", "37728", "1122", "44662", "52680", "18833", "41313", "4075", "33970", "41010", "28102", "7099", "23523", "1682", "56448", "16664", "15106", "31357", "54092", "4275", "19777", "33781", "26046", "2416", "18549", "4487", "51755", "14724", "40351", "8736", "30278", "39717", "40985", "14621", "42479", "8371", "43554", "46113", "59667", "21259", "6580", "18233", "10951", "7812", "24155", "27128", "35598", "45677", "56156", "31481", "53249", "236", "59225", "5950", "27906", "27146", "55012", "27665", "10192", "38170", "10017", "28635", "31460", "15780", "43758", "21881", "47498", "58685", "6633", "14982", "17648", "66", "1493", "17559", "36422", "1299", "46902", "16226", "25111", "19774", "64", "25807", "2945", "6037", "26457", "20807", "8402", "46435", "3954", "4180", "37368", "59422", "15695", "15482", "11773", "44055", "56261", "53560", "23993", "31883", "3460", "53215", "9403", "37686", "18255", "8", "17344", "24157", "23758", "20667", "30734", "41620", "4413", "29635", "24624", "35273", "35200", "17439", "10985", "7705", "54954", "19076", "44043", "53755", "57765", "27482", "17654", "32436", "55796", "46814", "40686", "8810", "37152", "1567", "2136", "40513", "9863", "13078", "5811", "27828", "13905", "40455", "39123", "50444", "5884", "56140", "9642", "39755", "14807", "10220", "49895", "45039", "20944", "7187", "13940", "42912", "47887", "58893", "6753", "49289", "3274", "4000", "31430", "51652", "52462", "56007", "35424", "21629", "6426", "21749", "56735", "48811", "20855", "33641", "25488", "18143", "1268", "18586", "22841", "28979", "17281", "31056", "10225", "39012", "16489", "14260", "20876", "46831", "47263", "16501", "14820", "30482", "41366", "45727", "51257", "42489", "11567", "21719", "58155", "33540", "30410", "17662", "54380", "46642", "44222", "7950", "57841", "55883", "32987", "25928", "48451", "38004", "36783", "51835", "3390", "31310", "43298", "15777", "8551", "34171", "7712", "15989", "53838", "59016", "16329", "4513", "30427", "58613", "31300", "14516", "17937", "48307", "21027", "57833", "15114", "53024", "45431", "30163", "24552", "32920", "56332", "34841", "28409", "10450", "47671", "52945", "7090", "21813", "51325", "18614", "25244", "7546", "20858", "37240", "5180", "16500", "25107", "18714", "31525", "38707", "9728", "29117", "1469", "49855", "97", "17543", "21794", "11740", "37028", "50426", "35372", "58990", "32372", "59669", "56887", "59628", "22167", "52195", "54716", "22783", "59954", "7641", "15939", "31484", "56833", "16285", "45483", "24264", "25139", "21132", "19653", "45383", "3462", "41367", "48298", "41085", "28108", "15323", "52268", "53512", "14320", "7951", "30490", "20682", "14961", "14493", "52671", "16153", "57058", "12119", "57255", "26679", "31137", "8335", "55804", "16660", "36617", "43710", "2080", "22137", "12132", "29861", "51859", "17072", "29330", "51943", "19663", "4684", "3064", "48126", "57133", "27149", "13347", "34055", "14692", "46804", "2530", "3431", "21704", "52099", "4886", "40476", "35899", "48623", "21191", "24496", "30457", "30156", "49784", "23292", "36099", "9361", "15222", "49865", "41196", "58114", "450", "22585", "46345", "42458", "27882", "31917", "56204", "29728", "10556", "20531", "11633", "47944", "45151", "41330", "54212", "7704", "47183", "41480", "25158", "19203", "33020", "55051", "45760", "59565", "3644", "8452", "18088", "39393", "37993", "2432", "10984", "19770", "31209", "4171", "7974", "12973", "50929", "8454", "48547", "11083", "6706", "24342", "38184", "1902", "24418", "6863", "42248", "17821", "27250", "46828", "32385", "30561", "51646", "14678", "56045", "8556", "704", "48342", "18587", "25703", "3611", "1490", "25971", "10283", "17231", "14996", "10480", "31591", "5937", "50138", "43798", "56872", "57919", "50268", "10655", "50677", "1040", "58444", "54415", "27317", "37297", "25137", "30625", "54674", "4580", "6950", "9052", "17239", "16393", "18519", "30207", "35637", "30266", "54974", "25458", "31879", "57122", "41659", "9363", "4571", "43963", "46428", "43029", "35357", "45919", "22547", "39159", "26083", "53052", "30806", "16690", "57565", "11638", "45257", "1226", "27827", "5383", "34218", "29968", "21979", "871", "59915", "20830", "6796", "57227", "22258", "922", "17371", "53312", "24028", "2936", "22771", "15576", "56971", "439", "1879", "23793", "52446", "53234", "37422", "33265", "32582", "59733", "31370", "34803", "59581", "24991", "16200", "12875", "34141", "10381", "41448", "55564", "45099", "11012", "54988", "8118", "28310", "57986", "57162", "41499", "9581", "31522", "2140", "8152", "13398", "8977", "20189", "48084", "30656", "23567", "22289", "5829", "24538", "53974", "57587", "50515", "59128", "36849", "48949", "58344", "34387", "4158", "10409", "9801", "6999", "3674", "29136", "21844", "27914", "24458", "7810", "29468", "7213", "52020", "14626", "19815", "39664", "52049", "38014", "275", "28225", "2260", "12479", "13690", "30939", "13946", "44452", "32152", "8939", "23230", "5199", "1545", "38459", "22148", "30137", "18628", "24886", "22591", "7609", "38986", "23368", "7750", "7223", "47781", "30811", "1419", "58585", "55327", "15310", "54199", "19366", "2948", "41614", "9711", "42561", "40201", "38701", "54789", "19786", "32166", "49515", "56998", "52536", "48542", "24852", "22291", "23709", "2377", "52109", "14047", "22193", "27995", "12307", "30001", "10098", "43463", "52019", "58712", "15688", "14847", "8661", "11466", "50441", "32453", "24511", "6312", "13610", "43302", "54588", "49130", "31586", "58135", "49375", "33123", "43143", "1694", "26799", "48179", "2745", "50620", "52855", "56640", "36018", "46073", "8289", "13384", "24758", "9788", "17259", "55915", "24561", "857", "29581", "16758", "31589", "59408", "18141", "58792", "46918", "28604", "46048", "56492", "26346", "33067", "59465", "31479", "16073", "55545", "58853", "59229", "26099", "7708", "16071", "11894", "47563", "38783", "26949", "39230", "40936", "26283", "29277", "47217", "23150", "55142", "22885", "7119", "4022", "59359", "31594", "40270", "36787", "47501", "38092", "37172", "50783", "23278", "33810", "36284", "18723", "31158", "39265", "40833", "1427", "16562", "2863", "47641", "16776", "55825", "24992", "11150", "20060", "27708", "43561", "5704", "8691", "43763", "53281", "39184", "58099", "44269", "10736", "50519", "43357", "9879", "308", "10879", "22958", "2339", "23686", "45053", "21709", "32242", "48197", "34265", "38954", "26440", "10790", "591", "5313", "39067", "17989", "14065", "42422", "21449", "44077", "53762", "15680", "14031", "4936", "11589", "48351", "38428", "53959", "43649", "51182", "52259", "317", "395", "48702", "40521", "13231", "20565", "58052", "32912", "25825", "18557", "40421", "46490", "12565", "12268", "34539", "22883", "44330", "29929", "39027", "21262", "14675", "47928", "27483", "51249", "6669", "8040", "57215", "52940", "59355", "48411", "23955", "31567", "47750", "48099", "2110", "24380", "17338", "54893", "11484", "27243", "5789", "36066", "41758", "12191", "43552", "9029", "26354", "28186", "26211", "25242", "42437", "51423", "14373", "49173", "29936", "14286", "27620", "18823", "56028", "18872", "716", "19216", "46847", "2050", "15854", "29189", "7528", "582", "37762", "25375", "1041", "32560", "26688", "59586", "16041", "32495", "55807", "48371", "45405", "53083", "10792", "32052", "3131", "33755", "15938", "50263", "53402", "40144", "1005", "12455", "45112", "8315", "10832", "33562", "19038", "17895", "7969", "41825", "7335", "48816", "7423", "37331", "31878", "3526", "22653", "27446", "42788", "54852", "32308", "38812", "50205", "26025", "39441", "11029", "19109", "8290", "38136", "29853", "715", "57403", "53046", "32005", "39654", "8153", "9462", "44335", "58248", "23829", "56993", "44334", "8170", "4383", "20241", "690", "10159", "37086", "27788", "24291", "22009", "41827", "39544", "54549", "42610", "41118", "53879", "56219", "40077", "38511", "33956", "22474", "42134", "47885", "2002", "28815", "45812", "2045", "7162", "36077", "2729", "58494", "11515", "39609", "10464", "14133", "35057", "28231", "17751", "27257", "49382", "34290", "29333", "13490", "31583", "3010", "22134", "30678", "46697", "59476", "27802", "54897", "2211", "24419", "39866", "35148", "31185", "41014", "14580", "11548", "13352", "52851", "10323", "11120", "40113", "12781", "5708", "33811", "18393", "45388", "33741", "15885", "37678", "51794", "6428", "11093", "40890", "16101", "25002", "12997", "51535", "19355", "53374", "489", "53725", "21838", "15377", "30069", "58014", "28923", "1404", "41598", "34736", "38841", "37962", "35625", "4449", "44286", "19215", "12818", "37328", "38609", "52326", "43650", "29816", "4320", "22539", "34980", "14388", "14833", "4701", "58936", "46669", "12769", "43715", "43829", "20999", "46878", "40654", "45309", "3007", "11011", "32644", "18461", "31726", "39484", "6258", "28648", "33874", "1802", "21402", "24020", "34001", "9837", "51562", "44164", "37590", "24583", "57822", "41858", "48052", "10675", "58210", "58471", "14123", "12023", "38801", "12080", "6735", "42260", "53502", "7773", "51877", "30944", "34123", "16276", "15047", "37725", "2356", "18139", "3502", "51471", "21165", "38115", "58600", "6310", "38893", "43675", "25225", "21904", "55802", "32559", "55234", "26726", "2464", "34185", "8952", "53715", "11105", "1235", "39073", "40467", "23571", "10011", "23040", "44884", "20262", "17014", "41434", "40749", "24197", "18941", "17997", "29339", "16217", "57512", "39450", "32865", "48444", "34562", "5793", "53538", "35564", "3803", "16091", "3448", "4691", "32341", "30040", "40399", "8869", "54334", "53342", "52809", "33173", "18937", "14953", "6631", "3963", "197", "49262", "56210", "25509", "39448", "33063", "23331", "15717", "33719", "10314", "29226", "1638", "35615", "37998", "54400", "48680", "29925", "50194", "12124", "55383", "40725", "13441", "54376", "3103", "15836", "23903", "15445", "25559", "49643", "40594", "21353", "15822", "46986", "16902", "40239", "21452", "59793", "56602", "26357", "46195", "22945", "26548", "46146", "9713", "40902", "16702", "47933", "29607", "31568", "33711", "21605", "44524", "54496", "40499", "15080", "8881", "16795", "4290", "37930", "44839", "37395", "3766", "45054", "22636", "42827", "44638", "3231", "31419", "12736", "34936", "51622", "11394", "55172", "54827", "6343", "30907", "35081", "35275", "47325", "50844", "48532", "58366", "11803", "7270", "54621", "12847", "47666", "28955", "50741", "12771", "37347", "5019", "3208", "52954", "31508", "20857", "34594", "38264", "36534", "36876", "5495", "12347", "10610", "20379", "48954", "54731", "31070", "46373", "19008", "26076", "10033", "58025", "37620", "15960", "10609", "50123", "22684", "42709", "21156", "16755", "30727", "53924", "49539", "37825", "58085", "44042", "52976", "12661", "15467", "20372", "37684", "48843", "36185", "15205", "33008", "53053", "10380", "43439", "2780", "53853", "36157", "2024", "40952", "51112", "33141", "16414", "13710", "3905", "43204", "10134", "51082", "50650", "30422", "51608", "36526", "52034", "25598", "37273", "25427", "30264", "22593", "48095", "47300", "11262", "5730", "11491", "10900", "40403", "27454", "28729", "52707", "54786", "27045", "49763", "30524", "2294", "26210", "6478", "633", "50080", "55293", "45550", "18344", "47013", "14416", "11274", "7637", "20570", "26060", "45075", "40029", "40567", "47943", "35284", "17912", "13855", "35073", "27224", "30028", "15438", "45811", "54326", "30094", "30991", "13094", "58363", "36988", "3521", "9037", "51200", "9245", "1807", "15155", "24892", "3315", "49843", "2899", "55198", "16365", "50558", "4260", "1367", "59288", "6127", "13007", "9218", "49589", "18056", "49368", "52820", "58772", "59803", "4135", "48215", "51498", "56052", "56349", "21681", "57977", "13132", "23597", "26762", "37016", "48459", "51521", "17555", "16926", "51192", "56522", "55816", "1242", "58886", "41309", "45878", "39148", "13214", "38447", "3537", "47125", "38120", "43972", "7483", "21461", "2309", "23240", "8227", "842", "6773", "47016", "48210", "9003", "22750", "13812", "58881", "3841", "50105", "49566", "10995", "12550", "9569", "35966", "31284", "15643", "47061", "6466", "31588", "6167", "23046", "40153", "57046", "21511", "49739", "57625", "30513", "11066", "46634", "5760", "28526", "12674", "56078", "8415", "22407", "44120", "47964", "57330", "16681", "12725", "48728", "40358", "17801", "28209", "15437", "25247", "28057", "36511", "40383", "14644", "40751", "14298", "30330", "47372", "45505", "21163", "51127", "12257", "35378", "30368", "49193", "16474", "55504", "13884", "39748", "30801", "21991", "34260", "21840", "42761", "55988", "32648", "35841", "17607", "8222", "31388", "52495", "20673", "26500", "14176", "50004", "9813", "9740", "3180", "57943", "7532", "56345", "46665", "40478", "21321", "48400", "23677", "55933", "16292", "30576", "57127", "58157", "43948", "59116", "14201", "24359", "42717", "51005", "10918", "32915", "22739", "57130", "48119", "42996", "24966", "40451", "39723", "53865", "17301", "59496", "44806", "40970", "14130", "13491", "31872", "4294", "32953", "40510", "16853", "47046", "13070", "42264", "35983", "16497", "13459", "44333", "39373", "1709", "15833", "56607", "33302", "22679", "26521", "24802", "37626", "32505", "51453", "44676", "25479", "12096", "45785", "41361", "14511", "20909", "43601", "4388", "51641", "32682", "15444", "25110", "46778", "37571", "27396", "50595", "38248", "47008", "34585", "19523", "54617", "7574", "24394", "20236", "18359", "33365", "39312", "48240", "20409", "34304", "47947", "58470", "56834", "112", "3033", "49780", "6557", "57068", "58250", "50710", "47173", "19327", "41970", "20027", "55637", "37396", "28632", "10721", "56894", "55016", "8288", "6279", "12600", "10916", "19824", "44865", "54851", "25737", "28241", "34106", "33929", "58605", "712", "20793", "9526", "51740", "37774", "7958", "39887", "34695", "17795", "14859", "38312", "37266", "34859", "37214", "39725", "21718", "27385", "10275", "12275", "52080", "23120", "8580", "42184", "35006", "3362", "26951", "1447", "11994", "49392", "16148", "10236", "59348", "15900", "24986", "3619", "38603", "34236", "39921", "11606", "48665", "14401", "3851", "24526", "1136", "39954", "51874", "57232", "36928", "41209", "57791", "29590", "42307", "27609", "55336", "11022", "38030", "957", "41872", "34053", "33600", "24761", "25052", "13571", "5507", "10472", "25791", "24129", "44175", "55382", "46928", "7076", "36762", "21168", "25499", "14942", "22984", "22364", "28475", "1413", "27961", "32506", "26406", "12570", "38497", "46689", "41567", "26470", "51744", "34335", "14520", "62", "6722", "12149", "53526", "7681", "35279", "49762", "15226", "23321", "13845", "22200", "58456", "43639", "52806", "25483", "45200", "3873", "2033", "16994", "53082", "48562", "31925", "31886", "43036", "58602", "18680", "21502", "38024", "50245", "41665", "41025", "43659", "25757", "1392", "40584", "39300", "15750", "4343", "58728", "9890", "3249", "5759", "19174", "51287", "13819", "21593", "963", "41455", "56187", "978", "39915", "40324", "22751", "56864", "23123", "4749", "8274", "57365", "29294", "31459", "43413", "6047", "43154", "22282", "55270", "38077", "13999", "56489", "54451", "34842", "47855", "10384", "24817", "28699", "46488", "45975", "13829", "58656", "14226", "1143", "51507", "19646", "32251", "53561", "18018", "6115", "53097", "10826", "10532", "39044", "45203", "1757", "36341", "14874", "14510", "53487", "53682", "6799", "5314", "59032", "50013", "50750", "54299", "55143", "12914", "124", "46167", "12848", "21878", "13470", "6643", "54687", "20504", "46921", "9030", "32957", "49473", "24189", "28271", "33202", "20737", "57970", "11995", "45342", "55633", "21801", "15372", "44684", "28044", "43325", "30572", "19545", "6882", "54637", "18515", "8533", "25658", "27282", "52581", "13450", "45401", "17692", "50703", "27590", "57334", "13488", "42121", "13666", "16679", "31840", "1224", "953", "8144", "58196", "59788", "40433", "23894", "25086", "57800", "32039", "19236", "45174", "31096", "43517", "38706", "13671", "4236", "32873", "53702", "33861", "50707", "14895", "7505", "29505", "1061", "16747", "12079", "21686", "12095", "6533", "12804", "46722", "15166", "41213", "12416", "830", "25745", "12180", "38366", "53434", "13376", "33093", "122", "18430", "25382", "42609", "3895", "34518", "12093", "27490", "46779", "55259", "41038", "11362", "48804", "23716", "43739", "4802", "35070", "36101", "44443", "1481", "41680", "42560", "22359", "56003", "30540", "14000", "7612", "48850", "48164", "11258", "4754", "40979", "39811", "11921", "38113", "46675", "21466", "9182", "31625", "49981", "45495", "5226", "29001", "9889", "12451", "6464", "49681", "26353", "20806", "10787", "28692", "5782", "11795", "28080", "29827", "26860", "2393", "32300", "27018", "27626", "49726", "41355", "40922", "51435", "55159", "53058", "33799", "37322", "40640", "40095", "1473", "15040", "21221", "4625", "8129", "55830", "55097", "28802", "30282", "44132", "33837", "28808", "25572", "14106", "43791", "5201", "21065", "7201", "993", "45809", "21115", "52677", "31939", "25065", "16646", "29750", "40814", "32325", "55112", "36862", "8883", "10378", "9703", "27169", "25307", "37528", "34364", "40769", "35265", "58971", "14217", "9331", "264", "26375", "39438", "20904", "29490", "40613", "40918", "31033", "55938", "34743", "11961", "31782", "13720", "55537", "14187", "59379", "7907", "1166", "21206", "4884", "35399", "41629", "52502", "43491", "11175", "56617", "36290", "47751", "41786", "26230", "20895", "1589", "59423", "47526", "4896", "19428", "4253", "30970", "21106", "3374", "31763", "56410", "18038", "47010", "25353", "28980", "48216", "7660", "1884", "10200", "14292", "14419", "29813", "9206", "19226", "41799", "14916", "19666", "14154", "15362", "3042", "23194", "32299", "57434", "36889", "40385", "32784", "25624", "4692", "19676", "4056", "22403", "35291", "15952", "22337", "12170", "27938", "35518", "16900", "20656", "48196", "17949", "37784", "58239", "51300", "47988", "58976", "24812", "21507", "9032", "18672", "20154", "13593", "5078", "37293", "26594", "54873", "12248", "36682", "7358", "15281", "3340", "17131", "35575", "42196", "54474", "48231", "40574", "41451", "37369", "55886", "46978", "13088", "51560", "58134", "49028", "32146", "49888", "4791", "2991", "5300", "41279", "16059", "13202", "48475", "58491", "51611", "20294", "52865", "6818", "59845", "57547", "17980", "46704", "32872", "1340", "3093", "6145", "26758", "13135", "26696", "43730", "43497", "13200", "2121", "55433", "37668", "35098", "55000", "17593", "36011", "565", "52487", "52746", "554", "21722", "25455", "29313", "35445", "20878", "54140", "3736", "307", "55158", "57087", "13295", "710", "46713", "16152", "48772", "57716", "18427", "13873", "52657", "15092", "53423", "22158", "26840", "9026", "32304", "54824", "31877", "23750", "48175", "36176", "31951", "38649", "6850", "14561", "27069", "50360", "9718", "49657", "52736", "51963", "14051", "56445", "53157", "24354", "56657", "48714", "58493", "12184", "30272", "46643", "21464", "10925", "2536", "29780", "22484", "30465", "45781", "14827", "14790", "20630", "16583", "11384", "7874", "42637", "47124", "34312", "31250", "57939", "10044", "35885", "15558", "13344", "45307", "25408", "5402", "37983", "8873", "693", "19883", "36243", "49370", "42105", "24851", "29315", "43", "17081", "3426", "56794", "18338", "17558", "39358", "57995", "55743", "43883", "53355", "1898", "32328", "42065", "46899", "5050", "54310", "12138", "57999", "22427", "32194", "27963", "8449", "8373", "21632", "59765", "57409", "17732", "9075", "16422", "34221", "639", "8216", "37779", "45268", "33157", "36161", "48929", "17244", "47735", "34476", "20716", "18561", "21790", "54562", "58907", "52346", "14225", "18361", "45833", "32256", "20230", "18530", "2983", "53223", "15036", "5911", "56579", "36655", "39102", "58730", "37622", "20433", "16119", "35582", "1147", "48790", "35294", "22459", "57080", "53763", "54531", "44419", "2813", "1204", "23072", "59159", "53586", "54537", "7965", "34339", "49404", "27633", "50042", "29280", "44960", "57686", "20270", "30847", "27339", "34902", "37570", "44889", "58744", "24653", "47434", "52827", "16418", "51823", "57217", "28713", "33027", "59637", "52531", "46558", "26801", "57171", "45882", "12743", "34254", "39902", "36118", "59265", "46571", "31492", "32318", "48019", "7857", "5896", "6552", "11292", "40348", "4854", "51951", "14067", "37777", "13936", "48720", "55410", "52021", "57072", "15601", "38727", "55530", "20785", "24484", "57429", "48473", "16882", "28694", "27781", "676", "34890", "8932", "56099", "8305", "40587", "51400", "31563", "58372", "25783", "24239", "27949", "43688", "49165", "41286", "42554", "20913", "10444", "33816", "58514", "19274", "39765", "41562", "29414", "3144", "12293", "9685", "52110", "42407", "28868", "12890", "57866", "50555", "48199", "7581", "8492", "41055", "57356", "19444", "28898", "54826", "44692", "729", "44502", "41829", "35013", "16904", "38107", "50790", "57581", "1796", "24171", "32775", "21184", "6932", "52164", "1905", "12834", "34897", "57239", "22201", "58141", "1308", "30854", "5846", "51030", "59801", "33453", "34322", "25241", "16774", "1511", "50523", "2093", "57160", "56499", "29329", "8329", "54107", "28340", "12314", "4441", "43889", "33342", "26593", "59144", "28164", "18583", "17449", "11487", "21161", "55680", "7992", "32188", "17381", "45655", "31255", "16490", "5157", "25288", "46728", "23531", "46372", "13082", "2630", "33508", "46104", "47127", "17663", "30243", "33972", "24079", "49172", "17271", "51104", "42807", "34320", "18650", "46075", "17877", "42914", "14037", "49889", "54279", "30889", "6360", "10554", "17070", "21673", "13644", "50227", "3953", "55662", "57218", "38824", "44024", "14843", "26693", "2166", "16708", "14587", "30496", "18184", "24779", "23191", "15529", "14909", "51961", "24204", "26183", "53110", "19300", "10648", "18433", "9516", "55295", "19169", "29461", "34700", "36235", "54407", "16438", "35777", "57763", "57386", "22744", "44832", "32540", "39268", "31909", "58011", "40379", "13182", "49100", "4406", "25078", "30543", "52209", "53391", "57205", "53728", "24587", "50068", "33903", "53825", "6659", "219", "59495", "41723", "29597", "21415", "26658", "12500", "30503", "17038", "30981", "30812", "7678", "22192", "25340", "56491", "56041", "55014", "6502", "34000", "25814", "234", "21928", "26792", "19467", "16325", "21337", "35230", "25503", "21554", "29842", "42922", "14596", "54232", "46446", "52863", "45319", "49683", "1916", "57248", "26428", "56121", "25997", "3812", "834", "3524", "6136", "26543", "19266", "15042", "13178", "15893", "1028", "51648", "46142", "46679", "52265", "411", "24236", "11908", "12235", "35066", "9873", "33573", "37636", "5087", "33127", "8761", "55621", "37040", "22195", "2528", "47726", "47825", "9761", "44124", "224", "29647", "49072", "222", "6711", "51418", "36927", "7440", "37607", "13653", "52725", "26787", "3378", "55025", "59000", "50458", "20723", "51439", "59335", "43550", "29123", "19795", "32150", "37765", "58625", "12332", "7555", "55337", "57369", "37790", "52437", "24637", "2783", "22580", "31309", "7605", "11059", "59744", "36192", "45416", "2412", "46754", "3871", "19840", "43607", "30663", "25436", "35906", "15609", "9482", "47154", "21252", "717", "17153", "19280", "8142", "42507", "32195", "19165", "14458", "7518", "20865", "14831", "32140", "4816", "29526", "37680", "39670", "47957", "15545", "11892", "21276", "14763", "20804", "55858", "28454", "3686", "50251", "52913", "47004", "14070", "17051", "48565", "37184", "57056", "43977", "25994", "1640", "32029", "1852", "5161", "2178", "47813", "18399", "54692", "27093", "33477", "12133", "36498", "52451", "32074", "18805", "34746", "11462", "42941", "49093", "42017", "20564", "30176", "38137", "30592", "55979", "32792", "7420", "34654", "10793", "49049", "56288", "21292", "41967", "30463", "8924", "33532", "47999", "15547", "31365", "32769", "21678", "35391", "15089", "36250", "5298", "56939", "509", "8220", "28791", "55133", "22079", "40312", "57569", "4465", "47932", "34056", "17191", "12318", "1389", "37920", "8307", "33813", "55473", "44881", "37306", "55845", "10734", "51480", "58480", "17365", "477", "56312", "52313", "5854", "3304", "7261", "10693", "11817", "31784", "52460", "4632", "22699", "58988", "17116", "5151", "43924", "17094", "25838", "29525", "37639", "12521", "12158", "37449", "1886", "22269", "27899", "45997", "46050", "27596", "26301", "13583", "38997", "30842", "36155", "54057", "24440", "10375", "7433", "41003", "28767", "47809", "7308", "31010", "46333", "16945", "3729", "35326", "22129", "8192", "21821", "18367", "8601", "10604", "18208", "33013", "47176", "35780", "4412", "26057", "30613", "50334", "46247", "9239", "41449", "54362", "31068", "50603", "55239", "370", "43217", "7197", "13365", "32565", "16233", "43599", "37267", "15842", "27689", "34124", "36121", "29147", "20253", "3958", "2900", "37201", "54281", "8672", "11788", "43329", "19224", "15587", "54682", "39494", "6982", "34356", "57659", "45118", "11598", "4369", "29811", "12167", "44306", "8732", "30710", "48419", "4454", "27090", "58951", "32824", "52421", "40124", "45189", "38403", "696", "9580", "53130", "16326", "56521", "57555", "16182", "44323", "21952", "38552", "28948", "41136", "26794", "56206", "8994", "25878", "48143", "8101", "20123", "12337", "44449", "38372", "5391", "46861", "27308", "812", "7317", "43619", "30859", "57951", "10296", "42353", "19253", "30383", "9194", "57451", "39611", "9790", "9301", "5557", "45003", "48860", "50760", "23756", "40596", "18919", "51", "49560", "3111", "27278", "15429", "31347", "37478", "56531", "30720", "4987", "42311", "11246", "17520", "26138", "59588", "48466", "12491", "7435", "26753", "1637", "33287", "48506", "471", "21622", "52208", "41566", "7626", "54066", "17427", "12484", "57638", "36841", "14670", "55125", "28287", "55558", "42988", "50695", "55227", "26606", "27667", "59147", "59081", "5145", "8885", "36196", "53454", "5566", "53708", "37618", "40000", "27049", "10593", "9543", "26735", "33749", "55601", "19090", "52577", "59011", "49620", "25310", "5253", "7748", "28387", "58329", "36524", "5956", "24500", "31868", "48170", "46098", "17341", "20678", "18071", "15724", "13209", "22693", "2693", "45061", "15550", "9290", "15694", "4585", "23732", "9278", "14382", "77", "43293", "5600", "43996", "14189", "30559", "55632", "56058", "36973", "44063", "22235", "843", "32827", "39909", "38735", "21843", "19350", "5408", "41988", "15255", "50056", "30476", "30428", "5182", "1485", "27307", "27773", "57397", "26825", "21059", "29477", "43541", "27505", "33268", "8455", "33939", "7459", "9982", "11371", "38857", "55061", "48744", "35217", "17222", "28959", "50500", "51502", "26844", "13945", "36975", "5651", "51465", "25395", "16612", "52668", "17333", "33077", "15401", "48614", "22722", "23028", "24939", "18605", "11608", "11711", "16525", "26610", "44161", "8268", "41231", "1077", "39286", "24553", "35333", "44078", "11287", "3119", "57835", "38081", "19249", "55190", "11406", "48877", "6476", "26394", "30335", "37576", "49216", "23117", "8346", "45901", "58389", "38961", "2747", "53675", "13504", "36856", "3392", "35194", "52943", "52287", "58072", "21872", "59566", "12733", "39309", "36604", "8654", "53021", "54096", "17287", "44392", "12611", "37083", "39454", "25812", "48642", "58137", "41624", "24071", "11678", "52768", "4242", "52840", "33864", "10366", "35008", "30749", "7602", "44639", "42719", "54925", "32316", "9743", "18200", "45768", "14983", "1895", "13569", "37187", "31304", "40705", "1869", "5172", "43661", "47333", "37872", "19316", "18608", "44254", "18802", "12329", "5477", "1782", "57531", "1438", "3605", "49349", "26250", "13822", "54807", "896", "338", "34759", "37845", "25203", "261", "8758", "54109", "9356", "26563", "26650", "40782", "15757", "3806", "3720", "23300", "7259", "33434", "48285", "10084", "45701", "53349", "4300", "37212", "31930", "35599", "32236", "41621", "3946", "43498", "34673", "42471", "33889", "57923", "54071", "45920", "59938", "600", "50879", "11300", "46000", "4549", "36557", "26489", "33331", "53822", "21886", "27488", "12808", "48445", "51886", "32035", "56444", "32613", "2711", "29775", "57073", "18459", "21949", "31953", "11390", "6201", "24750", "41997", "29777", "32988", "11991", "3877", "44000", "7537", "505", "50438", "35731", "43562", "11549", "25854", "13621", "2570", "2001", "45050", "39868", "18109", "53794", "55171", "56867", "8759", "29221", "57079", "49882", "45609", "39061", "28715", "8413", "18796", "30851", "59299", "29656", "44168", "39585", "55941", "38502", "36819", "56915", "22241", "20582", "25142", "4341", "28329", "46205", "11133", "47772", "57400", "11471", "12892", "2713", "19928", "48584", "21256", "43418", "25361", "11628", "23324", "45390", "12857", "3027", "13854", "17053", "29489", "52396", "10656", "57733", "8581", "27133", "52612", "7845", "4137", "34352", "28243", "37796", "36637", "28931", "54979", "27160", "47281", "17792", "29865", "23603", "51561", "1955", "13960", "18446", "4079", "29276", "27659", "39977", "3318", "17724", "39082", "13472", "53357", "57745", "40338", "48768", "4446", "57261", "24179", "1496", "25127", "48862", "12819", "13173", "38821", "31167", "50580", "28380", "38884", "25602", "46515", "9951", "16570", "15266", "8297", "780", "27908", "56110", "41667", "38809", "31452", "44972", "12398", "14539", "17992", "7084", "56589", "37360", "12850", "4118", "10912", "19177", "41365", "7692", "42680", "26914", "11977", "8433", "36072", "18905", "54130", "18715", "377", "755", "50853", "20950", "23926", "17468", "27130", "54572", "27683", "13669", "59842", "40817", "57000", "38822", "26867", "35825", "942", "387", "8947", "8298", "38903", "45651", "21826", "21286", "47407", "31261", "30529", "38579", "1785", "35537", "40575", "6876", "13053", "59316", "20581", "6442", "21708", "22803", "22168", "15286", "19365", "42130", "57117", "28168", "165", "15465", "2249", "37426", "58476", "20889", "57481", "49996", "41250", "1265", "29387", "20116", "40771", "37105", "53406", "56783", "1498", "7159", "57393", "52347", "32979", "28645", "58672", "26102", "2041", "49118", "4940", "46965", "42734", "58944", "49536", "27916", "46510", "41381", "19118", "45135", "40431", "22875", "496", "9348", "58427", "56506", "14357", "2525", "30530", "36186", "5276", "46746", "8468", "26214", "29373", "13525", "39864", "41631", "59354", "18249", "38544", "58294", "40166", "23818", "59126", "31427", "33416", "41336", "12902", "543", "58246", "10785", "47214", "37999", "30792", "21706", "42225", "21033", "52549", "24328", "48802", "24476", "15171", "6077", "52248", "39114", "41423", "12207", "18898", "33420", "56026", "53439", "519", "19407", "18940", "34301", "44687", "9172", "34491", "47054", "22385", "39481", "12483", "6381", "9776", "34801", "45023", "3391", "58687", "12219", "54110", "29804", "38720", "56215", "29764", "1872", "24173", "49874", "24499", "47701", "14206", "38485", "36403", "29732", "29630", "44097", "22977", "15483", "26171", "15698", "22125", "31967", "19611", "33162", "8943", "10848", "3604", "513", "44304", "59175", "17302", "27320", "29923", "57622", "30007", "48131", "46511", "58022", "18968", "9983", "17884", "14222", "36363", "51109", "44998", "54534", "59777", "14546", "13247", "11712", "50418", "3147", "43795", "40591", "7597", "32810", "10365", "41113", "28323", "9609", "37380", "16367", "50642", "28578", "36203", "6940", "38901", "39603", "22792", "34224", "36824", "55456", "2705", "8679", "52781", "30948", "34870", "43367", "55047", "44051", "57438", "22974", "9248", "53368", "12888", "43354", "50883", "17004", "22062", "47983", "27047", "14236", "37244", "8530", "7620", "2850", "22649", "12469", "58127", "5635", "53997", "57470", "7224", "51298", "31735", "15158", "54202", "38142", "26242", "53013", "5498", "14278", "50232", "3725", "17589", "31381", "25784", "52147", "56094", "48942", "6202", "45082", "5655", "18407", "41578", "23284", "36009", "16836", "56565", "57398", "54078", "4161", "11669", "16402", "31727", "54194", "18418", "24613", "51427", "56279", "9189", "11590", "41440", "1010", "58166", "8209", "44680", "638", "43711", "40913", "56822", "40604", "10420", "38858", "6980", "13379", "31973", "57025", "45471", "14782", "35095", "5666", "41899", "5788", "22375", "5794", "2177", "6666", "24200", "38112", "1516", "9764", "54624", "2969", "46638", "32943", "11220", "9885", "37756", "3382", "5008", "59472", "36489", "59460", "4841", "57100", "32379", "16245", "51350", "33362", "24290", "46092", "53395", "28381", "40589", "44094", "44285", "20571", "53867", "44991", "22236", "23517", "59735", "20593", "40277", "23317", "38597", "51827", "22249", "546", "25723", "59685", "43642", "55267", "5731", "40355", "4036", "36514", "9546", "36873", "52805", "30308", "56885", "21962", "24485", "38199", "55326", "21187", "23421", "34030", "31778", "35490", "28211", "53778", "48244", "4578", "653", "44488", "43781", "42653", "32956", "5020", "41533", "25959", "35359", "57867", "366", "2269", "42415", "3817", "10313", "59248", "30888", "58546", "32116", "1821", "25547", "46561", "44213", "59591", "39834", "47311", "56646", "49822", "34877", "45714", "628", "28439", "8057", "2916", "13816", "51683", "15309", "6189", "54975", "8835", "29756", "40251", "9113", "24793", "22347", "23841", "32759", "6308", "3269", "6518", "56825", "42504", "26545", "10571", "44896", "26957", "45100", "39331", "12166", "17102", "19520", "25790", "49973", "31988", "39072", "33011", "56898", "38456", "668", "41862", "49938", "35873", "34784", "19348", "37289", "22017", "48723", "36471", "30992", "28230", "16901", "36454", "22666", "4141", "825", "41654", "13262", "58748", "52716", "34873", "46219", "36984", "57292", "36776", "38259", "10436", "29956", "26368", "56511", "49875", "34264", "14389", "19346", "12651", "39247", "49532", "22991", "30546", "4924", "45121", "45208", "18346", "28674", "4358", "46448", "54966", "23521", "7836", "19268", "34061", "55128", "13438", "18818", "14634", "9311", "35152", "49358", "50367", "29823", "33868", "28984", "51017", "11506", "58069", "56989", "18120", "51458", "54353", "25768", "32758", "14488", "51856", "21783", "16194", "16048", "4346", "20744", "55385", "36814", "52013", "30290", "22434", "46071", "28452", "19491", "57785", "19315", "1067", "6138", "3060", "43779", "38951", "2047", "55863", "45538", "56346", "49634", "7374", "36974", "33964", "43057", "47653", "55449", "18731", "6540", "39140", "37235", "23876", "2885", "9063", "21128", "50550", "15729", "45859", "47981", "54519", "48404", "35393", "49109", "20836", "394", "25275", "35223", "11075", "23381", "13500", "6712", "4286", "55045", "31174", "21462", "36756", "53050", "3587", "8326", "20466", "56741", "19106", "15648", "7793", "3858", "42241", "1634", "20082", "14954", "25160", "43205", "34862", "54767", "44489", "31405", "8879", "36706", "13342", "58932", "20204", "36123", "53346", "58954", "45887", "48704", "24529", "15504", "53672", "40136", "14793", "51083", "6335", "37224", "15311", "8784", "8841", "31251", "29265", "25178", "56535", "37701", "27738", "31254", "52762", "34420", "54600", "58200", "47121", "1582", "30884", "33197", "35918", "26904", "23474", "39445", "22335", "8964", "21061", "34713", "59578", "5893", "2082", "17300", "56249", "22314", "7252", "28598", "6316", "33596", "57394", "6440", "29430", "59530", "25031", "42626", "19624", "40231", "6790", "51840", "24315", "36079", "44033", "21277", "51371", "39350", "28539", "13273", "12593", "48388", "55767", "11897", "16231", "13543", "34752", "32605", "4950", "15800", "14746", "19459", "19695", "12005", "25932", "24584", "21367", "17476", "43441", "32588", "6646", "22412", "31946", "55303", "56826", "24897", "37939", "32751", "46351", "46212", "17840", "56133", "9343", "55011", "16516", "19947", "27156", "55275", "41507", "40208", "48064", "59076", "50376", "50492", "50593", "18981", "28864", "55904", "23967", "39760", "32024", "18392", "28440", "54967", "13198", "57412", "19562", "42752", "12477", "49197", "56235", "3631", "26776", "49416", "51450", "30706", "18084", "10026", "12100", "50452", "45981", "31758", "37753", "54614", "292", "9503", "9892", "35888", "52332", "3473", "39749", "35061", "12075", "53801", "59484", "34792", "28497", "26676", "11445", "30232", "18694", "52015", "3094", "18617", "34684", "56138", "2131", "18321", "46370", "40723", "19422", "52800", "4552", "9866", "37694", "16218", "6508", "11017", "24387", "11256", "38509", "46572", "45155", "18552", "16672", "15392", "6059", "7075", "26095", "11243", "35784", "23458", "31608", "42967", "58540", "31836", "15892", "6550", "47022", "43347", "46850", "26979", "58271", "20986", "2733", "15887", "3413", "11713", "10827", "9329", "24437", "28861", "2199", "12885", "49963", "40015", "2915", "28762", "35450", "56564", "11683", "6583", "16917", "37234", "32976", "47315", "22990", "12668", "38386", "31449", "32027", "34242", "13922", "11207", "33676", "47426", "4485", "31169", "24650", "27769", "1597", "21895", "353", "42073", "47901", "31597", "39918", "2566", "9574", "34625", "1541", "54164", "924", "9043", "32747", "44553", "19595", "41064", "8323", "9200", "37390", "17943", "44142", "44240", "59107", "23564", "32706", "50127", "36278", "50994", "48644", "21433", "8333", "23546", "580", "52310", "56771", "19908", "34959", "59492", "25789", "6572", "59675", "47192", "11452", "4278", "36920", "4857", "40894", "53026", "21501", "28498", "44895", "7603", "53501", "29560", "7790", "3470", "20640", "57207", "41764", "29506", "34485", "18670", "9269", "47058", "27184", "48670", "50457", "30634", "29479", "57910", "25039", "51078", "56394", "6450", "59654", "44820", "7382", "25446", "13154", "43969", "4254", "20778", "399", "57435", "9321", "45106", "7832", "55431", "12111", "1696", "2198", "28677", "31843", "25622", "30403", "53040", "49684", "32694", "24246", "30312", "28117", "14967", "42542", "55281", "40824", "35670", "37432", "20140", "54132", "18244", "19072", "3108", "40555", "26007", "27848", "16694", "50930", "43916", "11930", "25320", "33115", "43140", "32371", "34003", "44867", "5790", "3733", "2812", "12880", "11809", "23502", "44394", "16128", "38657", "16860", "35823", "59280", "23348", "13878", "17664", "22619", "6158", "29307", "34115", "55742", "45960", "359", "51675", "23647", "6058", "17637", "44824", "45247", "34637", "31869", "5880", "14253", "33767", "51042", "17067", "40400", "48594", "8285", "15664", "56018", "59217", "3186", "43147", "22220", "14646", "23241", "59306", "32586", "48696", "30541", "27746", "1255", "27691", "8203", "50944", "18183", "531", "30573", "18800", "24513", "32776", "29416", "40325", "38930", "17257", "52240", "195", "41893", "50207", "51913", "37913", "26830", "58019", "34619", "58500", "9532", "34241", "36180", "22418", "16187", "26660", "39982", "42715", "11424", "20935", "18006", "14411", "31333", "41642", "14664", "4112", "43629", "20295", "35329", "52753", "46402", "22113", "56516", "42329", "37139", "58195", "4404", "49255", "14512", "33205", "67", "17187", "48347", "36593", "45115", "16149", "31044", "40868", "21705", "4473", "28609", "36512", "56224", "53721", "34723", "30246", "3280", "5711", "26365", "56732", "51217", "22188", "24648", "10051", "28196", "54813", "13277", "1950", "30680", "32679", "41362", "23989", "35622", "32795", "30768", "41147", "27378", "24287", "3278", "16065", "9180", "27919", "9509", "56512", "17686", "56808", "51800", "25020", "55373", "57387", "28734", "42090", "34703", "42795", "46015", "4200", "56273", "39896", "40105", "49076", "26326", "56456", "9505", "35990", "20361", "44340", "44742", "19427", "17398", "26934", "24888", "4561", "13774", "19401", "14932", "27194", "53216", "56571", "7247", "50975", "56128", "5361", "8751", "1037", "1199", "53382", "26097", "47387", "46412", "22910", "33451", "15407", "18749", "38776", "21114", "49649", "25927", "26122", "37175", "40820", "49761", "47", "47818", "25123", "44480", "31219", "10251", "20441", "32301", "39639", "41954", "5075", "8047", "58174", "57741", "23229", "10547", "25696", "14911", "41073", "59261", "43139", "31523", "6359", "330", "4492", "20061", "28434", "57358", "18149", "34046", "18334", "43950", "9316", "19024", "20649", "22892", "29251", "46182", "49088", "26435", "40167", "20146", "47096", "22384", "13089", "6217", "29952", "7098", "7479", "17120", "33151", "41388", "55246", "18059", "35430", "18775", "56430", "20386", "27990", "13617", "52398", "40800", "55427", "35748", "17105", "31690", "2273", "16558", "10005", "2069", "30092", "46509", "55467", "55290", "27656", "35836", "40200", "24678", "19258", "47557", "17380", "36953", "42050", "38971", "49633", "33192", "42380", "42252", "38280", "1928", "6444", "49377", "33515", "27138", "49031", "34164", "18257", "28461", "26020", "15673", "56252", "13079", "26156", "31259", "46444", "53410", "11114", "53151", "6236", "28841", "59477", "27353", "23987", "35123", "50780", "51984", "38229", "6957", "28940", "42013", "52062", "55848", "44191", "11943", "12194", "59420", "10262", "4588", "6815", "35216", "15244", "46905", "30688", "31815", "54998", "52477", "13662", "49852", "38293", "45979", "57978", "12903", "25467", "50488", "17852", "37434", "57424", "10662", "40107", "14531", "8669", "1472", "1570", "34586", "59258", "7967", "5105", "57658", "20505", "45179", "23954", "42886", "21689", "9798", "57414", "45447", "26645", "1624", "57762", "8963", "52317", "18027", "55273", "40437", "56496", "42440", "31", "17255", "32665", "19292", "24040", "6952", "26123", "59475", "32293", "16484", "5996", "3986", "21109", "18697", "29000", "18930", "7225", "4553", "47027", "52912", "41606", "1052", "32907", "50735", "24830", "3717", "799", "59677", "6784", "45704", "26983", "47144", "40160", "55522", "19735", "35816", "47361", "56619", "9405", "22406", "33620", "47562", "1937", "45606", "23386", "32584", "34724", "6233", "41859", "22465", "17080", "5073", "31812", "43320", "49718", "44104", "33309", "14472", "49248", "39341", "49522", "9038", "8311", "8715", "35650", "2870", "47360", "27730", "46617", "29983", "4908", "22378", "4558", "18911", "7310", "13361", "13362", "31144", "5539", "27589", "47558", "8882", "35788", "51315", "46740", "58967", "23604", "37547", "55617", "41230", "974", "35264", "33352", "15778", "51468", "627", "29631", "58882", "5807", "28410", "3225", "51921", "19127", "50142", "38663", "21823", "36803", "19152", "18405", "16769", "54742", "4489", "52831", "18054", "8868", "26458", "30130", "1104", "25492", "31306", "41816", "16974", "57067", "13373", "48587", "50381", "41648", "45442", "48943", "46745", "20978", "868", "30097", "2893", "1153", "5815", "58370", "51474", "58437", "58898", "30698", "32689", "17200", "35540", "13987", "27421", "21666", "39335", "2825", "55553", "2815", "58518", "16828", "24162", "14221", "56897", "17314", "28362", "3910", "52482", "34215", "53464", "27866", "17170", "57496", "20137", "33575", "4496", "37550", "22462", "58547", "48497", "23481", "35768", "9273", "29168", "31882", "5739", "35305", "1883", "40642", "45181", "36581", "12829", "31720", "43034", "58516", "728", "16038", "2730", "4048", "52797", "34172", "55878", "20110", "8165", "20442", "21227", "7094", "4391", "23615", "16252", "28877", "39750", "14725", "46476", "56958", "24759", "32803", "2103", "45922", "21241", "18384", "5466", "36330", "9275", "49053", "57788", "55750", "2838", "33888", "45521", "20275", "25354", "43528", "15194", "18172", "45224", "15823", "3364", "58564", "38698", "25025", "40861", "31635", "16921", "56490", "46761", "30080", "54899", "8933", "49957", "7097", "24585", "33327", "46173", "7761", "43859", "43646", "42109", "56629", "1015", "21812", "25626", "4231", "37970", "15414", "38681", "19871", "1844", "25453", "40582", "8414", "44883", "45168", "21930", "27383", "41061", "49863", "41559", "35555", "15986", "46895", "6883", "36848", "28825", "13169", "25929", "51981", "24590", "46461", "21459", "7441", "1294", "49949", "51063", "47654", "51995", "51299", "25585", "15565", "2354", "4999", "56638", "2264", "52620", "24787", "23866", "21039", "49920", "43930", "7985", "50830", "44057", "52470", "27517", "10620", "25516", "2662", "38246", "7540", "18178", "43878", "38010", "15008", "5559", "24193", "140", "27886", "26839", "39397", "45232", "20157", "43746", "44766", "50299", "42860", "43179", "6861", "25619", "1125", "10097", "28925", "45786", "56786", "22143", "40537", "16619", "48036", "59912", "13715", "32281", "24885", "29896", "45883", "3193", "7437", "48766", "13750", "20378", "45392", "38451", "50174", "8738", "22835", "55919", "54859", "840", "24733", "28152", "12517", "27252", "57608", "59932", "25306", "40579", "19273", "55447", "47421", "24805", "54345", "35901", "44242", "14852", "6849", "13939", "5606", "41782", "55418", "38441", "2314", "51767", "48376", "6779", "40617", "46457", "44661", "16923", "41224", "40084", "19874", "42607", "44562", "14999", "52329", "28995", "12501", "44416", "4640", "33976", "57882", "11453", "35151", "39333", "10901", "24267", "33236", "10877", "2588", "14188", "36111", "13015", "30300", "19961", "53961", "22477", "59171", "56626", "47257", "50325", "54201", "33247", "1612", "21592", "49201", "22310", "10087", "19124", "56845", "15029", "35963", "9970", "54639", "27330", "10396", "42106", "11376", "30813", "39778", "35908", "9993", "3161", "48256", "732", "48970", "30611", "13505", "38257", "3708", "26453", "5264", "31203", "5339", "23390", "59571", "12923", "51605", "43951", "25935", "5132", "57809", "6333", "43817", "3457", "14564", "30746", "19104", "39175", "38005", "52393", "4664", "45926", "39787", "28041", "5991", "46888", "52350", "34049", "35364", "19953", "32363", "22506", "37202", "39619", "29924", "44603", "24350", "28141", "51321", "48988", "12936", "8665", "47400", "14570", "36416", "22203", "51833", "11922", "3485", "26736", "38243", "20192", "7630", "27324", "28606", "12200", "55986", "47207", "16863", "495", "24508", "44709", "246", "19956", "5000", "46802", "29431", "49581", "21083", "22184", "16608", "1899", "15463", "40031", "47880", "27896", "13839", "21569", "28003", "54241", "7900", "33033", "27051", "11981", "5520", "55666", "44115", "40669", "37586", "45260", "40848", "25426", "839", "52607", "7141", "52342", "26767", "9705", "266", "26125", "45898", "21836", "41006", "2219", "11381", "33435", "45804", "32889", "20644", "9883", "35219", "27560", "18505", "16662", "18077", "47877", "54414", "59145", "35032", "9911", "18254", "52584", "21983", "34929", "50427", "2806", "12048", "43584", "59321", "36543", "11765", "46626", "46078", "33662", "10729", "35328", "9177", "50126", "14036", "8527", "5183", "7108", "31603", "34799", "56400", "20949", "47523", "3726", "23266", "46248", "51116", "8858", "56367", "41182", "44134", "11470", "20576", "58386", "57192", "11538", "46857", "5033", "29470", "47863", "17696", "23196", "57920", "11365", "22181", "26846", "5567", "46581", "22120", "47511", "48281", "4447", "233", "58357", "38582", "54187", "49789", "4067", "35215", "1987", "19263", "38941", "6251", "55229", "15004", "32357", "2824", "55630", "21361", "2030", "24156", "49095", "16033", "25240", "21385", "3695", "2911", "7343", "37714", "23943", "17033", "50384", "56293", "2970", "22932", "49703", "53325", "12969", "55260", "49820", "36266", "11819", "50617", "18948", "59104", "22919", "45481", "14111", "40774", "6966", "40668", "7666", "48166", "7562", "2402", "46547", "29379", "38873", "43587", "48698", "305", "34451", "15265", "48952", "34826", "56195", "17359", "2460", "55859", "23847", "6110", "33061", "23689", "27532", "13681", "59315", "17420", "21392", "10445", "18484", "31844", "41096", "45127", "55814", "16517", "41513", "6948", "37884", "52452", "51388", "29278", "17899", "6596", "16859", "1342", "12836", "50793", "44740", "26449", "38271", "29664", "27177", "23545", "33945", "3808", "7289", "28536", "45571", "2357", "54064", "25847", "45746", "35730", "40509", "8815", "4143", "17419", "751", "7436", "3430", "15400", "29362", "31379", "26853", "35905", "32346", "32968", "4109", "31376", "7012", "46296", "45063", "42907", "13659", "5800", "57483", "42274", "6411", "36253", "8200", "58764", "21522", "14113", "24762", "27431", "25274", "26371", "27588", "46534", "34038", "35261", "10578", "17006", "20628", "47343", "52289", "17621", "32670", "9351", "41417", "48679", "12104", "50387", "20105", "35953", "40420", "53671", "52886", "18246", "31799", "14289", "10070", "2086", "27359", "7404", "11719", "34861", "49647", "15883", "7007", "2603", "48335", "9241", "10526", "10720", "49595", "17132", "3744", "56162", "30785", "36729", "12463", "26337", "56661", "15843", "30665", "27041", "46160", "36228", "20645", "58232", "17179", "25429", "7759", "53360", "28886", "25621", "14041", "15751", "47150", "23328", "42950", "53310", "18300", "47288", "16122", "10799", "56378", "23374", "50751", "32589", "14703", "31582", "50947", "17236", "20174", "1712", "27151", "49209", "42740", "19564", "10104", "18487", "26942", "45624", "36249", "35910", "26114", "12505", "2446", "48238", "1038", "40958", "58299", "57648", "33643", "49285", "33433", "22837", "12360", "9515", "25813", "22800", "4274", "19351", "14906", "17175", "15647", "8243", "33081", "7203", "28436", "5874", "9570", "28627", "42671", "29378", "54008", "13798", "6743", "30140", "32428", "34892", "16142", "1625", "7664", "26463", "24838", "37520", "17086", "6330", "18851", "14363", "21329", "58211", "33074", "29094", "33482", "48485", "17195", "36421", "20383", "46977", "56446", "29423", "14542", "50812", "17749", "59485", "47370", "25028", "3797", "45909", "23251", "8593", "18235", "1875", "28816", "40944", "17104", "48564", "4768", "31382", "15341", "25077", "24381", "52617", "5918", "8167", "57967", "36516", "35833", "41776", "19469", "12696", "54971", "39546", "18809", "14663", "9362", "44933", "44700", "15520", "55341", "36322", "46838", "55479", "59866", "46144", "51475", "18644", "7911", "51124", "35872", "47920", "3740", "11482", "51982", "38232", "39141", "35654", "6413", "37047", "45066", "7867", "57007", "52858", "42592", "53036", "4503", "37342", "17307", "45985", "29799", "3613", "42020", "19321", "15548", "55785", "46531", "9673", "23232", "38825", "38114", "55614", "26943", "46467", "9921", "27977", "39209", "54283", "55057", "36149", "2791", "26586", "26754", "48589", "45893", "5346", "58632", "7781", "49078", "54325", "23581", "54248", "8443", "56866", "23751", "21360", "51438", "8989", "42911", "30363", "5321", "31669", "26246", "26027", "47808", "2323", "40372", "47116", "52799", "24274", "22425", "48262", "25144", "55188", "17601", "18053", "961", "26766", "58317", "3820", "35489", "27533", "25377", "45153", "26349", "45983", "3704", "56754", "59140", "27872", "53735", "3680", "52383", "6255", "27935", "40118", "45713", "26641", "2614", "23995", "6194", "14677", "33225", "6081", "4041", "33758", "24083", "44321", "34326", "48049", "39206", "28009", "55114", "57190", "57250", "10623", "1353", "26600", "25211", "13152", "51817", "17827", "20065", "34109", "16355", "40370", "26828", "7635", "566", "54960", "5090", "41312", "21703", "28465", "38179", "28571", "53917", "20934", "5464", "11395", "39895", "1356", "55860", "22648", "22796", "33102", "21630", "15451", "43991", "25396", "53343", "20249", "35702", "923", "48671", "58462", "12014", "20654", "26322", "34306", "53316", "32255", "13084", "29062", "2786", "20139", "49223", "36879", "32860", "51632", "35635", "58103", "53055", "22044", "14823", "13866", "39528", "24186", "23654", "55887", "48314", "17034", "36340", "35010", "55490", "4835", "52252", "54113", "47908", "22251", "19776", "55976", "52528", "18297", "17986", "55109", "37581", "14600", "4049", "42495", "2739", "57538", "11103", "53498", "44628", "34937", "55105", "48414", "33189", "59981", "57527", "28106", "17454", "10934", "58725", "52102", "58925", "59233", "7682", "25443", "8357", "14249", "14680", "1829", "2317", "5123", "17032", "12582", "2355", "4152", "56275", "17689", "54179", "17570", "28701", "13385", "53381", "7394", "42395", "2245", "41645", "34222", "4985", "21887", "8863", "39197", "53599", "8064", "50950", "47528", "27940", "44801", "47079", "35180", "58252", "50858", "27044", "23042", "24252", "22363", "56891", "36030", "36052", "29708", "16871", "53601", "20750", "31030", "23839", "58013", "36566", "27159", "24717", "52834", "21112", "14396", "43845", "1450", "9417", "43369", "24607", "37739", "25304", "6105", "54900", "39543", "28443", "27197", "44003", "1958", "58641", "24642", "6788", "48577", "50659", "28551", "57064", "34800", "10797", "31692", "58058", "5217", "2854", "41801", "1921", "30708", "44434", "12816", "1885", "24149", "18813", "55261", "23255", "7880", "1618", "33044", "6983", "10142", "29981", "51966", "6898", "58049", "58193", "20677", "54865", "20586", "39455", "40267", "21026", "51383", "36898", "13804", "32768", "53659", "42175", "22651", "36102", "58877", "24410", "49977", "29116", "29322", "18855", "50211", "58945", "55758", "52699", "7371", "5504", "34632", "57382", "6968", "43949", "22721", "50690", "34726", "42873", "27382", "10520", "43487", "10105", "18166", "2524", "17826", "50640", "27756", "51284", "41714", "39248", "33789", "7095", "8808", "8946", "50318", "13249", "13613", "11700", "24864", "4944", "21914", "50485", "11427", "35982", "31407", "46875", "55798", "2503", "15876", "26709", "50036", "48129", "54448", "52727", "30961", "59411", "41098", "5213", "15069", "24700", "44720", "35285", "40359", "49054", "46163", "6612", "4421", "46086", "31544", "9833", "24490", "35778", "48875", "43224", "7947", "56534", "30433", "7328", "42108", "23784", "50727", "14778", "6042", "11465", "49815", "17811", "47996", "8484", "41326", "1803", "10223", "54007", "53868", "39218", "15296", "30866", "17260", "46562", "46901", "53421", "2726", "47067", "15411", "58091", "1808", "25792", "3863", "16081", "41583", "27568", "47163", "34223", "46246", "41726", "54989", "28373", "43250", "11190", "42138", "28023", "34823", "2954", "9867", "36880", "32030", "2482", "37476", "9558", "7577", "11867", "31470", "55996", "57329", "14826", "36847", "39050", "19689", "33111", "59055", "32782", "31399", "43081", "53224", "38863", "37150", "19931", "53078", "32894", "34614", "50522", "7264", "1975", "31272", "42797", "994", "52596", "23145", "28780", "22608", "34785", "11617", "29205", "2000", "37337", "35978", "3565", "19555", "45270", "3157", "3501", "38009", "58268", "59276", "5842", "32598", "41464", "19560", "25035", "24786", "11030", "16207", "17933", "43560", "49429", "45162", "29244", "33915", "4755", "45733", "10730", "8370", "18138", "55181", "51397", "48456", "6914", "10512", "49758", "8896", "8678", "14873", "36188", "9404", "18990", "2020", "50759", "18409", "2935", "7929", "54432", "27048", "17375", "10834", "28467", "18213", "15366", "19652", "44349", "29958", "3512", "48383", "42985", "15181", "30150", "45427", "20055", "27126", "40173", "32161", "32321", "42039", "43493", "45876", "42202", "45659", "33085", "41290", "35470", "47266", "9392", "25172", "5312", "7507", "36033", "8412", "14413", "6782", "27964", "39458", "53594", "42570", "38757", "42961", "45822", "11599", "35724", "27358", "9002", "4092", "15256", "59810", "6731", "45234", "36264", "44909", "31642", "18326", "58570", "49344", "20059", "46145", "44821", "12498", "27141", "4735", "20389", "39283", "19310", "24112", "57618", "49624", "17418", "55255", "53165", "55550", "20467", "33072", "57011", "45393", "54045", "48046", "45372", "59975", "20643", "13008", "45692", "43421", "45281", "48060", "11272", "57619", "46235", "44189", "59785", "36957", "39004", "59386", "26608", "38805", "58181", "56374", "7895", "44267", "983", "25960", "46682", "51247", "19909", "32239", "1952", "41452", "34401", "34817", "55512", "50265", "23800", "50051", "44169", "52066", "44088", "51581", "8912", "39625", "14131", "53455", "27207", "31038", "9606", "41820", "54611", "3581", "38311", "13801", "31753", "57030", "17647", "54584", "47852", "33082", "2125", "56031", "38265", "3063", "59787", "49777", "52416", "15926", "46644", "27615", "24355", "36350", "6716", "7217", "23665", "28494", "43767", "48336", "38573", "13663", "35435", "3802", "53564", "48685", "30904", "45430", "30679", "20887", "17556", "45071", "7621", "36165", "31978", "46997", "43108", "21389", "3523", "32215", "26893", "52205", "40386", "26443", "1229", "27472", "48169", "2109", "24413", "26486", "20435", "14150", "50140", "3531", "57591", "3469", "9981", "34390", "10733", "32019", "5966", "58263", "46301", "175", "18205", "46170", "40615", "50891", "20278", "4778", "26075", "35607", "19896", "20578", "48174", "21044", "23451", "45944", "35562", "3654", "52340", "29592", "44192", "52115", "32716", "22657", "50354", "45777", "8316", "33595", "29786", "12718", "55082", "6909", "52579", "36877", "40741", "59809", "30807", "33390", "53225", "33328", "7544", "6610", "28043", "50467", "55576", "10259", "33262", "48334", "40681", "16459", "37066", "7299", "46369", "58819", "34566", "42636", "3792", "34535", "33070", "16544", "57946", "58720", "30329", "3455", "7185", "56739", "14583", "44954", "5735", "54090", "33912", "16396", "15562", "1882", "47279", "36025", "13595", "59931", "22255", "51201", "50108", "58024", "42461", "4728", "48617", "48029", "13276", "21557", "3781", "53068", "29004", "13762", "20408", "54595", "28411", "26902", "24806", "11704", "33818", "57187", "49134", "26402", "3048", "17996", "47435", "40960", "28612", "21667", "47532", "35477", "46109", "3698", "30495", "32699", "43700", "44981", "21900", "9935", "38566", "12273", "35938", "29847", "26775", "41925", "36380", "49989", "16439", "24310", "30724", "31801", "19324", "10146", "30773", "19251", "16082", "21456", "18479", "17494", "13639", "57697", "30673", "54295", "53288", "51379", "8083", "12107", "30458", "43546", "32620", "41822", "27864", "36628", "14254", "21662", "13011", "50874", "19367", "23625", "2399", "39558", "36090", "7266", "34944", "5266", "10441", "41944", "1997", "26067", "18943", "21548", "14840", "50472", "1614", "41847", "36103", "56601", "21164", "1951", "50122", "49534", "18888", "46735", "10959", "11881", "30497", "44204", "50675", "725", "4505", "29736", "10525", "3504", "11997", "1422", "10605", "10531", "7948", "30670", "58527", "34284", "43540", "8930", "18871", "36470", "27994", "11344", "26612", "54207", "48652", "37744", "14066", "30185", "47477", "11118", "30756", "19746", "37732", "26002", "16830", "40096", "59063", "2904", "38771", "6987", "4297", "21149", "59507", "33446", "35907", "25246", "56526", "7003", "41278", "46716", "53470", "24115", "39342", "8918", "37768", "35521", "22455", "9192", "11241", "37124", "4155", "53431", "15031", "47481", "35138", "37934", "51271", "27978", "37481", "53748", "3915", "47378", "17869", "20007", "37001", "43516", "23883", "9586", "10265", "58551", "6336", "21993", "52218", "316", "12105", "59289", "33622", "31972", "16239", "11947", "26332", "10208", "15067", "45889", "40779", "13831", "11702", "30166", "2191", "42745", "16618", "52194", "51456", "49323", "12591", "29334", "21536", "10923", "36147", "11635", "3305", "54358", "30195", "5097", "48253", "56693", "2923", "14158", "15489", "10382", "9727", "1379", "6813", "24730", "49918", "16183", "53598", "25190", "56042", "17107", "21122", "51779", "43058", "44627", "48428", "55939", "21244", "7636", "11745", "6494", "52813", "55585", "43608", "47363", "23442", "22162", "34532", "38652", "43395", "16297", "32736", "42848", "637", "19563", "27981", "31899", "59378", "5972", "3656", "3297", "8263", "25494", "13019", "35992", "4897", "49230", "55837", "7818", "469", "3218", "36360", "40226", "54166", "43632", "49868", "25951", "27833", "26845", "54311", "54705", "47091", "26459", "22025", "23015", "33339", "58448", "23334", "36128", "454", "27717", "36060", "8603", "7199", "18435", "41297", "38484", "44198", "57350", "47889", "2259", "2865", "53898", "13478", "37871", "31971", "41644", "50487", "17636", "41465", "54458", "40606", "19530", "34327", "37343", "11249", "13242", "22729", "15082", "54083", "3542", "1609", "48624", "4533", "25513", "53894", "35376", "45815", "20874", "435", "39860", "1001", "19990", "12265", "31942", "53367", "38865", "30367", "29614", "42035", "54484", "14019", "50428", "25186", "51019", "35549", "33940", "43776", "30583", "51953", "53723", "54060", "40064", "17110", "44525", "30145", "32640", "104", "51183", "37177", "34271", "27426", "5513", "29904", "8445", "2767", "48140", "44731", "59721", "44219", "40852", "29337", "47153", "34735", "20128", "39606", "54112", "28227", "43164", "13131", "26597", "17480", "1764", "31125", "33312", "54477", "55765", "35590", "31160", "2248", "57561", "52726", "22198", "30384", "46955", "43048", "23488", "36943", "56882", "55068", "58472", "19065", "27787", "47838", "9756", "7282", "27071", "34912", "48177", "13417", "37519", "15977", "31385", "11768", "17241", "26922", "290", "17291", "13072", "56202", "50073", "53045", "35509", "5776", "51662", "34095", "22149", "48384", "303", "45267", "5294", "48062", "47624", "22381", "39211", "55867", "49256", "53931", "31657", "36408", "53411", "16534", "3543", "55669", "40955", "55656", "17656", "56903", "36809", "2193", "15030", "2292", "22252", "14305", "32857", "44056", "32440", "40047", "17670", "1685", "58119", "5903", "21715", "3713", "24256", "5256", "14103", "57038", "20602", "38558", "34796", "7144", "21668", "20709", "53745", "17329", "6007", "23116", "11154", "5257", "33083", "39256", "8833", "41608", "19362", "30949", "16186", "33058", "9282", "57816", "41637", "51218", "32028", "59761", "20109", "53450", "13897", "6878", "57494", "949", "54054", "28947", "28639", "33867", "30751", "41987", "33320", "30033", "9933", "5912", "7980", "30963", "15404", "23337", "58682", "7014", "43073", "34921", "53650", "12136", "36286", "24661", "23367", "8010", "14759", "22501", "19952", "17452", "20646", "9592", "22929", "19756", "10866", "20318", "31317", "37134", "9576", "10914", "33487", "419", "29623", "41148", "25678", "38429", "15813", "15077", "10964", "45122", "8794", "56651", "34544", "12366", "50002", "16685", "55828", "54817", "50152", "43260", "11008", "57526", "34412", "10023", "5249", "48713", "39991", "45573", "7383", "8430", "31704", "2256", "52578", "43937", "56291", "17347", "40171", "34598", "31159", "14700", "41633", "770", "36759", "52081", "36948", "23486", "41476", "25604", "8262", "46568", "5785", "16909", "35214", "29424", "8463", "39363", "34893", "57312", "5669", "25155", "41945", "37997", "8908", "10424", "6222", "9629", "28499", "56046", "51879", "25638", "7662", "35185", "54351", "25523", "41780", "56477", "46407", "11455", "42037", "58504", "35250", "59313", "28146", "27911", "15110", "1881", "42068", "23113", "38957", "4480", "48328", "21950", "44587", "20589", "43520", "29005", "31970", "54500", "46586", "33792", "45730", "28796", "34939", "27707", "31031", "27749", "41172", "19107", "12753", "32297", "45271", "20614", "24947", "30574", "56504", "25449", "23593", "16563", "24595", "28742", "44429", "42714", "12342", "58834", "21984", "23471", "56659", "11762", "20449", "41900", "42399", "43637", "4107", "47417", "619", "56223", "9215", "29616", "2866", "47502", "29212", "27105", "24955", "34353", "1213", "27950", "33055", "3514", "375", "36411", "11441", "6889", "12539", "25043", "33582", "27638", "27684", "39848", "48280", "42663", "12323", "55577", "51848", "4207", "22798", "47261", "13161", "19287", "57620", "39029", "42229", "48501", "3540", "24628", "13067", "10180", "39040", "10278", "18955", "2782", "48905", "39736", "32056", "44907", "41905", "55514", "54782", "38163", "1291", "14279", "2674", "44505", "4622", "53754", "36084", "16310", "51791", "5140", "32125", "36493", "16898", "28252", "39500", "27333", "15197", "50497", "26416", "7932", "12650", "18431", "2493", "40961", "17367", "56033", "46791", "27031", "18746", "47834", "50273", "36027", "5443", "59463", "1725", "59127", "8190", "54907", "52401", "8349", "10310", "31210", "59919", "10740", "33918", "29371", "12763", "39655", "23000", "42889", "29020", "7255", "57771", "52937", "16189", "9228", "20796", "21255", "30942", "26340", "22756", "59878", "7447", "30567", "41860", "40158", "25512", "24977", "57391", "25471", "3859", "47446", "25753", "21048", "50034", "24621", "14802", "17783", "26037", "32375", "53348", "8259", "13213", "28990", "15091", "44378", "58467", "30090", "16273", "34880", "6650", "27180", "5997", "5108", "35192", "54150", "37357", "12394", "29161", "11661", "10286", "58282", "1066", "56871", "4920", "55721", "55702", "45791", "55103", "55355", "15159", "53054", "3404", "32835", "40413", "45721", "27774", "47204", "25045", "44560", "30732", "2173", "6277", "15219", "53817", "30901", "48604", "35659", "11500", "29331", "57024", "7819", "18253", "42129", "54079", "19094", "10462", "54550", "50748", "21846", "24301", "23273", "3748", "5101", "14166", "39580", "59409", "32673", "25114", "23269", "48579", "22078", "45230", "13428", "33987", "31895", "1700", "42136", "13364", "2388", "40314", "54684", "30547", "4546", "38817", "34234", "48500", "15789", "36518", "5387", "54315", "33732", "9824", "46203", "7579", "35856", "54791", "5930", "30639", "4584", "10428", "48002", "39743", "51689", "28486", "1512", "6761", "43613", "753", "44611", "2787", "43844", "50137", "24748", "51574", "31545", "40945", "18237", "55898", "43927", "24704", "53768", "47309", "3171", "17700", "28860", "54778", "13455", "40678", "26617", "12976", "17185", "49329", "6107", "4629", "29054", "40300", "27544", "29665", "59526", "52217", "2034", "19322", "59905", "44507", "29287", "2537", "26348", "53992", "42187", "40714", "24021", "47994", "42503", "38182", "59150", "21832", "22497", "12712", "50364", "9379", "15691", "10686", "43492", "24434", "28962", "47167", "38641", "49574", "44143", "46265", "40108", "58815", "31028", "10243", "56932", "23111", "40786", "26539", "48077", "19549", "45354", "49503", "34739", "57861", "35577", "25858", "5041", "23695", "36801", "37910", "27195", "20261", "52870", "34489", "3948", "30014", "50150", "41918", "17013", "35530", "57315", "53208", "47441", "55454", "41593", "19787", "2645", "52239", "34560", "12762", "51305", "1155", "30004", "45661", "9306", "11898", "41544", "8367", "25037", "56991", "28228", "24080", "6586", "50730", "38630", "54498", "17088", "48187", "17118", "31905", "20182", "58684", "3018", "34753", "42681", "47683", "56985", "24203", "29637", "48822", "19941", "36095", "48427", "54902", "22207", "8273", "10161", "8999", "14182", "38475", "44531", "39696", "41375", "25327", "47581", "51624", "36439", "33459", "11581", "18221", "5858", "46069", "58264", "25571", "30056", "54162", "53177", "38883", "10828", "23220", "59146", "54476", "9414", "24486", "26438", "40007", "6598", "44498", "55946", "18556", "29234", "2738", "9428", "24217", "20210", "16572", "6693", "52527", "3484", "46565", "7644", "51401", "18708", "21967", "30229", "22069", "10591", "43066", "51515", "15543", "54685", "23828", "51092", "37630", "8789", "57110", "25798", "42098", "9068", "12094", "5035", "15879", "2013", "22165", "15967", "33122", "32895", "27439", "54507", "29627", "25308", "47209", "5957", "13844", "38209", "19360", "21034", "44199", "35248", "6555", "12306", "52628", "58212", "34750", "24097", "35600", "46897", "7736", "28030", "954", "26814", "4199", "50664", "46270", "47074", "43747", "49348", "33570", "28147", "30136", "49787", "51554", "25097", "29684", "41323", "5835", "38369", "56908", "40525", "41952", "30803", "12106", "9826", "55646", "48859", "8380", "44859", "10691", "34918", "52749", "40245", "42386", "52362", "18132", "44809", "31001", "14578", "28790", "23782", "15246", "40163", "39254", "24265", "51723", "45438", "43866", "26868", "55937", "3121", "42392", "25876", "21547", "34573", "3836", "15686", "1613", "42132", "23479", "14204", "42016", "5165", "34710", "49721", "58050", "11694", "10518", "40719", "24128", "2072", "21370", "34012", "8792", "18438", "3975", "24152", "18058", "48127", "46130", "53129", "39998", "11805", "23032", "15", "769", "37271", "19133", "56265", "2772", "59699", "1237", "3369", "56881", "39322", "20650", "4376", "20457", "51022", "32740", "42755", "15018", "58596", "35142", "9387", "31887", "47759", "20036", "21188", "5936", "18066", "36296", "15156", "40463", "25939", "17744", "38512", "36843", "1491", "26373", "7339", "5413", "6406", "38859", "55871", "31322", "16013", "33652", "14399", "30489", "28470", "28353", "52629", "18358", "41386", "31630", "37084", "14607", "29773", "55952", "6579", "22954", "33937", "7851", "30420", "52665", "16530", "25444", "41140", "26392", "57044", "7320", "4639", "28317", "46752", "5034", "34034", "36906", "29175", "8163", "16126", "47248", "29628", "40183", "39636", "23212", "37991", "7795", "2102", "29689", "37669", "1513", "3585", "25165", "50087", "26667", "7340", "19661", "59786", "30086", "5009", "57155", "36075", "35436", "52890", "53272", "32023", "13759", "56302", "15720", "44814", "569", "51415", "47380", "13473", "16825", "24377", "54374", "26094", "27822", "37862", "38846", "31493", "48353", "58424", "34831", "24925", "5283", "49760", "19845", "4007", "2353", "36472", "14598", "41498", "30989", "24841", "36042", "18739", "12137", "8241", "18722", "8102", "17548", "35487", "21929", "18476", "47841", "5974", "3564", "8219", "15430", "50307", "53341", "35086", "7709", "40199", "21683", "29916", "44307", "57163", "36134", "54697", "32444", "32278", "18087", "12624", "6184", "26877", "36400", "19112", "29758", "31941", "47313", "45374", "51757", "48782", "59680", "35196", "41321", "33481", "14235", "56338", "796", "45873", "49936", "3", "975", "6738", "22640", "8224", "9312", "36806", "56848", "47489", "36647", "49592", "8218", "32291", "60", "28911", "30836", "844", "99", "21070", "5927", "4028", "54663", "25762", "37007", "9102", "405", "507", "19758", "4902", "59162", "5181", "52960", "15581", "42894", "58737", "15131", "48128", "22532", "11489", "32117", "56343", "26053", "59257", "102", "17415", "23957", "27471", "57521", "739", "24639", "53685", "31677", "52458", "49335", "20357", "16391", "17446", "57506", "41689", "28978", "9692", "7623", "33523", "38572", "24448", "41311", "41941", "32720", "33991", "48518", "22005", "44177", "59517", "21827", "45036", "16725", "2937", "59767", "781", "15284", "1425", "31761", "584", "6742", "6881", "42113", "21898", "38364", "56412", "4524", "53445", "25579", "43464", "28298", "15915", "31226", "1831", "2315", "31218", "27637", "52946", "41222", "6714", "15726", "4834", "57508", "15360", "21448", "29504", "40417", "22085", "15207", "7601", "39434", "9631", "31062", "38580", "37327", "38412", "48966", "27917", "33804", "51156", "10209", "45776", "10233", "31242", "40197", "52351", "7240", "28431", "48593", "8625", "16628", "59067", "57969", "40209", "26131", "39777", "2720", "10630", "35937", "5175", "59531", "53221", "20722", "47442", "26176", "41001", "7019", "17987", "30804", "54203", "31263", "14590", "50336", "4477", "11414", "22342", "58827", "26153", "37631", "24372", "45557", "55582", "2488", "55510", "3499", "57669", "13648", "11501", "34835", "57708", "11969", "16949", "37567", "36306", "53025", "32919", "5797", "4637", "29273", "26727", "28190", "11751", "48937", "55312", "11323", "44520", "59684", "46298", "22672", "23538", "25704", "24785", "53902", "19675", "25597", "37546", "57987", "47473", "26078", "813", "42486", "5943", "53752", "10024", "4394", "35874", "57609", "46822", "53613", "43078", "16450", "12654", "50672", "15679", "17384", "40530", "54785", "45000", "59850", "1891", "38924", "31109", "36562", "50250", "27435", "41612", "23525", "12349", "53885", "25309", "13407", "5351", "22199", "15742", "43056", "55033", "5344", "24706", "55708", "39087", "3684", "17911", "4723", "5255", "12270", "32526", "37659", "50088", "38409", "55381", "23076", "45055", "34250", "51807", "26016", "37897", "8109", "20396", "442", "21923", "52169", "52682", "45966", "10696", "31958", "27783", "10002", "37778", "25276", "9774", "22728", "36932", "7498", "25930", "8493", "12703", "48934", "43591", "18322", "15479", "33882", "54522", "9560", "42595", "9318", "50288", "52091", "53626", "37091", "45773", "52924", "35237", "56017", "652", "10264", "45086", "2644", "3230", "16827", "43173", "37655", "34281", "20951", "3433", "45771", "2890", "43370", "39510", "46419", "618", "3292", "13606", "16924", "49733", "8352", "25300", "50243", "37461", "9088", "21322", "58552", "52792", "828", "59758", "15098", "23325", "29283", "52260", "12123", "26723", "24284", "6916", "15759", "55392", "29726", "42596", "16095", "3864", "1514", "5232", "10454", "34460", "4494", "12894", "40544", "21422", "34686", "41267", "16941", "7634", "1765", "41065", "30745", "55647", "1093", "26972", "1444", "28764", "54889", "21294", "49866", "20440", "4150", "53134", "4608", "32002", "59433", "14573", "52936", "59870", "30997", "53353", "44549", "34659", "85", "52583", "14132", "25774", "11980", "35381", "46576", "38723", "15100", "13252", "14361", "4957", "620", "17166", "27598", "34257", "9608", "3154", "9659", "57807", "43121", "53840", "46551", "44406", "42450", "37314", "56380", "14987", "52247", "1297", "39330", "4251", "52465", "19856", "13590", "19325", "24412", "21970", "15662", "27419", "5092", "33264", "47780", "40776", "21864", "18625", "34009", "10919", "432", "32355", "20455", "23971", "25349", "29629", "56902", "27392", "12864", "12532", "43934", "24867", "53106", "51188", "17804", "20307", "55302", "59770", "29145", "51945", "37710", "15128", "18542", "57602", "52734", "33866", "55588", "2548", "44070", "33355", "7838", "11526", "36693", "5857", "37855", "55808", "26404", "35782", "22901", "57097", "2346", "45265", "59237", "41832", "32906", "17366", "15329", "57236", "3136", "45630", "24518", "50089", "55678", "50410", "20777", "40024", "1324", "32548", "55409", "33422", "46171", "50631", "42351", "13918", "16286", "40982", "18740", "45176", "52478", "44195", "19729", "34563", "14653", "57156", "54333", "24612", "49010", "43406", "13210", "23122", "32120", "21043", "11553", "8865", "28576", "39979", "29527", "47182", "44862", "54863", "1761", "50949", "16075", "5131", "52235", "9187", "44231", "46843", "51273", "43192", "30943", "2623", "21885", "8594", "24729", "14554", "58908", "1931", "7698", "39474", "25135", "35409", "3399", "567", "35545", "55571", "30065", "18036", "2891", "57197", "45370", "58320", "23621", "32841", "51129", "31771", "44614", "23752", "57042", "56655", "10344", "31266", "57610", "14768", "54131", "46680", "8773", "8250", "27548", "7021", "18494", "50873", "537", "26996", "14924", "1818", "54022", "9455", "22942", "28386", "59674", "48668", "26588", "48034", "2690", "1999", "20057", "38749", "10153", "54027", "47327", "38686", "42255", "37043", "19976", "2242", "13661", "48928", "11882", "431", "57128", "50225", "49570", "45145", "37750", "21147", "26908", "29852", "33199", "46933", "52973", "19261", "58536", "25279", "29928", "2392", "18340", "2192", "39295", "10128", "13035", "22001", "54690", "41794", "23676", "54888", "30834", "27222", "52656", "26874", "23931", "4279", "31401", "22095", "44892", "21785", "39578", "13121", "22976", "45846", "56842", "6027", "14517", "38053", "17294", "196", "36315", "42991", "37731", "34247", "55", "18798", "14232", "47002", "11776", "7792", "42693", "38491", "43677", "34043", "17920", "7256", "46101", "24840", "19303", "57854", "42993", "19129", "13043", "25536", "54450", "59285", "7722", "726", "26166", "32722", "28759", "14627", "38795", "21037", "27962", "56578", "33897", "12151", "40067", "23112", "29985", "34052", "57153", "1756", "42510", "11556", "17617", "24311", "17628", "30467", "19506", "28712", "34858", "40694", "53465", "6166", "49180", "38138", "59542", "40442", "37491", "42943", "46468", "53956", "6617", "48552", "13628", "54242", "7489", "44309", "9545", "7590", "37517", "53114", "2996", "26264", "23633", "41843", "9350", "41795", "49527", "47319", "55127", "39129", "7452", "3277", "3715", "12209", "57966", "47385", "47458", "6679", "50980", "12520", "12656", "2586", "14486", "48375", "57346", "4583", "12357", "37262", "15439", "47681", "35203", "132", "1823", "42579", "58722", "15261", "46743", "16110", "4586", "50132", "48191", "46865", "16452", "5658", "28614", "51034", "49811", "35572", "12031", "36517", "33743", "5179", "10853", "43468", "19448", "23425", "3090", "3846", "47339", "10874", "9823", "41514", "43227", "24788", "54788", "48923", "26132", "16982", "59790", "13099", "35195", "36656", "9401", "46443", "41059", "12157", "40829", "48986", "47336", "12121", "34508", "38705", "54001", "30472", "38173", "14506", "57909", "22996", "13763", "12428", "36491", "37140", "42207", "35400", "43166", "29694", "59202", "28908", "36467", "47951", "13576", "7496", "46500", "35867", "31867", "2338", "57478", "29899", "12059", "54142", "58208", "48690", "22142", "47131", "44594", "54183", "39463", "17787", "59254", "41393", "8641", "55315", "13904", "35144", "50612", "55423", "5059", "44371", "27654", "18668", "48684", "29355", "54504", "45847", "3236", "32459", "50015", "3352", "14797", "30177", "54552", "3333", "13826", "32124", "26471", "14406", "22710", "55095", "37705", "40634", "46441", "22670", "38643", "42452", "13464", "42300", "51863", "39690", "20824", "42601", "53209", "49732", "6357", "51243", "30776", "47495", "53385", "8386", "44869", "25840", "48325", "57505", "59973", "46577", "41891", "32126", "17172", "44849", "29459", "29602", "25235", "16170", "36462", "45991", "53032", "9598", "12693", "19227", "43824", "271", "16504", "56943", "15134", "14945", "30113", "14897", "9714", "18055", "56933", "12421", "40371", "49837", "14483", "54312", "58665", "10702", "10679", "15505", "59695", "50646", "18455", "30873", "3350", "10961", "33418", "31383", "6993", "53636", "39962", "49208", "467", "16271", "10944", "38619", "23696", "49237", "36484", "49510", "50578", "51523", "7357", "27005", "2420", "5261", "18388", "12895", "26172", "17653", "22008", "45443", "767", "1867", "31262", "56268", "27907", "20946", "53489", "15844", "33421", "23875", "12934", "55605", "43422", "47815", "10904", "14658", "35675", "49541", "2156", "21607", "53770", "28916", "44202", "41187", "21559", "48746", "32261", "33850", "9968", "35083", "45938", "1641", "45149", "37143", "5518", "13405", "41826", "9952", "47465", "26598", "4736", "47452", "7530", "20861", "15487", "8528", "4506", "47080", "51541", "28004", "10501", "3211", "5576", "53262", "32399", "49513", "35437", "41548", "12292", "49038", "4956", "33591", "19128", "39681", "50166", "33829", "15204", "21223", "5571", "6752", "56061", "10422", "3586", "7963", "18572", "22080", "31380", "49512", "24576", "24609", "10280", "23043", "43784", "40317", "20015", "48553", "48591", "8774", "44125", "4851", "374", "49024", "54455", "12154", "45540", "53079", "40046", "42884", "59305", "54516", "54573", "43868", "30064", "50517", "8485", "7672", "4408", "54154", "7267", "56660", "48122", "7334", "45079", "59649", "6409", "9735", "2827", "53155", "1320", "52804", "23066", "53254", "59277", "57150", "48069", "23618", "45465", "18152", "32866", "22108", "7058", "32419", "19480", "21446", "40939", "22712", "56934", "12538", "27603", "10221", "54356", "1172", "48810", "28263", "26252", "40473", "45959", "2801", "57443", "18229", "28757", "9566", "47389", "22564", "705", "52748", "5176", "49906", "49519", "14912", "30797", "50432", "41333", "13979", "7756", "39496", "25752", "10291", "30631", "45286", "29666", "25999", "50902", "22724", "8677", "664", "3548", "14691", "9359", "40543", "27123", "18611", "36960", "15175", "7454", "20470", "58849", "10820", "21940", "50777", "30365", "15484", "8121", "3789", "31162", "35262", "47905", "30940", "16249", "3187", "45969", "27825", "967", "30577", "19959", "42845", "5289", "45031", "41099", "25642", "7229", "20359", "21458", "41570", "1260", "39062", "44295", "25554", "58770", "48300", "58695", "55005", "51425", "28733", "13709", "40423", "18901", "33348", "49653", "42999", "54305", "41922", "20960", "36392", "43018", "47909", "35700", "50396", "40036", "20966", "184", "27984", "51444", "48672", "12858", "4317", "54434", "43311", "53472", "45101", "58745", "13280", "18748", "5392", "48070", "58718", "51713", "15615", "58059", "8984", "52965", "17089", "43992", "14767", "9309", "40129", "38008", "13614", "45857", "7502", "12595", "55949", "56323", "38150", "34780", "14537", "59111", "57037", "49795", "304", "35682", "9108", "59603", "57892", "52096", "11959", "31302", "38670", "10921", "53356", "16444", "13409", "14977", "3135", "7921", "26625", "56886", "10932", "4526", "33212", "40055", "9668", "5888", "2247", "43331", "36271", "13603", "47777", "25379", "6172", "32483", "10909", "3782", "2202", "10305", "34291", "51830", "38207", "6577", "14460", "41812", "30431", "22237", "5366", "39520", "2244", "7192", "26072", "18212", "30328", "37833", "18950", "49457", "38238", "50811", "25145", "52236", "20113", "3832", "19151", "32788", "8084", "57543", "6299", "1539", "5103", "59605", "32932", "58306", "14712", "45474", "28157", "32383", "445", "48933", "56352", "33002", "28835", "5712", "10194", "48354", "30799", "59163", "58243", "29567", "16928", "40505", "28706", "25693", "56493", "57694", "33258", "44007", "1979", "58916", "47661", "2794", "50172", "38439", "10778", "28480", "32989", "18377", "58825", "27863", "18298", "7953", "39899", "21947", "50434", "46832", "21359", "32974", "50871", "4304", "42204", "30162", "29410", "47085", "55286", "15834", "2731", "19164", "23041", "40450", "49701", "47189", "37079", "53967", "46947", "36205", "43993", "25538", "16448", "48558", "10653", "25699", "59083", "8777", "2696", "40405", "47169", "51344", "29934", "47829", "17275", "59432", "9209", "55726", "38744", "54542", "16670", "45706", "50383", "25641", "31974", "719", "29311", "198", "58413", "49621", "45310", "47728", "9709", "38856", "55565", "17646", "24968", "36024", "35758", "44922", "4301", "22513", "23103", "52220", "34131", "7287", "42517", "26158", "47564", "48185", "33825", "42343", "15809", "7293", "28267", "54905", "35034", "890", "46417", "10606", "1449", "32925", "17027", "3956", "4223", "56918", "21243", "30439", "51103", "1386", "30181", "56276", "2359", "45952", "10508", "12098", "25831", "59910", "29481", "48784", "38353", "10419", "18526", "42032", "32206", "38844", "29976", "14282", "7575", "52920", "52222", "44248", "48880", "30423", "9991", "15142", "4078", "22153", "54399", "21146", "32630", "7490", "12790", "20313", "3555", "17201", "7742", "15721", "35114", "30105", "5243", "17704", "7617", "14894", "17217", "40666", "22652", "47395", "34757", "29342", "15539", "46465", "37442", "700", "36427", "38347", "539", "35820", "6227", "44032", "21055", "38778", "12659", "34229", "16866", "30417", "28236", "19439", "4310", "28339", "35914", "17138", "40116", "38450", "1672", "43623", "15005", "42798", "7539", "27955", "20618", "10742", "21517", "18916", "29788", "25722", "54829", "11597", "10418", "27594", "26466", "31673", "3510", "20783", "17361", "5971", "25417", "14043", "7622", "21782", "43504", "46703", "2410", "34307", "40286", "34951", "10057", "21527", "11174", "33275", "57241", "47795", "15799", "56715", "59187", "4214", "47871", "4221", "9381", "34626", "4864", "14588", "34787", "31146", "22156", "10707", "37370", "26052", "4211", "36258", "5875", "28617", "32412", "36285", "8358", "54034", "42836", "13032", "46051", "27042", "48519", "27386", "1522", "21245", "45930", "25797", "6003", "30041", "19381", "46931", "43142", "32503", "43689", "21247", "26824", "9954", "16244", "6094", "31392", "58889", "17644", "26966", "36712", "42763", "45336", "57032", "23968", "48573", "39264", "29218", "18440", "32934", "9042", "9330", "8052", "44154", "53331", "8141", "51645", "17377", "54259", "4589", "43796", "27327", "38754", "1913", "57705", "26515", "44870", "48418", "58757", "37038", "50284", "12783", "56820", "29690", "24598", "8781", "18000", "278", "17133", "4292", "39118", "49389", "54412", "10001", "30053", "26209", "22896", "26581", "55469", "33879", "28055", "48657", "12397", "5427", "48082", "29393", "48195", "41291", "28873", "35904", "37215", "39536", "41655", "50973", "15517", "27428", "43236", "16802", "53213", "40993", "24216", "25853", "58820", "59050", "26405", "21285", "58345", "39030", "18770", "48666", "28346", "31819", "9254", "8864", "53973", "10435", "48717", "4668", "7892", "19511", "56246", "26970", "29663", "35097", "31575", "56658", "3580", "43510", "27193", "1049", "40676", "56862", "3472", "20136", "11853", "34865", "40879", "24300", "30175", "38719", "25329", "40787", "13321", "41074", "46368", "6985", "44938", "28045", "6534", "54555", "30396", "23089", "30466", "37350", "18989", "4326", "9336", "56354", "403", "17518", "13354", "15184", "54799", "1908", "44671", "35474", "33708", "53939", "8444", "42210", "14830", "31550", "19400", "32807", "41030", "36131", "49774", "31339", "41692", "46686", "16097", "22514", "12350", "52381", "42509", "25570", "19299", "55997", "53047", "26851", "52037", "48919", "3432", "54370", "6432", "50681", "51269", "52638", "39602", "53253", "5894", "33049", "46268", "1218", "7228", "16024", "19470", "2184", "57476", "28249", "3459", "31767", "25289", "17789", "2331", "5664", "54390", "22488", "42193", "47548", "52369", "39931", "2512", "11730", "4091", "40134", "39270", "51053", "4424", "39940", "40810", "35916", "297", "7294", "30632", "27248", "7196", "51329", "30643", "23691", "52250", "15257", "19371", "33865", "19142", "24068", "14862", "20611", "5986", "23529", "10126", "20280", "49228", "2529", "29009", "16778", "23843", "41928", "54996", "19313", "6187", "14628", "26778", "46004", "9803", "9025", "33341", "28216", "18276", "38684", "1683", "3842", "22350", "3672", "17250", "10062", "55834", "19070", "52591", "19727", "54347", "17545", "41969", "39221", "47299", "50357", "21964", "11618", "9916", "45460", "5585", "34375", "37410", "38", "14220", "50913", "45515", "43000", "56079", "44503", "21803", "21652", "16986", "32405", "3493", "1753", "38221", "44924", "8934", "57530", "8037", "52711", "43175", "44351", "49599", "2509", "22650", "47676", "15227", "8398", "37835", "21555", "27583", "33245", "10402", "20018", "596", "6820", "18131", "9738", "48180", "2907", "20693", "58950", "32367", "35502", "10164", "53095", "27115", "6030", "32583", "50439", "57834", "21772", "33273", "26974", "20984", "41292", "15147", "41485", "41824", "41439", "54322", "45980", "16592", "56725", "58542", "47767", "4716", "2842", "49898", "50942", "27066", "51756", "20661", "18983", "57500", "47415", "27496", "14364", "8137", "36044", "10486", "162", "34189", "45425", "50668", "14779", "33034", "17249", "54101", "38225", "15815", "28739", "23884", "21224", "2183", "39166", "29247", "19100", "19245", "54297", "8536", "762", "53407", "26504", "20429", "23389", "44425", "13029", "23889", "35236", "49363", "6748", "14347", "28652", "34818", "58587", "16156", "17809", "15188", "2284", "15570", "1947", "8562", "11893", "27604", "48530", "44519", "5088", "15605", "51039", "16315", "44325", "9652", "19399", "1146", "49312", "27456", "36807", "39151", "38968", "27411", "57595", "10779", "1877", "43815", "28747", "35205", "27558", "32502", "54378", "24167", "30218", "1462", "34881", "9931", "41424", "23282", "14787", "17488", "56683", "7645", "39994", "58319", "11137", "47001", "39782", "22638", "6344", "4945", "35631", "50958", "4077", "573", "26551", "7221", "46812", "15367", "5265", "23636", "10483", "56277", "40624", "3844", "44468", "52932", "26768", "20884", "32854", "25653", "43404", "46223", "38046", "677", "29644", "52778", "42253", "1577", "35986", "48947", "51384", "58930", "48608", "59716", "48557", "11443", "47304", "23235", "31747", "37358", "35047", "56890", "13861", "56067", "30877", "40695", "43602", "51240", "12945", "1747", "30798", "18869", "40740", "23270", "57972", "43315", "56650", "54794", "4887", "59775", "12691", "8755", "3386", "35282", "12709", "9128", "47381", "30621", "24221", "47427", "3074", "51125", "7989", "37564", "46390", "20802", "3996", "30895", "14003", "35260", "5724", "44675", "46911", "54689", "29869", "37489", "29993", "49354", "58965", "33853", "14344", "53827", "58473", "51558", "28006", "41946", "18976", "46388", "31049", "281", "4277", "50395", "14319", "39414", "12754", "48559", "4881", "55390", "25301", "49433", "47146", "7278", "38315", "7899", "51832", "15919", "49891", "1373", "16625", "39124", "52051", "29805", "44891", "52471", "58892", "10076", "748", "27965", "19018", "23606", "21312", "20864", "9383", "31117", "6622", "160", "48139", "6717", "12278", "23918", "1841", "32111", "50752", "20413", "45902", "46470", "14141", "34545", "5801", "37268", "31601", "14844", "33383", "10322", "31543", "54069", "11787", "17965", "56040", "55240", "23412", "2422", "55542", "20347", "38325", "10654", "46943", "41524", "25955", "38378", "30313", "54422", "3690", "46556", "56450", "31424", "47142", "39128", "9296", "7315", "613", "44218", "7534", "42485", "42842", "12198", "22295", "50210", "11358", "28113", "21437", "1266", "24680", "33908", "15468", "50328", "36221", "51126", "9202", "26862", "2169", "2598", "13744", "54763", "24341", "20273", "38064", "33118", "47883", "17586", "56258", "247", "3774", "47977", "46277", "14175", "45386", "34846", "44963", "18838", "27745", "40784", "8727", "45130", "10538", "20125", "8296", "41639", "25860", "9658", "20776", "15931", "56893", "37889", "27046", "9486", "10657", "41683", "44863", "13284", "43865", "40622", "30881", "7690", "28272", "36032", "21375", "30260", "18289", "6253", "47711", "10553", "41154", "5240", "10327", "29070", "18478", "33771", "16574", "45485", "51155", "57453", "26987", "57216", "2119", "4666", "52318", "53435", "28879", "14788", "2756", "47786", "45531", "26511", "58340", "43735", "35800", "39483", "29165", "45328", "51720", "46116", "3765", "23245", "16031", "27512", "28806", "22704", "5337", "11573", "24971", "12723", "35427", "6800", "45974", "42669", "30299", "14689", "59375", "10671", "39873", "43267", "58082", "41445", "22304", "34311", "58280", "57310", "3050", "53903", "44984", "44648", "58574", "56755", "27755", "4130", "51596", "54953", "36010", "5162", "51990", "42527", "46996", "12249", "53279", "4631", "1850", "48965", "28197", "43808", "41740", "24646", "59937", "14368", "34717", "44970", "35149", "18648", "5496", "186", "11255", "56199", "18234", "55074", "27674", "58287", "10165", "6613", "6627", "9699", "4457", "27365", "38527", "35696", "54006", "59094", "9056", "10768", "52476", "3477", "57244", "43559", "37285", "8255", "11439", "46041", "18210", "31364", "12782", "38300", "19978", "48805", "54073", "12898", "6749", "28514", "2718", "18464", "41397", "21787", "49440", "25937", "8790", "16361", "18312", "27503", "3047", "36508", "18472", "5230", "14921", "13797", "57431", "3024", "56407", "8767", "27440", "3287", "55405", "38305", "14437", "12077", "51996", "48917", "41041", "46089", "39997", "51043", "1623", "20177", "35799", "24010", "12348", "17754", "24659", "398", "23404", "46028", "21650", "23791", "13926", "3250", "7227", "29949", "31506", "34564", "929", "24860", "3482", "4518", "9071", "37188", "24086", "30748", "57077", "22445", "13045", "1936", "16620", "27089", "50541", "39790", "32867", "43049", "36541", "19868", "37211", "1710", "59734", "57933", "52411", "10747", "54840", "29703", "39144", "7684", "5593", "4690", "20330", "53461", "7002", "25740", "26935", "56575", "44679", "1033", "48851", "13153", "18194", "10583", "19265", "59920", "14694", "42159", "31599", "56111", "57890", "13501", "2810", "34832", "54189", "3719", "40182", "8303", "55104", "11346", "4467", "26678", "40984", "6439", "38431", "28352", "4605", "28858", "39303", "25899", "33450", "11347", "46311", "59148", "39038", "27176", "24727", "54570", "59185", "22245", "33833", "29721", "13894", "49475", "13004", "58643", "382", "38591", "21434", "28902", "57229", "18363", "17462", "4435", "51766", "2882", "36314", "56049", "11407", "55135", "54268", "45472", "16805", "58937", "7026", "32273", "56134", "35367", "27753", "44341", "13660", "56405", "54704", "59211", "2959", "39375", "48909", "13920", "59033", "28149", "16295", "28049", "17450", "17477", "55366", "2740", "28283", "18821", "55403", "34986", "56576", "54982", "37119", "33978", "52338", "52353", "17995", "31963", "47132", "24627", "7209", "43997", "57655", "34769", "8804", "1990", "44569", "30800", "1866", "56527", "33751", "57640", "52504", "7345", "9635", "1311", "46606", "14853", "37850", "46877", "17824", "47599", "8802", "24222", "2792", "8889", "50804", "27027", "9929", "36745", "36779", "33373", "10515", "52165", "58080", "52014", "7703", "11176", "18830", "47177", "52688", "29257", "21625", "23329", "19534", "54403", "21097", "26700", "50134", "53558", "44129", "3933", "38153", "12872", "3173", "32884", "30986", "8910", "21481", "54429", "5629", "12238", "49384", "587", "50820", "35936", "46084", "58060", "4103", "50728", "28655", "8587", "27894", "19285", "33071", "21859", "28732", "53533", "19363", "48826", "16634", "13410", "3112", "20918", "36585", "26275", "21996", "53290", "59329", "55972", "34923", "41735", "44159", "51738", "26900", "34526", "49576", "55744", "42843", "25366", "55634", "25521", "28114", "47990", "13286", "11028", "44315", "5955", "23553", "9749", "23144", "18266", "2232", "4166", "6307", "51667", "59264", "43647", "39610", "53857", "3945", "56502", "13278", "33649", "18128", "43268", "29426", "43825", "58852", "13486", "32324", "11876", "33969", "47373", "11324", "4455", "24459", "26960", "12511", "640", "13674", "7973", "58434", "43161", "22259", "53437", "23678", "29140", "59524", "42530", "33317", "28586", "23726", "27460", "9049", "2036", "31509", "10536", "41823", "54629", "33305", "24011", "48771", "57383", "39716", "6013", "22738", "42086", "45187", "23540", "11834", "29072", "45821", "53019", "48906", "57633", "33216", "27881", "14643", "13205", "37941", "6144", "10673", "3458", "23984", "33153", "44361", "33499", "47462", "26217", "24764", "36358", "20017", "34135", "29705", "28338", "6404", "56300", "10046", "15198", "23086", "34641", "24385", "58524", "10397", "2998", "29159", "23190", "6249", "19318", "6899", "59255", "34336", "29282", "28357", "12016", "36559", "39469", "22266", "11724", "34620", "27608", "54413", "4760", "26858", "45984", "50220", "11975", "21484", "28945", "16118", "25591", "3530", "22614", "18499", "44945", "31394", "44148", "56608", "32336", "19649", "18057", "40034", "6527", "56793", "45986", "45344", "2581", "5246", "53339", "36206", "41108", "9944", "52153", "50475", "36714", "37074", "6422", "9680", "9729", "3248", "56663", "15651", "21651", "48677", "13258", "48838", "51087", "9646", "57448", "47583", "59861", "58266", "29970", "43038", "11891", "36207", "39794", "17983", "13495", "13577", "24749", "40529", "32270", "44200", "12464", "19452", "51204", "54944", "4899", "30536", "38219", "13596", "11237", "20081", "50052", "27086", "17991", "38938", "1272", "41731", "52304", "48628", "16537", "48006", "13117", "41200", "37891", "30558", "58404", "21331", "14679", "36394", "34915", "31041", "17904", "58549", "23639", "34121", "58796", "24277", "26729", "53477", "30554", "47224", "19329", "18719", "53359", "57479", "44696", "3767", "14435", "54848", "51836", "57278", "14792", "59556", "49490", "25728", "38344", "40728", "47642", "373", "51569", "30043", "33030", "47727", "38965", "49277", "28400", "47274", "15744", "38624", "10611", "16814", "16377", "41508", "37137", "24457", "55602", "19708", "20227", "23526", "14386", "8515", "41266", "12921", "20726", "22105", "36334", "31817", "21921", "34682", "7719", "3655", "35280", "38424", "11499", "26338", "42971", "32509", "35074", "45256", "30711", "18098", "55944", "30502", "20422", "568", "46546", "35238", "44540", "5952", "38200", "50824", "57829", "2637", "42430", "9488", "58368", "38565", "35156", "11731", "30718", "12633", "52755", "46044", "10371", "16222", "53442", "56636", "53101", "24520", "40637", "11218", "31313", "2734", "53606", "49150", "14875", "28460", "34550", "44621", "21725", "26308", "53772", "28847", "12529", "43356", "49514", "25362", "58192", "45695", "50003", "41602", "51862", "24108", "8171", "24911", "40236", "44180", "30564", "32381", "38542", "31023", "6847", "27867", "44920", "57650", "54480", "11109", "44658", "34438", "31806", "14586", "49387", "22561", "20688", "50737", "51211", "40566", "2364", "47456", "35634", "941", "34086", "4218", "33191", "14541", "32627", "21968", "7639", "13754", "28377", "49013", "52862", "24751", "59638", "13001", "6192", "11279", "24145", "14624", "59501", "29990", "35822", "35471", "39498", "49909", "31722", "31650", "25992", "57689", "3942", "26258", "2434", "3200", "57757", "26259", "49114", "51584", "25506", "31531", "46175", "53908", "49189", "23560", "15203", "34483", "23735", "36513", "58793", "6904", "10249", "41580", "8365", "52874", "27721", "4709", "59798", "47945", "45131", "15356", "42916", "3009", "40976", "54444", "57992", "44011", "59515", "30449", "52188", "18108", "10171", "55345", "37035", "41489", "40931", "16327", "55747", "35685", "5210", "34445", "6639", "10847", "38045", "54815", "29761", "44013", "43243", "49540", "36008", "22050", "49655", "43515", "50473", "54926", "34689", "31811", "16967", "44982", "53759", "10678", "44563", "58016", "36860", "37295", "57695", "54123", "12414", "48829", "42084", "44828", "12361", "2022", "10716", "23660", "54217", "26384", "24332", "36861", "34295", "12493", "46781", "42314", "28880", "16783", "19847", "4247", "8576", "16166", "9065", "42325", "40203", "59996", "13976", "43895", "29942", "8291", "4977", "12317", "14114", "14495", "42899", "5441", "8395", "58406", "15149", "45193", "45586", "57750", "25756", "17026", "50424", "33346", "37644", "21997", "35611", "20035", "45337", "38189", "13623", "13805", "9819", "10063", "42381", "44475", "24847", "9662", "44827", "15167", "27528", "13741", "45335", "55811", "22966", "40659", "59854", "37290", "40172", "34503", "47765", "45462", "6529", "59052", "21071", "33117", "14184", "24470", "58847", "33692", "44186", "21734", "37537", "28881", "57132", "4825", "3762", "17867", "23963", "14247", "11793", "26952", "45544", "53273", "40402", "4002", "53889", "56158", "52575", "35099", "21417", "50633", "11502", "12457", "49254", "34916", "36694", "10683", "32114", "45527", "17373", "43745", "11490", "43841", "52987", "27876", "52695", "9593", "44262", "54254", "57243", "31252", "59040", "9627", "15959", "1064", "17583", "15168", "1540", "41372", "27813", "48540", "49142", "52423", "1431", "7607", "45263", "28885", "42312", "23006", "44092", "34885", "51979", "4921", "47904", "39443", "13279", "48831", "20585", "25826", "14513", "3275", "5463", "34733", "13988", "1532", "46423", "38119", "49485", "1665", "53909", "9008", "26411", "17863", "26879", "52466", "35042", "46770", "21721", "53514", "59794", "2436", "39986", "41753", "8133", "42271", "5371", "28560", "14093", "38506", "6229", "39468", "9923", "35497", "59481", "21319", "8251", "30598", "33357", "15614", "18339", "3353", "41509", "37933", "2141", "1670", "29603", "48112", "18247", "5422", "31622", "18881", "17230", "36829", "55626", "6673", "50470", "37230", "29157", "56011", "18533", "31604", "39825", "33394", "16353", "20214", "57200", "5678", "7163", "37604", "12637", "59740", "42871", "39382", "40065", "42753", "548", "43509", "53832", "19536", "2117", "52474", "5853", "37101", "23121", "26682", "32603", "50208", "21158", "52072", "12545", "15560", "10924", "32013", "28776", "3120", "12567", "13600", "26374", "49826", "18080", "11068", "7978", "58369", "24308", "10954", "50173", "48155", "49376", "36560", "59645", "23627", "1992", "43534", "25044", "22569", "55862", "32638", "21366", "49050", "43100", "28085", "41247", "37403", "38080", "54946", "32774", "30355", "32187", "56371", "37220", "45637", "33790", "40851", "44985", "51416", "309", "41959", "58420", "16505", "49991", "55340", "39967", "4204", "16553", "8688", "35493", "9632", "7389", "9085", "22453", "3647", "10151", "55884", "51150", "22536", "50139", "40687", "25314", "32487", "4738", "22616", "21564", "22373", "46136", "56570", "32421", "56455", "40943", "58412", "50704", "44559", "34238", "59634", "48304", "33201", "42982", "43476", "32572", "26636", "47044", "10930", "51875", "21637", "9386", "52216", "44953", "16931", "53977", "23574", "11652", "17714", "5109", "41573", "46337", "10983", "917", "25569", "22805", "53401", "52058", "35553", "36940", "7428", "37056", "14660", "28871", "27910", "39758", "46917", "27328", "36361", "39836", "51939", "39502", "47960", "29179", "42276", "25830", "42257", "7016", "18877", "35826", "22463", "21601", "5196", "49322", "47255", "14544", "6036", "46429", "51633", "2545", "38110", "20880", "12648", "43847", "53317", "658", "45046", "49000", "29343", "47186", "26313", "30599", "47376", "14087", "14301", "23937", "13567", "3443", "33846", "23844", "33406", "29759", "58902", "55587", "20782", "54104", "44232", "39262", "40141", "37943", "7133", "46835", "16887", "35714", "39879", "57296", "30104", "40599", "14699", "6481", "35525", "16125", "43968", "23855", "26403", "9090", "2335", "29408", "17410", "6396", "24137", "17916", "17813", "29034", "20892", "33474", "50262", "23986", "25205", "57644", "43713", "57048", "56421", "20925", "49724", "22402", "13302", "5747", "36503", "10012", "57570", "11860", "20392", "48884", "46714", "2896", "44660", "26684", "3215", "17030", "9523", "28636", "4898", "14190", "7000", "25845", "43322", "13068", "5006", "50506", "17010", "36867", "21658", "17279", "42921", "2676", "47341", "33871", "41940", "38273", "50866", "39015", "53452", "58436", "893", "23827", "27939", "31046", "56899", "6835", "39564", "41797", "18540", "38411", "18121", "44255", "10876", "18223", "3955", "38935", "6133", "30023", "13186", "36972", "38916", "11214", "57596", "1504", "4145", "49463", "11201", "679", "52928", "35173", "57520", "37037", "21266", "17797", "17459", "4319", "18165", "56727", "25023", "18269", "14118", "319", "22429", "33108", "52006", "33475", "17447", "16362", "44472", "656", "11074", "35779", "4966", "34405", "3356", "14074", "28277", "59480", "35733", "34308", "2267", "51959", "56149", "31695", "38383", "31825", "27798", "28275", "24856", "52391", "38251", "55462", "40517", "18647", "58891", "50496", "47314", "32778", "23640", "5682", "35694", "32688", "37298", "48066", "55017", "52915", "24209", "51270", "3076", "19085", "20122", "51137", "26127", "49564", "20980", "30965", "31721", "55752", "13106", "5191", "7798", "41804", "49894", "49972", "54070", "49692", "29940", "59450", "37715", "56060", "10121", "4678", "20462", "33408", "42566", "28321", "40916", "43400", "53065", "41225", "23985", "28370", "38848", "515", "4875", "28139", "10735", "21325", "10232", "26498", "28792", "41977", "19161", "57371", "52950", "1482", "28722", "32014", "26379", "26479", "2707", "43821", "38952", "2627", "47065", "24502", "27114", "59763", "40986", "47226", "23909", "8913", "28244", "9506", "59425", "11547", "33911", "45134", "38621", "13468", "40376", "54026", "34077", "47840", "59552", "22925", "49108", "39969", "52291", "42127", "7919", "4029", "36993", "37573", "44210", "49200", "23406", "4741", "59505", "20746", "58180", "42897", "36865", "43069", "25515", "24345", "30298", "18043", "12454", "3451", "2599", "49908", "14259", "41911", "53642", "17061", "37838", "45059", "59323", "51455", "33958", "53616", "18615", "56211", "53524", "50321", "13565", "13378", "36017", "31276", "57917", "18537", "57108", "14318", "32352", "31724", "18277", "58879", "6903", "52508", "30079", "39832", "10065", "9856", "49383", "57452", "52903", "29800", "9840", "13044", "50285", "47605", "8839", "44139", "18728", "47337", "20012", "55500", "31950", "44090", "7029", "27410", "21441", "8948", "231", "2929", "21178", "41560", "12216", "48886", "47787", "26106", "28116", "55310", "16689", "50241", "57790", "3823", "14464", "5389", "19138", "35112", "55321", "17355", "51952", "58355", "11800", "45240", "11224", "23188", "18906", "14946", "46262", "12308", "35001", "23257", "1846", "55466", "391", "15642", "41436", "38898", "19185", "12683", "43126", "29851", "24667", "33427", "36662", "46286", "20126", "56927", "1995", "33184", "37634", "56303", "41670", "17144", "40911", "59779", "54908", "28024", "333", "41675", "44196", "37530", "53379", "15151", "31299", "29941", "20695", "46125", "40990", "42149", "8950", "21955", "49970", "42457", "4437", "9578", "59906", "33933", "39929", "29253", "14929", "56373", "31166", "529", "2785", "3705", "52941", "46719", "27113", "8256", "24302", "32639", "17503", "12599", "45459", "51655", "9927", "30860", "46475", "59640", "35588", "58601", "8559", "30485", "37700", "835", "41002", "43683", "10331", "29577", "44107", "17675", "28862", "3643", "31651", "44517", "25474", "809", "9682", "47625", "19501", "47725", "12324", "26797", "55711", "15511", "44677", "14350", "19912", "15903", "44669", "4010", "20354", "4359", "26524", "42463", "23679", "30886", "30705", "49033", "47200", "57473", "54184", "557", "40316", "3840", "55568", "45869", "2869", "38143", "20927", "47285", "41446", "42322", "41169", "58391", "25416", "53701", "46172", "23407", "10009", "4011", "23682", "3156", "15491", "38211", "35744", "57380", "20437", "12122", "44708", "58079", "35388", "5318", "2555", "14749", "53800", "50279", "26503", "23011", "8743", "57911", "43918", "35857", "40261", "8722", "44759", "57814", "16754", "13489", "20619", "35895", "51264", "15708", "28079", "28970", "46871", "18754", "9864", "34567", "860", "50754", "8966", "10473", "49166", "2347", "55707", "6924", "40165", "42741", "39506", "55026", "58894", "4785", "46421", "32567", "35771", "41462", "1085", "11674", "12702", "48721", "48476", "25064", "54198", "33441", "48287", "20403", "59834", "44235", "36485", "29882", "19524", "11136", "3029", "59296", "3839", "54680", "37966", "47910", "19556", "56495", "55415", "9054", "27384", "37801", "6195", "24699", "25014", "45413", "42828", "12543", "28929", "34243", "29231", "29195", "37574", "54510", "53195", "36468", "23402", "29110", "44683", "33040", "50063", "45085", "46491", "30323", "46157", "26327", "47273", "817", "4965", "41779", "13800", "43465", "51131", "21635", "19733", "980", "22549", "59069", "10625", "37458", "35896", "26708", "16770", "8393", "25583", "54493", "58651", "28982", "57873", "22530", "17864", "25011", "48132", "17473", "7843", "26351", "55249", "39946", "31732", "56225", "33087", "6390", "26986", "49011", "25518", "37597", "34358", "7051", "55912", "59262", "2038", "38697", "23070", "38781", "9162", "17600", "5922", "48316", "26761", "29384", "22422", "6860", "5039", "15146", "29239", "19971", "34023", "36558", "29028", "53315", "15072", "18342", "21799", "31853", "58215", "19631", "53934", "25412", "39181", "14392", "27370", "15621", "23373", "21849", "38384", "31474", "6839", "34504", "58508", "29767", "11368", "41045", "54208", "42499", "3488", "1397", "27003", "59319", "30662", "22944", "49133", "30835", "41", "42564", "6972", "3962", "18064", "5783", "1045", "41016", "55439", "45867", "26930", "32275", "23160", "27734", "42122", "34486", "5792", "59199", "27705", "20349", "41314", "25530", "29044", "28394", "43377", "26915", "22509", "55205", "4745", "12932", "26668", "23738", "14042", "20268", "54596", "28061", "20647", "41405", "40919", "56869", "41043", "55148", "4591", "46049", "1270", "27442", "2115", "44515", "50653", "52211", "38258", "37844", "21295", "51566", "2764", "43469", "48522", "26995", "49802", "53536", "11779", "8705", "23753", "4427", "53485", "20969", "13712", "37448", "4657", "43274", "11900", "32796", "49337", "48239", "4971", "20350", "47160", "59158", "36853", "34642", "56313", "22953", "12284", "58238", "21747", "29596", "24523", "21542", "956", "27296", "40627", "53583", "44518", "8196", "13635", "6266", "2300", "28507", "20252", "53478", "44008", "24417", "52139", "13818", "37026", "7096", "1478", "36570", "12371", "37703", "41081", "59837", "10770", "926", "10837", "46005", "57266", "32359", "58620", "36951", "43539", "44225", "9533", "4973", "20732", "1217", "44597", "14835", "12408", "33671", "58182", "2130", "34114", "34611", "53142", "6785", "15267", "44477", "20669", "49056", "10572", "2465", "32106", "48021", "17735", "6024", "1840", "39154", "11507", "40762", "6547", "40112", "31135", "31991", "30794", "39791", "39754", "32500", "24467", "35212", "23856", "20659", "25546", "55806", "48739", "47864", "49593", "55013", "12369", "21780", "33554", "29306", "24324", "45312", "42148", "52111", "22573", "20916", "34205", "54508", "26366", "25337", "33830", "55146", "3547", "13216", "52432", "41329", "36424", "8894", "14372", "23936", "11748", "1787", "47496", "49914", "19377", "6824", "7422", "16598", "26535", "58241", "23183", "5208", "54134", "29564", "5023", "49845", "6995", "13895", "17531", "4167", "33616", "5", "58838", "8623", "39833", "18922", "16012", "27616", "48568", "49385", "30757", "42491", "42704", "28547", "34156", "6858", "50894", "16314", "11336", "22708", "14079", "34122", "51237", "31977", "21699", "8843", "9941", "17959", "42425", "10457", "51443", "55923", "1140", "57781", "19033", "36547", "51695", "18497", "41212", "19410", "33737", "12967", "41301", "5260", "35495", "48003", "53476", "21170", "42421", "30358", "241", "144", "22793", "17551", "31129", "8653", "51221", "37443", "16875", "42056", "42243", "19812", "26247", "54912", "10475", "52492", "24636", "9142", "19665", "41418", "21643", "26444", "36353", "1984", "2694", "43271", "22254", "35591", "955", "44301", "42608", "47659", "28064", "56856", "20424", "14721", "55579", "29097", "24369", "34894", "24803", "29964", "24314", "14408", "13449", "30456", "31126", "35776", "8253", "53409", "28838", "56676", "11383", "23346", "44046", "9470", "29498", "28432", "36497", "28345", "12714", "42296", "39924", "6607", "3261", "26541", "2122", "46165", "11098", "4693", "22307", "51459", "54416", "30646", "12939", "16224", "7896", "16952", "42041", "33656", "29289", "37251", "24205", "37923", "9942", "3291", "37216", "27390", "53222", "42622", "22253", "42168", "28193", "26527", "43754", "19495", "25157", "35510", "27219", "34863", "46858", "32168", "2031", "56160", "15997", "48241", "28222", "49545", "59294", "44579", "3117", "23915", "22696", "32319", "43306", "58281", "12012", "48819", "47846", "21235", "15233", "55538", "41430", "33769", "17968", "51024", "51054", "34208", "30265", "59835", "39533", "58743", "48016", "19807", "54959", "52560", "32709", "40318", "23411", "10895", "1759", "19671", "23155", "14668", "2099", "45726", "16043", "33397", "36728", "33706", "8487", "29118", "43965", "2105", "3553", "32151", "42441", "33489", "17906", "49715", "27301", "28771", "15290", "54671", "6516", "5572", "13081", "6658", "26561", "27657", "38694", "29554", "2625", "57944", "35535", "49071", "22333", "13892", "2879", "54545", "50722", "265", "59209", "43576", "16281", "2014", "41555", "56227", "31293", "5619", "3610", "40162", "37572", "8314", "20996", "33624", "51601", "30982", "40645", "1488", "25760", "27768", "14545", "50887", "51491", "6649", "9236", "601", "470", "37800", "31590", "26425", "9955", "24718", "23910", "28876", "14818", "25746", "15143", "51094", "54335", "51705", "14575", "55494", "26932", "47832", "30980", "55682", "31095", "2306", "41450", "42906", "25260", "31429", "39168", "53705", "4883", "54894", "4812", "30438", "7787", "17091", "31106", "12628", "24778", "47032", "58723", "56597", "38923", "1599", "19155", "25735", "35709", "51703", "17946", "1442", "27592", "13172", "42557", "32576", "20556", "38032", "14140", "39984", "41086", "4894", "58510", "44059", "43648", "19206", "6400", "12833", "9007", "7728", "38967", "28969", "5795", "15586", "38397", "52633", "59251", "14370", "40546", "50538", "17988", "16719", "20827", "30525", "34323", "37932", "51851", "5062", "58339", "31949", "9977", "5714", "49325", "50154", "24555", "43719", "27538", "34749", "1581", "47812", "37529", "25758", "11401", "29837", "58442", "7869", "36919", "49153", "34333", "16960", "34380", "10762", "7184", "45874", "43275", "3043", "2654", "8575", "40154", "33511", "15675", "54795", "40804", "7326", "14228", "36246", "52854", "9623", "47656", "12826", "36352", "18041", "44233", "4203", "30612", "35913", "50001", "34317", "8693", "10563", "48925", "6121", "40799", "21455", "793", "7045", "13675", "38236", "45917", "50237", "31517", "12938", "44670", "11541", "51822", "40232", "17882", "56066", "39189", "20509", "13333", "5262", "45829", "258", "59295", "26718", "26714", "2290", "42616", "34280", "49876", "20067", "45103", "22906", "59092", "29354", "38881", "52261", "37099", "22131", "42288", "59560", "38171", "3568", "29935", "15145", "8206", "53566", "44976", "21839", "27404", "52555", "53139", "36582", "53713", "2223", "24548", "37098", "28083", "58789", "15432", "50259", "56950", "26724", "8719", "34593", "42294", "31088", "37032", "43933", "18695", "23243", "26089", "57709", "48399", "47219", "43244", "47975", "41159", "29713", "23873", "46111", "16083", "7248", "37405", "48072", "11777", "24588", "47129", "27354", "21479", "37742", "10567", "13518", "35240", "44463", "33765", "50200", "51983", "21779", "40447", "9708", "50097", "55875", "22323", "33119", "1096", "31287", "55616", "32423", "36657", "55065", "4711", "10936", "58429", "969", "13255", "56614", "40675", "38581", "7250", "36143", "26478", "26649", "45693", "55396", "26452", "48634", "47229", "36796", "26278", "33909", "35105", "13056", "18154", "52065", "24755", "55347", "11949", "2152", "57767", "37142", "55655", "7180", "11989", "43263", "4702", "12974", "25423", "49783", "37288", "36153", "52193", "56159", "46583", "14702", "17158", "7280", "17802", "107", "26849", "9624", "38646", "17728", "56633", "24567", "17348", "14741", "13256", "11124", "52898", "21418", "1798", "44356", "11634", "47342", "26652", "23587", "55506", "43358", "44537", "35129", "43628", "54664", "4037", "36171", "9691", "12266", "3146", "11191", "56090", "18004", "12937", "48250", "17351", "55185", "24783", "30484", "50415", "42513", "13130", "27724", "11986", "36388", "47618", "24550", "18502", "16641", "22887", "20899", "40155", "59446", "37424", "53206", "16435", "50067", "54870", "28899", "39761", "22903", "4470", "25576", "48745", "33300", "59416", "31140", "1982", "56624", "2263", "17764", "31464", "25828", "24510", "38549", "27346", "13465", "49600", "32311", "3620", "36500", "45479", "9374", "8832", "16614", "54556", "9450", "56023", "47178", "29507", "50421", "16734", "47544", "10587", "58484", "21688", "27821", "47328", "13834", "49790", "28112", "28487", "14068", "15669", "51414", "54257", "20511", "22576", "32310", "17640", "34775", "19500", "35133", "17402", "31668", "39274", "43568", "51610", "17623", "37053", "1316", "58301", "11620", "22560", "43104", "27429", "54641", "32840", "37864", "14923", "33544", "36851", "1278", "32103", "52604", "10336", "36996", "10786", "28288", "1223", "10213", "46187", "26035", "9183", "10463", "43254", "26472", "56482", "56922", "30996", "25871", "17760", "12940", "38363", "49141", "22909", "2710", "13080", "24916", "17280", "56879", "3425", "51012", "24837", "36307", "30717", "12126", "3265", "50238", "5066", "5110", "21986", "38915", "22502", "32852", "11049", "10362", "33924", "16434", "58037", "50861", "52554", "50717", "24339", "14046", "3055", "7783", "31964", "28996", "49085", "53932", "6565", "47805", "38224", "2919", "34209", "53042", "43598", "50161", "10704", "54580", "41886", "3435", "20475", "40819", "34623", "14354", "37531", "3596", "51663", "58158", "48876", "40873", "21110", "31352", "51831", "59167", "39429", "2462", "24406", "47265", "7271", "31396", "16227", "34379", "34652", "5332", "49672", "40224", "50700", "29848", "38365", "48115", "31369", "24138", "44481", "33086", "49107", "44781", "16008", "36255", "34565", "12887", "10116", "7930", "12426", "12854", "21145", "59700", "33560", "36894", "8281", "13091", "3080", "58681", "22400", "42100", "51821", "57564", "48957", "52162", "30931", "7854", "18558", "17382", "25450", "54161", "32342", "47839", "47461", "25343", "59417", "50440", "55348", "47843", "31813", "4100", "2940", "33821", "48201", "50009", "44716", "44118", "25317", "16819", "10032", "22914", "39652", "4974", "50039", "15376", "4990", "59138", "2642", "51374", "50845", "28832", "13476", "43436", "45566", "29356", "48114", "27564", "58575", "35405", "12470", "16947", "24965", "3452", "51928", "47475", "28361", "58798", "55562", "39178", "17671", "53700", "25061", "48727", "19916", "26344", "42414", "51971", "24067", "16968", "38362", "39163", "39978", "51496", "52172", "9059", "37811", "52002", "6937", "53198", "12471", "7823", "55101", "30587", "50670", "58598", "52822", "40422", "38699", "49001", "50823", "19532", "36137", "48612", "26547", "29436", "27099", "38140", "16381", "8481", "30533", "38658", "50153", "22495", "16235", "51398", "53260", "44491", "8753", "20826", "20632", "6159", "42484", "59437", "52060", "3500", "40085", "57899", "44486", "9325", "59241", "44362", "17443", "31414", "12948", "57753", "9232", "20819", "2692", "7385", "52442", "39559", "36385", "14553", "32700", "59781", "24828", "35312", "41985", "34907", "15361", "30499", "19357", "32180", "59256", "7740", "29081", "1508", "49812", "49068", "36342", "59290", "14794", "15768", "19080", "37816", "2342", "46121", "41622", "31111", "11714", "51018", "48629", "11955", "35662", "18292", "42334", "4948", "19334", "33648", "35699", "4196", "38745", "20402", "22891", "56668", "8991", "56620", "28749", "7378", "5317", "21752", "3868", "40388", "6276", "47729", "29765", "1280", "30256", "45561", "36193", "32046", "53462", "3442", "5710", "51318", "54479", "37648", "55723", "23244", "47072", "35995", "58563", "23797", "35467", "1098", "21810", "36269", "2064", "16565", "2293", "947", "42454", "52613", "42781", "38420", "7277", "659", "9520", "46929", "24768", "38116", "8002", "49867", "11655", "44061", "28973", "29465", "51731", "3095", "11326", "6637", "54025", "33807", "24792", "32008", "39786", "32282", "23777", "31547", "55821", "5238", "12838", "4679", "426", "11460", "50437", "15878", "48121", "47414", "56916", "36825", "10867", "17123", "2194", "26958", "19551", "40106", "29124", "33536", "49186", "52389", "41834", "43442", "3371", "24663", "53734", "38537", "28511", "55386", "53375", "43206", "34427", "50355", "35398", "525", "36531", "47106", "11637", "29795", "52801", "25782", "22066", "49902", "36546", "14576", "32770", "32053", "16356", "48655", "11584", "25418", "26961", "18554", "58822", "33714", "48495", "1903", "38432", "17485", "42867", "39183", "36673", "27504", "52956", "32533", "55567", "39070", "23469", "30750", "4371", "12379", "55768", "28166", "40346", "32071", "20279", "28334", "47118", "35571", "49324", "58046", "19235", "41919", "8193", "23205", "31441", "19032", "22183", "59692", "27258", "1239", "19741", "11261", "19636", "27309", "3853", "43443", "9881", "42101", "24614", "33791", "30610", "35950", "26717", "55358", "40466", "26309", "15941", "51708", "43070", "23009", "57219", "11631", "46245", "59799", "42055", "58221", "47842", "30085", "9847", "48044", "10717", "10429", "8127", "52756", "59646", "5118", "28579", "34272", "33998", "76", "24833", "4012", "52455", "29098", "55093", "27807", "53119", "42856", "41845", "44352", "56696", "19704", "5809", "17574", "15519", "38051", "12384", "57059", "29194", "38062", "29107", "45968", "18169", "54525", "36791", "59120", "23477", "44815", "56860", "986", "31961", "33530", "24243", "43023", "22228", "4707", "27920", "58769", "8300", "3372", "57868", "52830", "34529", "43301", "17940", "44209", "8385", "51718", "55596", "21561", "5106", "53219", "47210", "5160", "53268", "16677", "7848", "51338", "3266", "42178", "55515", "48176", "52400", "16174", "19415", "36191", "16824", "58693", "46662", "29678", "311", "31408", "9390", "42638", "8126", "9333", "8928", "13218", "26218", "33090", "12404", "47898", "10497", "29057", "19650", "2129", "43470", "39016", "58102", "37738", "18037", "40689", "11516", "26228", "57928", "2481", "52821", "21972", "8042", "2958", "5823", "44593", "26049", "56685", "2793", "35324", "45742", "44776", "6701", "35062", "7926", "57091", "15353", "15373", "31403", "25371", "14241", "38087", "814", "31706", "14035", "3258", "20296", "16850", "38540", "45802", "43270", "17194", "59774", "55304", "3036", "38223", "9327", "6114", "3293", "1752", "40152", "55448", "40500", "39215", "58031", "11708", "25199", "52602", "49395", "2476", "17290", "26212", "33872", "51754", "17850", "19567", "22714", "41657", "31480", "5124", "17272", "32805", "4284", "35760", "21829", "36303", "22586", "59271", "8321", "16376", "4798", "21038", "3288", "9119", "4673", "28685", "647", "24433", "39225", "28608", "47930", "27313", "51672", "58727", "17820", "59969", "30640", "7467", "13891", "27956", "40311", "53039", "11442", "33485", "30167", "58073", "212", "15363", "36480", "53440", "37786", "2651", "24617", "501", "7814", "16020", "808", "37834", "30143", "59980", "43989", "30568", "51248", "46415", "17223", "53980", "35193", "58314", "485", "15195", "17604", "29701", "24041", "24446", "28795", "57321", "17215", "40502", "37470", "10112", "7589", "37987", "12436", "50101", "38919", "7784", "45245", "58183", "26611", "25390", "18383", "10195", "3852", "44721", "12390", "41547", "27880", "358", "49271", "16455", "33483", "50839", "13485", "44384", "29967", "7513", "34411", "23014", "28138", "18074", "19432", "20772", "12906", "31022", "113", "14424", "29382", "58029", "27397", "56024", "18842", "34293", "38659", "14023", "12476", "52167", "54468", "12614", "57509", "27444", "7912", "46271", "54317", "13616", "29173", "19581", "52150", "26722", "26476", "47574", "43990", "57973", "9433", "46881", "54602", "58095", "42007", "22985", "38522", "15803", "14197", "51803", "58191", "55166", "35794", "42021", "5415", "39384", "38928", "59030", "37741", "55089", "26968", "6282", "20997", "10207", "25126", "24790", "29380", "5032", "23608", "48113", "17999", "38950", "27712", "32130", "53389", "16545", "40056", "27338", "56255", "39934", "37161", "12794", "2673", "17376", "1650", "39948", "45391", "18753", "55237", "54764", "39096", "25036", "20535", "55793", "45440", "42088", "11760", "21297", "43200", "37690", "43314", "23649", "15196", "31358", "22518", "22789", "47762", "19893", "52606", "11524", "40995", "12009", "49729", "18319", "25085", "56847", "4826", "43681", "23375", "53011", "4767", "26967", "3498", "36257", "20866", "40343", "2941", "20342", "35402", "46915", "42887", "37829", "10158", "19664", "51907", "2754", "27205", "7260", "20216", "49700", "55906", "23763", "10888", "38443", "26760", "41216", "21226", "749", "54111", "24363", "39332", "26680", "28944", "51296", "20397", "4686", "39679", "27571", "4646", "44877", "13560", "45415", "16429", "10078", "14756", "54080", "35765", "38435", "27501", "16236", "18566", "51164", "43399", "4918", "25218", "22810", "42723", "58721", "56340", "8011", "6544", "12685", "3227", "48689", "39916", "36963", "2925", "14503", "25476", "17759", "40601", "11244", "16251", "38525", "45529", "59519", "53104", "36675", "35213", "3329", "34676", "18167", "26800", "16241", "56305", "21401", "39170", "45478", "51503", "42040", "18544", "33838", "14300", "35383", "48956", "35031", "45641", "9", "5308", "51898", "36475", "51088", "19466", "21556", "26832", "54551", "41539", "32083", "19238", "28650", "50678", "16699", "10805", "33501", "4910", "18604", "13462", "19061", "24322", "27091", "343", "52737", "48687", "9191", "45870", "54028", "7030", "59796", "18629", "55973", "35787", "55766", "48350", "48656", "10557", "36842", "50511", "49828", "9060", "28476", "22599", "42078", "18638", "46087", "38290", "44846", "44971", "50840", "57860", "45276", "31538", "29839", "46376", "5052", "11440", "11298", "25908", "46724", "33032", "49807", "5923", "10349", "27809", "850", "32380", "39793", "19207", "17305", "3618", "21269", "10821", "43137", "59491", "37132", "23949", "46840", "22493", "53614", "50615", "44421", "2897", "45331", "27215", "27904", "21862", "56", "47527", "30790", "33728", "20605", "40497", "27803", "10184", "2924", "5014", "6147", "33512", "50430", "14582", "15666", "22482", "21314", "35715", "28620", "38861", "42387", "31863", "30689", "30522", "11962", "13108", "53553", "36114", "49523", "9496", "21583", "39180", "11690", "45012", "17092", "42802", "46254", "4601", "22615", "44089", "15225", "32946", "22216", "15025", "10038", "44", "19121", "7218", "22414", "41739", "21714", "15552", "28714", "37036", "31326", "27762", "2212", "46054", "35063", "56652", "18876", "57778", "29619", "35434", "34418", "5166", "6685", "42206", "5286", "45955", "18649", "18303", "13261", "996", "51881", "46323", "14138", "26707", "33661", "27679", "34286", "3968", "11913", "21309", "54424", "14731", "29831", "21047", "12977", "3518", "7906", "22867", "38390", "94", "51497", "40079", "504", "28002", "25191", "36597", "3041", "19635", "28221", "935", "40238", "11529", "13757", "205", "26757", "18248", "2352", "2104", "31187", "24278", "5372", "48638", "5333", "39109", "31188", "33490", "9136", "7432", "37128", "46856", "9320", "51504", "15187", "27685", "33812", "31881", "19078", "43785", "24641", "26043", "29187", "54597", "5409", "27898", "1906", "26369", "17527", "40863", "20426", "23045", "41149", "30928", "19837", "58854", "13144", "21236", "40410", "53736", "20427", "30566", "1087", "34059", "40039", "6297", "49147", "48972", "38766", "28537", "32850", "21806", "26378", "58346", "19559", "38410", "16221", "25743", "59752", "12241", "48659", "59886", "41062", "19200", "19066", "21999", "34989", "3798", "33410", "2220", "32247", "33137", "27595", "21717", "50189", "59619", "13520", "35380", "55692", "9568", "34269", "40469", "16749", "14422", "36983", "23393", "31264", "30020", "31189", "44950", "9807", "31995", "36227", "27073", "49691", "11090", "49364", "38082", "54745", "18735", "25500", "15779", "17776", "18931", "30946", "51336", "33281", "7788", "53220", "37416", "23992", "21119", "28906", "30668", "7311", "9154", "59252", "58953", "47673", "32982", "28414", "17320", "58676", "16903", "6085", "17286", "40983", "4537", "46353", "39003", "20845", "26054", "25785", "12566", "18306", "10248", "44454", "22727", "52003", "1730", "7614", "40030", "58795", "18462", "17818", "40701", "38871", "30223", "13964", "38886", "8245", "1966", "29225", "28192", "8132", "23437", "19616", "49351", "17114", "55438", "1393", "15210", "54369", "34062", "44545", "44612", "26311", "52823", "35665", "40221", "54819", "15199", "48448", "2057", "35601", "14737", "35567", "38375", "53849", "49352", "40481", "46953", "2322", "42183", "5134", "8870", "28379", "54995", "26277", "1215", "43482", "30062", "1070", "41273", "26886", "1675", "59527", "57713", "30296", "31163", "3419", "12741", "9961", "32452", "7118", "36789", "39174", "43976", "58296", "8915", "52385", "651", "57014", "14886", "53227", "277", "36820", "6729", "34414", "28205", "21397", "43885", "9495", "49794", "21543", "39361", "6895", "39320", "7800", "15792", "20692", "24737", "23279", "54604", "26474", "38539", "15542", "18078", "38426", "40390", "57039", "6365", "31893", "27167", "54816", "13229", "17812", "41618", "27617", "30151", "21771", "46316", "44620", "42658", "30754", "49360", "55700", "45353", "32566", "40009", "4185", "30285", "26861", "6941", "31292", "51545", "44087", "19333", "41810", "52866", "11163", "53641", "30011", "37291", "858", "8090", "16766", "42804", "39242", "56709", "379", "57384", "24316", "29327", "14339", "56004", "20021", "31954", "47535", "43536", "54717", "30506", "13968", "11132", "45881", "32455", "34103", "38407", "46805", "55650", "39572", "13555", "55987", "22043", "32749", "44664", "17577", "20432", "2475", "7536", "13771", "23362", "37642", "56974", "19614", "8113", "38583", "9265", "51589", "10067", "47051", "32265", "8586", "37474", "35341", "5980", "37130", "29683", "33088", "5094", "13865", "35606", "51245", "22630", "5126", "40432", "47545", "43547", "54651", "15925", "59538", "13545", "53066", "5353", "3801", "912", "39797", "20003", "51868", "46540", "36660", "32221", "34126", "52990", "5497", "41651", "946", "32681", "283", "15816", "49614", "53080", "10202", "16335", "13776", "54732", "44229", "25958", "55625", "18226", "34313", "12074", "25674", "24392", "4255", "14917", "4478", "42962", "31918", "22825", "52293", "1420", "50946", "39139", "43906", "7088", "36273", "36031", "58501", "50785", "3539", "19521", "4627", "10022", "54252", "30125", "58698", "20218", "50637", "37003", "46952", "2701", "22138", "46140", "22643", "19196", "14877", "52103", "27299", "48839", "43245", "56335", "14734", "4628", "20706", "5444", "30480", "38999", "27980", "46983", "28728", "56205", "24912", "32543", "55351", "1737", "50799", "2979", "31820", "14462", "6944", "16791", "17374", "51967", "19593", "16750", "44165", "50662", "855", "33084", "20492", "24122", "46674", "10069", "245", "33182", "23310", "39207", "16695", "22439", "10307", "25713", "59855", "29303", "23666", "46916", "3142", "37895", "11967", "20450", "9874", "27332", "26284", "29855", "10962", "14743", "45793", "23671", "38802", "4832", "50647", "39251", "1616", "31514", "54365", "4047", "9332", "25026", "46796", "29576", "22242", "11993", "5258", "57031", "33564", "35858", "49039", "17680", "8962", "30413", "783", "56919", "36561", "773", "17725", "48426", "31680", "7652", "33154", "45723", "46984", "35420", "53562", "43892", "20526", "20053", "20846", "1571", "52524", "27635", "29437", "43193", "59009", "37367", "51066", "46699", "37704", "52152", "35331", "17358", "44568", "46610", "10189", "4862", "2845", "54417", "34289", "6327", "52278", "24141", "36376", "51165", "57354", "53991", "7785", "38376", "50606", "22109", "18239", "53551", "646", "11111", "48105", "10447", "31741", "10431", "15009", "52187", "2425", "11932", "24573", "6977", "59858", "185", "55789", "27536", "52367", "6254", "20224", "58896", "33527", "48600", "27814", "32781", "32392", "1586", "52055", "15099", "28830", "10426", "5752", "27343", "23387", "39714", "16981", "38335", "48322", "36553", "5470", "21920", "45213", "47345", "6831", "46736", "19904", "31783", "45575", "36651", "8099", "648", "35443", "22654", "20856", "36054", "54147", "9564", "39539", "46948", "6837", "1783", "48730", "15480", "15337", "58830", "1555", "36934", "21910", "13572", "41132", "16368", "26510", "6281", "2312", "4849", "12140", "56418", "22272", "12264", "55222", "27369", "24466", "35019", "5862", "49437", "14645", "17243", "49567", "945", "56120", "38528", "3159", "2297", "33858", "43887", "44247", "49913", "4732", "6910", "43187", "53022", "35532", "15912", "55230", "27316", "6926", "32287", "59887", "48327", "4764", "779", "47924", "2942", "1596", "16069", "47893", "25881", "7580", "39649", "36846", "28129", "2424", "22683", "21649", "55024", "12208", "58279", "27681", "24415", "51722", "13702", "41658", "6434", "19482", "15907", "43490", "35612", "7516", "5274", "13236", "52925", "46694", "45068", "1525", "44113", "150", "54917", "48550", "42942", "34952", "56717", "4021", "30037", "56951", "36739", "9869", "41479", "50865", "22285", "19079", "40093", "28372", "20823", "23739", "20411", "59101", "39565", "27030", "114", "4748", "12512", "59893", "51101", "13524", "8745", "2706", "30107", "8269", "55593", "41289", "28789", "21600", "21090", "16509", "11868", "43479", "9724", "3209", "7377", "53122", "7130", "6196", "6118", "58177", "41725", "19254", "12063", "57006", "42612", "8174", "34436", "34999", "25034", "13859", "53545", "4495", "35064", "3201", "43410", "25692", "27251", "41117", "1459", "10379", "6885", "36001", "38834", "45657", "31177", "8078", "23507", "2160", "45249", "37019", "28450", "8612", "47632", "18891", "20514", "59819", "46418", "28956", "13345", "52953", "24383", "13516", "29745", "34866", "57282", "37560", "45320", "11159", "16631", "29871", "40748", "5836", "26813", "55717", "47241", "52131", "34595", "45904", "33667", "22774", "59812", "29870", "37501", "15469", "13320", "25553", "40991", "14654", "51413", "34256", "49924", "12773", "24030", "21537", "26193", "46687", "18945", "13220", "16288", "33966", "44649", "25993", "59945", "22697", "57499", "26677", "35854", "53188", "24077", "7878", "39031", "18567", "31975", "27573", "8147", "48151", "44699", "44376", "13071", "30702", "51527", "5696", "5396", "14144", "10698", "59596", "41253", "47931", "29420", "32575", "44698", "53557", "36037", "30627", "20456", "16730", "5965", "5621", "48718", "23181", "14461", "21485", "27969", "43960", "11119", "30473", "30857", "17658", "47884", "37249", "41711", "59555", "31557", "41993", "36616", "47447", "24045", "49246", "21833", "45570", "43196", "7586", "16078", "20713", "31257", "12044", "21148", "3927", "31116", "26215", "55710", "47721", "20176", "35929", "49079", "15940", "6371", "51375", "47645", "11474", "25263", "21169", "34360", "18939", "58076", "56713", "43417", "30294", "5382", "37637", "39827", "57738", "46630", "52331", "51537", "28495", "12647", "11533", "16874", "22597", "17836", "37878", "23225", "48452", "39043", "12705", "54810", "28291", "13542", "9382", "48528", "22558", "51651", "14771", "25700", "33534", "18745", "26432", "55476", "8860", "45359", "9590", "39727", "9112", "50115", "40026", "45884", "58841", "28859", "40319", "6517", "30292", "2992", "47468", "47490", "25047", "5904", "13553", "34680", "31389", "8525", "46940", "28010", "24495", "50507", "2453", "9385", "30500", "33112", "33538", "49261", "49813", "13529", "43582", "19720", "13049", "28684", "42470", "46196", "1550", "35222", "35651", "22020", "44277", "10037", "47678", "24990", "47675", "46672", "50016", "11276", "46647", "50798", "57195", "41910", "24052", "35189", "16648", "13471", "55069", "47307", "30475", "10771", "7469", "6420", "8805", "23632", "28183", "49326", "24630", "13697", "40586", "12480", "56184", "40954", "46341", "7023", "43234", "51358", "34705", "3780", "14900", "48232", "30501", "35096", "41835", "23027", "10897", "36238", "33042", "31221", "13673", "40287", "26492", "4970", "12737", "28210", "698", "34510", "10577", "6543", "32344", "46021", "34026", "54430", "56436", "8608", "13148", "49587", "50945", "10841", "4938", "2135", "51556", "55623", "34368", "33875", "38217", "35433", "42374", "48147", "36366", "25285", "35652", "579", "161", "30364", "5221", "35892", "4001", "45956", "34213", "17847", "11357", "21445", "42309", "41676", "23774", "25632", "42951", "56837", "55369", "23132", "37445", "33468", "7200", "5691", "50361", "3194", "14021", "3896", "41682", "24938", "25141", "53093", "29879", "9077", "33566", "49401", "19465", "56765", "16102", "41259", "20331", "26770", "35307", "21577", "20088", "57254", "58435", "42256", "55674", "36145", "41298", "59709", "37258", "33834", "12409", "25631", "57607", "13972", "5273", "11830", "42186", "53446", "9614", "39378", "54921", "17068", "15620", "23568", "27892", "3351", "40138", "10385", "46133", "19973", "38476", "59462", "41681", "786", "9924", "56242", "30343", "35229", "46844", "19817", "2095", "36680", "17918", "30778", "12263", "22485", "29606", "58977", "25749", "40384", "7413", "10053", "1079", "45885", "13730", "31202", "7873", "32445", "56088", "1076", "55714", "9169", "50329", "8021", "36354", "15176", "1321", "14812", "36589", "45866", "44327", "3086", "17939", "39845", "42847", "28680", "19113", "1692", "6448", "48738", "2527", "59732", "41127", "58655", "33314", "38040", "24900", "7244", "14595", "19010", "57739", "26604", "44442", "53143", "9922", "7157", "48763", "58752", "55147", "46851", "39645", "10929", "17687", "19988", "9337", "53162", "59001", "38734", "12276", "12250", "7910", "30388", "4110", "6125", "55609", "3716", "51639", "42792", "23206", "6089", "31423", "13283", "26842", "28837", "19702", "26287", "38970", "56076", "38131", "22106", "32870", "49935", "4393", "35234", "12002", "468", "58918", "3646", "44635", "14352", "53940", "1914", "23236", "32801", "36755", "29818", "55960", "51793", "7835", "25793", "43068", "59091", "25504", "14988", "30716", "47191", "28531", "34192", "56580", "36911", "11405", "57425", "47895", "53852", "641", "19678", "1659", "30415", "23534", "28286", "17221", "13012", "46564", "9757", "5135", "7663", "58864", "34522", "55442", "36618", "51222", "4354", "12087", "54016", "29158", "18613", "34511", "42348", "27494", "10333", "26071", "38900", "55088", "52802", "50414", "3054", "11744", "28126", "14234", "54485", "35935", "46739", "42794", "31507", "19192", "51045", "1128", "44902", "53767", "37012", "37210", "2415", "51841", "38729", "33252", "24957", "9157", "25271", "4360", "26184", "50963", "1590", "19937", "21381", "1773", "3794", "16147", "33238", "19932", "15973", "14569", "52355", "7439", "32874", "51630", "2083", "11629", "19820", "57164", "36854", "2458", "30810", "58726", "23877", "46833", "50000", "4198", "15241", "25677", "38740", "12220", "45797", "4516", "19481", "38339", "38914", "2902", "32250", "4329", "54533", "47201", "6744", "31435", "10784", "24981", "32568", "50104", "13742", "13149", "58584", "59948", "36106", "30488", "24346", "30235", "26534", "15914", "43870", "24533", "55154", "21131", "2023", "57245", "9122", "9170", "35805", "28433", "44802", "36040", "48950", "6539", "27647", "14120", "12800", "36579", "57684", "9133", "33725", "2714", "34617", "44258", "6190", "52406", "15649", "7316", "33645", "22504", "43361", "51737", "20108", "48008", "37554", "6154", "43610", "20142", "39861", "19945", "29041", "57884", "2508", "34605", "24263", "24429", "50765", "57656", "49047", "50767", "14824", "8917", "9058", "47073", "12468", "1327", "59372", "4705", "8442", "32732", "30394", "34822", "35499", "53706", "28869", "47000", "24929", "12615", "37940", "44116", "34990", "24013", "53622", "34067", "5973", "37977", "52445", "3756", "6383", "23100", "39800", "20172", "37917", "47868", "46007", "26777", "11946", "510", "33174", "2582", "37887", "36910", "4380", "36891", "25732", "44082", "54792", "9429", "16232", "37076", "20078", "30049", "38016", "57823", "19277", "9725", "10703", "15130", "42833", "51777", "19601", "6657", "27547", "511", "15917", "39553", "13113", "57318", "33677", "14246", "14903", "11477", "1100", "27953", "15038", "37927", "57212", "56819", "34314", "15307", "14007", "18032", "2417", "41646", "20272", "14054", "52847", "16482", "18270", "37365", "24640", "31814", "56178", "27434", "54939", "33984", "34383", "19042", "2841", "40592", "38309", "50955", "9279", "22658", "53655", "56870", "29669", "8330", "20431", "9430", "1337", "51909", "7920", "3096", "50195", "25006", "33772", "9046", "6385", "36368", "4891", "12223", "57936", "43757", "17354", "1938", "25491", "39045", "46344", "240", "28509", "19834", "26734", "46377", "14033", "18702", "520", "36915", "39505", "43051", "47165", "27096", "48106", "50711", "55411", "53292", "21205", "24964", "38794", "22734", "43168", "40057", "55245", "42451", "32101", "14915", "21062", "59814", "52201", "35044", "40559", "48920", "11140", "27267", "32649", "17582", "12328", "28562", "51023", "33243", "41942", "19194", "34734", "1820", "40377", "18761", "46717", "829", "35943", "45894", "22176", "23371", "25367", "34820", "42642", "12261", "35706", "45163", "11761", "2467", "4969", "43155", "3509", "52877", "51858", "15299", "20454", "51512", "9550", "38788", "56151", "35464", "11095", "19643", "56059", "23823", "46774", "31880", "59947", "58910", "51215", "45848", "59804", "52573", "57623", "50240", "14010", "31475", "758", "22764", "35408", "57663", "30322", "7288", "31225", "45272", "59974", "27706", "20982", "5129", "17642", "56253", "15242", "39961", "141", "49432", "22247", "42550", "17012", "16188", "3903", "37898", "55334", "50098", "37813", "34694", "30604", "29833", "14839", "3928", "57109", "41257", "50881", "42621", "31204", "15001", "10764", "53070", "44761", "39762", "7888", "45201", "32153", "59955", "14376", "25894", "15855", "29987", "48301", "3718", "45209", "35318", "45301", "30210", "4934", "14083", "17829", "29763", "13961", "19188", "21383", "4750", "7569", "17404", "32875", "5781", "57832", "56604", "50530", "6630", "57070", "1497", "34631", "18560", "37181", "23332", "21030", "19305", "19620", "18220", "27974", "39959", "23994", "52214", "39252", "52852", "30933", "27888", "1352", "30628", "40353", "37985", "8134", "34217", "37231", "48812", "17023", "44483", "29442", "21126", "16935", "4234", "30641", "6705", "1832", "18734", "22534", "56799", "59444", "38556", "18889", "49207", "22391", "25170", "49869", "51664", "11308", "16413", "11198", "48425", "52694", "43393", "47854", "15859", "58528", "11402", "27542", "11613", "40252", "25217", "22476", "16953", "17345", "16623", "7142", "27374", "29891", "6969", "28396", "32897", "20028", "15273", "25734", "53600", "42427", "13061", "2257", "27056", "20257", "18506", "36474", "54544", "11121", "32618", "26441", "54649", "2789", "16212", "26388", "50952", "22874", "8056", "46785", "38161", "21192", "46999", "38213", "40453", "4793", "12242", "56244", "36214", "36409", "11769", "12851", "3889", "20297", "50624", "42700", "30270", "36327", "58765", "36935", "45285", "46876", "2495", "46801", "46352", "43255", "23583", "31558", "18357", "11850", "4756", "36435", "5186", "16157", "24224", "11651", "44182", "49695", "50466", "334", "14718", "11196", "11905", "2844", "56910", "56761", "38792", "5554", "41849", "17442", "22791", "37455", "1325", "44261", "31301", "27575", "14956", "44399", "37538", "49575", "52008", "43917", "42501", "6563", "37955", "47188", "39949", "38926", "12032", "57963", "48695", "1030", "25979", "9315", "23174", "12556", "7918", "56889", "59784", "54148", "16386", "22413", "13221", "8125", "19565", "51176", "45493", "3326", "27715", "2755", "56068", "13051", "25646", "34334", "11156", "13896", "41468", "39374", "18081", "51734", "654", "26461", "17805", "48699", "7072", "7549", "5270", "4831", "42281", "42649", "19047", "26817", "22362", "39658", "13469", "17585", "48312", "31244", "55948", "50330", "22933", "20338", "7125", "25729", "42552", "27163", "44073", "19420", "5560", "2101", "39698", "10843", "28718", "24697", "58197", "43084", "3118", "20522", "50054", "47941", "32917", "11841", "7933", "19639", "30109", "52388", "31723", "5978", "2363", "47619", "20070", "38164", "58078", "23340", "54040", "13645", "22239", "29558", "18949", "38065", "7210", "41565", "11485", "26829", "16374", "34468", "8421", "42051", "10631", "12874", "15023", "26134", "42944", "39160", "33257", "37566", "43116", "10476", "57840", "3125", "5280", "3657", "4991", "30370", "15470", "52770", "8038", "5827", "32961", "14414", "34766", "12064", "36890", "2886", "55176", "5475", "58017", "33938", "18147", "12764", "40649", "45088", "56081", "56581", "23919", "59113", "52697", "25391", "6242", "25451", "53571", "28892", "22702", "32366", "7085", "5338", "15421", "55460", "21040", "20315", "25665", "52129", "22457", "42063", "21746", "12551", "13387", "20553", "19647", "22211", "39465", "18281", "10113", "51467", "29859", "53092", "45340", "5546", "57735", "57226", "28640", "24845", "25", "45332", "43684", "9769", "7786", "18048", "30352", "34391", "44506", "31653", "6614", "33163", "56434", "13270", "45284", "44930", "9481", "14185", "7354", "8336", "55072", "31584", "30865", "18445", "44440", "24364", "10148", "15549", "40672", "27447", "9465", "47976", "8249", "26946", "10276", "25200", "23214", "1549", "12373", "30073", "49582", "7625", "32038", "44514", "59503", "57691", "56457", "33003", "17242", "34419", "31510", "41762", "51788", "43867", "49829", "37408", "37896", "16406", "55422", "41833", "24903", "52622", "39292", "24100", "29620", "48692", "27694", "3132", "6832", "34687", "42376", "5609", "16642", "8567", "37510", "47789", "9514", "32279", "51505", "7069", "53422", "48358", "30250", "14867", "57719", "40661", "45006", "45649", "34640", "38458", "26198", "47560", "35660", "41317", "10603", "4845", "48148", "42275", "1888", "23267", "29359", "40909", "20683", "31519", "2973", "121", "42496", "32525", "7457", "47782", "19187", "34509", "3938", "3943", "45366", "47720", "6484", "44595", "58152", "14016", "31891", "4248", "59859", "53803", "58663", "56453", "21429", "7584", "3228", "42372", "34777", "36065", "8916", "13827", "46321", "51582", "6044", "29249", "48407", "47105", "40444", "4190", "45958", "12699", "20610", "43862", "40192", "33577", "52661", "42349", "8819", "9491", "37565"]]} \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist-test-split.txt b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist-test-split.txt new file mode 100644 index 0000000..548e17e --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist-test-split.txt @@ -0,0 +1 @@ +{"xvalid": ["int", ["9927", "7226", "7711", "7504", "6015", "2924", "5588", "9080", "6835", "548", "41", "9349", "2134", "1315", "1953", "8430", "953", "5563", "8696", "6879", "9010", "2782", "6518", "635", "8580", "9817", "426", "9464", "1157", "3532", "6066", "992", "1426", "3371", "2983", "9727", "1490", "2010", "6199", "9563", "6650", "520", "8246", "5174", "3344", "8539", "970", "3370", "545", "4062", "9741", "7653", "433", "2949", "3177", "4083", "5556", "9023", "2054", "7721", "7119", "7608", "7098", "6761", "2188", "1358", "4420", "6317", "1413", "4702", "7399", "3084", "2856", "7931", "8789", "1999", "4289", "2009", "7791", "4437", "1191", "7544", "6788", "5706", "6814", "2557", "9432", "9509", "6830", "6176", "7836", "8787", "157", "3881", "5022", "5857", "5462", "9812", "6467", "2569", "5746", "2376", "6551", "3009", "209", "2437", "7890", "148", "3321", "4701", "3555", "2212", "1554", "3143", "3223", "155", "4848", "7845", "2071", "8945", "7740", "8069", "9425", "3041", "3266", "2661", "5376", "3104", "7457", "4528", "5110", "6870", "9898", "3521", "5846", "2220", "5085", "8232", "5525", "9859", "4309", "3579", "8281", "3526", "3199", "5675", "7333", "3191", "9700", "3896", "4564", "6533", "4462", "4846", "7884", "3500", "3294", "9180", "7825", "9461", "156", "6065", "8200", "9532", "7222", "4078", "1208", "2191", "5067", "4056", "8169", "332", "6614", "5815", "8487", "8483", "3608", "5480", "740", "6815", "2099", "745", "489", "6138", "6827", "1704", "1596", "7509", "3938", "9539", "3386", "3567", "8175", "9137", "8163", "9490", "8999", "3150", "1087", "3355", "5354", "1738", "785", "7246", "5414", "3776", "8237", "1683", "7956", "4439", "614", "3835", "3156", "3721", "667", "8797", "7675", "1128", "6203", "2653", "8009", "8274", "4512", "8523", "9925", "8090", "5107", "9462", "4535", "2243", "3292", "8809", "3771", "7586", "1135", "7948", "3954", "5801", "8638", "9717", "1346", "7525", "702", "3250", "9558", "2532", "8844", "972", "9139", "6012", "5729", "8283", "7448", "7773", "9554", "4816", "5551", "2852", "993", "7599", "1480", "3667", "1581", "8260", "9419", "2961", "4854", "3048", "9298", "4291", "7701", "3987", "9627", "487", "1621", "179", "595", "8938", "7412", "990", "1296", "229", "9525", "4540", "4130", "83", "113", "1796", "5736", "4328", "9493", "8161", "5", "2977", "9096", "6576", "4249", "5647", "2593", "6487", "6027", "6226", "2187", "1449", "6101", "123", "3269", "4847", "7708", "134", "9619", "8769", "9099", "8435", "6850", "9428", "7765", "7651", "6940", "4678", "9353", "1824", "8746", "7938", "9348", "5189", "7941", "9151", "5302", "4578", "7407", "7572", "2475", "7735", "461", "7492", "300", "7989", "2202", "6667", "4007", "9517", "8499", "7562", "5942", "5874", "9453", "9389", "8923", "6420", "4352", "4442", "3242", "9971", "7332", "2289", "6334", "6007", "9826", "5446", "2273", "89", "2883", "2316", "7288", "1775", "3458", "7219", "3718", "6961", "9198", "1685", "5805", "6753", "5844", "6781", "9444", "3174", "7234", "3057", "7555", "4667", "7617", "10", "8327", "1520", "2635", "1352", "2216", "2975", "410", "2183", "4253", "2686", "6574", "6676", "7058", "6654", "2693", "8121", "9057", "1696", "2474", "1669", "2979", "131", "9637", "7051", "653", "8572", "9393", "8693", "1096", "6578", "8753", "8538", "6395", "8955", "2038", "908", "3983", "5850", "1148", "9787", "9930", "562", "1679", "9739", "8340", "3744", "6605", "6216", "1797", "4990", "8035", "7451", "8907", "853", "6941", "4031", "4739", "4299", "5829", "3221", "2022", "4964", "3044", "1505", "6262", "5546", "1850", "7878", "3559", "8374", "2307", "5267", "2966", "7606", "4189", "2560", "9820", "3280", "1458", "3573", "6048", "4502", "6315", "8612", "3887", "9229", "4579", "5954", "8255", "9413", "5725", "2324", "6653", "611", "4193", "8802", "3910", "8917", "418", "326", "1793", "372", "1070", "4153", "9729", "5528", "1494", "8288", "6303", "8047", "1102", "1323", "9078", "6225", "6726", "9847", "5745", "8725", "6923", "4250", "6696", "8347", "7786", "6450", "9395", "1445", "9417", "2627", "1607", "2955", "7263", "2770", "6580", "7648", "3442", "9447", "9863", "6001", "6316", "5822", "1369", "7456", "2471", "9723", "8140", "3759", "4030", "9359", "1216", "2588", "1023", "8051", "5499", "4797", "5603", "9150", "3980", "6463", "3146", "634", "6069", "6906", "7829", "5644", "1921", "2320", "1175", "561", "8491", "8515", "2353", "7667", "8381", "9206", "6254", "6857", "7201", "3086", "2638", "3591", "6049", "9060", "4754", "2675", "5601", "6190", "5612", "6688", "8636", "2622", "5720", "9917", "7835", "1174", "6126", "2252", "4436", "3615", "4146", "3372", "9094", "4961", "8502", "3017", "5761", "8975", "21", "3356", "7236", "6881", "346", "3735", "2290", "6143", "3101", "4292", "2809", "9996", "922", "6025", "4150", "6256", "2854", "7539", "6666", "8633", "9324", "421", "6109", "4473", "357", "5540", "6706", "5604", "9630", "122", "6035", "5296", "8227", "1946", "137", "3284", "2527", "8153", "9255", "791", "1084", "636", "6607", "3408", "5534", "5967", "1500", "9570", "4529", "236", "2024", "6708", "2394", "5496", "6374", "4693", "9830", "1933", "730", "2893", "2887", "9888", "975", "6020", "5891", "2904", "6051", "4722", "9079", "7955", "831", "1097", "7980", "7837", "1044", "1354", "4819", "2836", "6983", "815", "1755", "2659", "7545", "6370", "361", "3956", "6164", "9824", "3991", "5620", "9387", "9672", "1889", "4143", "4196", "6010", "6366", "2462", "8928", "2873", "7866", "4527", "8549", "6584", "1138", "6246", "2718", "2057", "3699", "9496", "4463", "4246", "8517", "7928", "9695", "8698", "3493", "9231", "6560", "7640", "2441", "4050", "6843", "9641", "6895", "4247", "9189", "8775", "1153", "3720", "445", "59", "8306", "2764", "1836", "7804", "5547", "5562", "6107", "2886", "4531", "5033", "5199", "2082", "2765", "2456", "3001", "2756", "6922", "8729", "6352", "2603", "3528", "1492", "6208", "2176", "951", "6494", "280", "8776", "3697", "8257", "499", "5087", "8958", "7902", "9515", "6358", "3981", "6248", "1725", "2704", "9823", "3303", "9289", "8440", "1552", "8224", "652", "5134", "3645", "5527", "6475", "3210", "9303", "7561", "5981", "2799", "1250", "497", "7762", "6723", "5120", "7681", "6061", "2812", "4243", "6037", "5778", "6006", "1561", "8641", "9774", "6709", "7537", "80", "7799", "1842", "7524", "2385", "2680", "5584", "1594", "50", "5976", "7839", "7322", "1985", "3994", "6477", "9840", "3830", "5315", "5569", "3401", "9883", "1863", "1201", "4956", "2841", "3301", "6582", "2950", "1254", "6083", "6917", "599", "7392", "818", "2587", "9959", "6659", "995", "9682", "106", "6394", "1822", "7010", "5657", "6639", "5121", "9284", "3216", "1234", "5316", "6112", "5969", "1419", "7526", "3359", "5781", "2281", "7679", "9606", "3024", "9956", "4913", "3595", "5517", "3879", "2802", "2604", "8339", "8295", "9429", "9136", "3404", "8036", "5564", "2523", "9127", "4812", "6894", "2393", "4177", "4381", "4271", "764", "6916", "9046", "1592", "2709", "4144", "4612", "5438", "6842", "7505", "6009", "891", "6215", "3923", "3384", "9655", "2796", "7830", "3622", "6016", "5670", "8284", "4341", "1785", "9588", "3314", "1059", "806", "1588", "9598", "4164", "8012", "8500", "9158", "9599", "9758", "7242", "8632", "3172", "7581", "4082", "3262", "56", "4277", "2426", "2888", "1132", "4331", "3554", "6738", "5904", "1264", "99", "5395", "546", "5627", "8354", "51", "6981", "5400", "9557", "7776", "233", "1538", "4303", "2162", "1222", "2785", "8142", "7092", "9716", "8503", "5420", "3846", "6545", "1005", "1632", "9740", "948", "368", "9452", "2206", "4232", "754", "4906", "1281", "295", "3635", "5924", "8454", "3704", "3192", "6206", "2368", "3482", "6241", "2577", "9833", "4451", "3122", "4113", "9526", "4625", "5876", "9873", "4872", "4112", "1595", "5304", "651", "9114", "7703", "390", "5531", "4751", "6105", "491", "6271", "5794", "7155", "8165", "7998", "495", "9130", "7256", "3380", "1195", "3224", "1137", "5368", "3377", "6573", "6784", "4719", "2610", "8588", "4393", "6565", "1031", "7657", "5309", "9187", "8249", "2611", "4295", "4282", "1080", "9405", "7595", "4026", "6925", "7696", "2163", "6924", "8265", "4338", "3644", "1344", "2582", "244", "964", "6740", "8755", "3917", "7317", "5948", "8476", "9356", "3750", "3234", "7780", "203", "3965", "7557", "6811", "4343", "153", "1789", "6954", "3293", "7273", "2642", "6508", "8837", "2280", "2132", "3722", "7286", "1885", "183", "1633", "1068", "2460", "6966", "3815", "8369", "46", "6124", "9311", "5254", "4748", "5079", "6880", "7188", "2998", "4841", "2810", "3212", "7142", "688", "7751", "6568", "642", "4938", "4583", "2906", "1781", "3213", "5300", "4274", "5108", "2772", "5858", "7593", "1348", "9436", "1579", "8873", "6722", "5716", "6949", "4728", "2978", "1278", "4714", "7239", "9085", "3825", "86", "1409", "3547", "9251", "4301", "8338", "8496", "488", "2780", "9743", "1523", "6570", "7748", "5125", "5383", "8497", "1028", "6982", "584", "8194", "1307", "1107", "3075", "7893", "8551", "7402", "264", "3050", "6428", "7021", "8287", "9802", "2234", "1433", "8253", "7131", "5018", "4618", "6489", "4692", "7203", "8603", "6774", "6288", "7495", "9310", "2379", "1406", "9771", "8624", "7371", "7724", "58", "4744", "4018", "344", "382", "4698", "6567", "775", "2089", "9703", "1371", "3679", "3437", "8024", "6419", "2155", "6219", "3190", "5377", "2798", "5605", "619", "8059", "4984", "6305", "1930", "2985", "303", "1768", "1926", "5052", "6279", "807", "8650", "7535", "7551", "8070", "7831", "5916", "9777", "6439", "8417", "6313", "9089", "3717", "1342", "6221", "6380", "8286", "9318", "3779", "9492", "8139", "7880", "9944", "276", "6848", "9227", "9849", "5195", "6698", "3353", "9065", "8716", "9954", "6082", "5910", "2205", "6036", "1979", "4332", "2287", "6437", "8248", "7589", "5837", "1124", "565", "6399", "8158", "3758", "2553", "8209", "9800", "2820", "1156", "8461", "1337", "366", "7", "324", "1565", "2540", "1257", "8543", "8851", "4344", "1364", "3028", "8840", "3511", "2947", "2391", "5669", "6345", "8008", "6575", "1333", "3230", "386", "6914", "7116", "3958", "1303", "9209", "3061", "7832", "5374", "720", "2292", "9747", "8477", "4557", "7009", "3797", "8841", "3052", "9113", "260", "8250", "1935", "2928", "761", "2524", "8611", "3587", "4837", "8931", "4132", "9620", "4882", "827", "8779", "2037", "6360", "3862", "7467", "4372", "8731", "7264", "6934", "6033", "856", "8530", "425", "1209", "2803", "3728", "1829", "9953", "7781", "4864", "710", "1006", "2305", "3426", "145", "5558", "9334", "7785", "2354", "8969", "1420", "4522", "2377", "9259", "5913", "2448", "3285", "2261", "645", "2053", "5909", "8093", "6402", "3624", "4903", "8054", "2514", "5116", "8447", "9706", "1774", "4725", "1763", "2539", "5062", "7811", "4815", "7339", "7499", "4768", "4985", "7588", "7150", "1381", "3913", "1577", "6778", "8599", "146", "5149", "8310", "7170", "3782", "4116", "4327", "2390", "1560", "5961", "6792", "3590", "5000", "5622", "8987", "8691", "7997", "5911", "2118", "8690", "5749", "1360", "7915", "4770", "380", "9213", "4785", "2033", "160", "5680", "8508", "855", "6945", "8178", "7634", "1073", "4647", "3171", "400", "5919", "8373", "6873", "1244", "5668", "9985", "475", "5580", "1114", "1470", "456", "3439", "8821", "288", "1422", "8429", "2835", "9336", "4006", "8902", "6142", "7347", "379", "5554", "9407", "196", "6631", "5667", "758", "6269", "570", "9810", "3054", "8408", "2833", "4080", "2119", "8831", "8724", "2223", "5478", "6149", "4425", "6137", "5009", "7318", "4835", "70", "1945", "6184", "6936", "7553", "3533", "3823", "722", "5045", "6672", "7216", "7777", "6068", "2571", "1172", "3874", "3175", "1411", "2689", "8390", "2777", "7335", "1192", "5281", "7094", "8399", "5481", "454", "7083", "345", "2951", "149", "334", "4525", "4001", "9400", "3610", "753", "5963", "9204", "9827", "5161", "8398", "1666", "4514", "2227", "3378", "1025", "2607", "1744", "2816", "849", "3619", "5465", "3690", "363", "37", "6554", "9365", "4096", "8738", "4937", "8244", "2794", "9005", "862", "5719", "8552", "7901", "4071", "6604", "5931", "2042", "9105", "9243", "8667", "9374", "5243", "4413", "1386", "6816", "3535", "369", "7114", "5202", "9322", "2016", "8204", "536", "1223", "7452", "5855", "2757", "6514", "4021", "2327", "1776", "2899", "8488", "5440", "6323", "6728", "2131", "2015", "5466", "1687", "8876", "2067", "2636", "4546", "1978", "4368", "813", "1144", "5115", "6588", "2435", "9247", "9708", "7787", "4589", "4491", "4152", "320", "5138", "3318", "7284", "857", "7079", "8014", "2389", "1240", "1159", "3053", "941", "5784", "1310", "1258", "5241", "6601", "1322", "1295", "917", "3089", "5508", "9533", "1801", "3066", "9860", "8001", "5104", "9566", "7283", "4614", "7760", "8375", "2336", "2195", "6498", "2828", "202", "2025", "2298", "4081", "1623", "2624", "8595", "7857", "9260", "9102", "5358", "9756", "8868", "129", "4482", "9221", "9369", "1752", "1710", "5194", "9015", "5516", "4279", "6645", "6024", "610", "723", "5404", "8721", "6181", "8251", "4680", "788", "6749", "2165", "342", "98", "9766", "7054", "7512", "7779", "9035", "5063", "8379", "8294", "8182", "1173", "5084", "6797", "6461", "5430", "8527", "7700", "1922", "533", "3139", "2534", "6364", "3102", "8478", "871", "9986", "6099", "9145", "4284", "1610", "3163", "3502", "9110", "3070", "9003", "6990", "2762", "8832", "1110", "9635", "2105", "9176", "1686", "1567", "6957", "6608", "2536", "9293", "7418", "5248", "7272", "5284", "9702", "3814", "1072", "6736", "7036", "4176", "9761", "1294", "5567", "7964", "2643", "6018", "8464", "9762", "4874", "7864", "5391", "6995", "3800", "9663", "1180", "1676", "2712", "2754", "9642", "9068", "9442", "6929", "5055", "4855", "3004", "868", "1572", "800", "2000", "3497", "7729", "3571", "2572", "4371", "6757", "7600", "4729", "3279", "8619", "6423", "7384", "5938", "1656", "9634", "8786", "3883", "8960", "9905", "192", "8791", "1974", "3951", "5164", "1548", "8718", "293", "5935", "7171", "4723", "2859", "2", "1063", "5533", "4839", "2525", "1643", "6552", "8099", "7728", "6987", "1742", "452", "6525", "8396", "2498", "5656", "7730", "2095", "4167", "9438", "877", "9093", "6909", "434", "7220", "8585", "2196", "1912", "4408", "4631", "7320", "9769", "7055", "6960", "6550", "2908", "1313", "7530", "3034", "4865", "2387", "5427", "7041", "6822", "4028", "186", "9793", "1630", "8532", "9372", "4095", "142", "4403", "3948", "3520", "4866", "8985", "1190", "3374", "3430", "3921", "982", "2358", "6805", "1444", "9922", "1261", "9299", "8537", "3578", "4515", "4334", "333", "1171", "4044", "2941", "4042", "2101", "2006", "5364", "225", "9834", "9477", "9799", "1321", "7056", "9384", "4202", "9499", "6052", "4127", "5691", "3773", "5777", "2229", "7107", "4426", "6229", "2203", "3117", "2317", "1739", "338", "9808", "3445", "1625", "8732", "8451", "4782", "9903", "4005", "9185", "1986", "2428", "1587", "3678", "6768", "9479", "7733", "2981", "8857", "3909", "726", "3914", "1008", "2537", "5200", "1841", "9937", "5355", "3629", "8279", "6056", "949", "7348", "1920", "4762", "7243", "3582", "4849", "9775", "586", "4357", "5732", "7097", "4942", "3354", "8915", "8679", "8276", "3791", "3443", "6158", "7754", "4041", "2800", "1152", "375", "1050", "9043", "9375", "4370", "5917", "7816", "1473", "2662", "2083", "8102", "823", "2505", "6445", "8400", "6172", "9451", "3855", "6088", "126", "6150", "5314", "72", "496", "5775", "872", "1474", "1064", "6734", "6907", "1848", "2818", "5008", "1665", "4063", "2065", "9950", "2729", "6100", "7489", "3642", "7797", "1248", "453", "5188", "3626", "9306", "9004", "4774", "5375", "8509", "8859", "7244", "6823", "1578", "7514", "8562", "1251", "3727", "3510", "1806", "5763", "6927", "3328", "4418", "8860", "221", "1357", "7885", "4558", "8592", "8205", "6897", "9437", "62", "542", "7298", "916", "1942", "5408", "9906", "5152", "8598", "9087", "4989", "6724", "8292", "1403", "1557", "1290", "3523", "2615", "2825", "5163", "7925", "1820", "4977", "5630", "2467", "8443", "8513", "4566", "3979", "2240", "125", "4342", "1094", "6978", "9488", "7962", "4000", "9957", "8728", "5663", "7924", "5526", "4145", "7444", "694", "9246", "2253", "4507", "8833", "8042", "8827", "471", "3680", "9196", "8256", "7442", "3997", "864", "2170", "8074", "3631", "9955", "4435", "7277", "6185", "5624", "5128", "8634", "5776", "3376", "5451", "6755", "3220", "2829", "7482", "111", "2584", "6432", "6813", "9266", "3400", "4902", "8564", "7088", "1719", "1332", "4203", "242", "4981", "997", "2425", "6499", "4642", "9885", "304", "3930", "9730", "121", "1534", "5157", "1463", "3912", "8148", "9862", "4221", "8606", "4241", "9712", "2705", "9640", "150", "6812", "7035", "8704", "3347", "2834", "1550", "4601", "7649", "8687", "6959", "9033", "9942", "9929", "2247", "5626", "5762", "5094", "24", "6233", "6829", "1866", "7953", "5495", "523", "268", "1906", "8020", "6837", "8291", "2395", "7459", "8808", "6074", "5812", "100", "3593", "6469", "8816", "6542", "6092", "731", "2250", "989", "1349", "9770", "5819", "5201", "1760", "6710", "8433", "7659", "3766", "4828", "3649", "7334", "6304", "5884", "7792", "8774", "9877", "4805", "30", "1522", "6503", "9362", "4440", "1429", "5196", "7312", "2122", "4484", "216", "8241", "5277", "3274", "4845", "5788", "2892", "2439", "3045", "1892", "7120", "3434", "1204", "5615", "2945", "443", "7541", "8053", "8079", "8037", "305", "7783", "5834", "3173", "3255", "287", "2547", "5936", "6095", "6258", "8756", "5318", "6397", "9115", "1840", "1756", "2147", "6975", "2530", "8404", "9366", "2136", "5229", "7001", "560", "7716", "1546", "3866", "7125", "7472", "7947", "5210", "6979", "6046", "4862", "6682", "6086", "1677", "9143", "2673", "2235", "514", "8363", "6320", "5259", "8107", "1331", "6444", "8852", "1777", "6266", "2040", "1401", "8290", "290", "7523", "9969", "4073", "3281", "7082", "7202", "3592", "2036", "8803", "2238", "4738", "3465", "2097", "5929", "7769", "3035", "1859", "240", "170", "3734", "7161", "5789", "4012", "4423", "2629", "6854", "7374", "4470", "1680", "8930", "5392", "2814", "5455", "626", "5602", "9624", "9327", "8553", "7377", "6223", "271", "7198", "213", "3261", "6810", "1432", "4236", "4908", "5312", "6920", "9518", "5397", "3118", "7484", "7048", "1013", "7238", "3729", "1684", "1227", "7942", "5370", "8658", "7638", "661", "8423", "6300", "8853", "4465", "9783", "370", "4619", "9273", "5979", "1103", "8382", "1118", "8455", "3033", "8735", "7768", "4955", "9062", "207", "2133", "3594", "5261", "9412", "5951", "6414", "9658", "2943", "4088", "3153", "3010", "8234", "8906", "509", "3864", "249", "5036", "7604", "4909", "6752", "2559", "6390", "325", "2519", "6179", "7990", "5406", "5271", "772", "2443", "7075", "9722", "313", "3361", "7064", "6595", "9465", "8711", "4765", "8492", "3214", "6353", "9245", "9536", "7401", "7906", "646", "8467", "9875", "1913", "84", "2047", "4584", "5586", "231", "2600", "6510", "9551", "8601", "1462", "557", "4090", "1547", "2388", "308", "406", "8628", "4244", "8267", "837", "8084", "4204", "2372", "1611", "235", "9201", "4054", "6151", "6454", "6862", "1862", "365", "234", "3491", "1340", "2447", "8540", "3996", "8957", "6404", "4915", "4489", "8908", "531", "1454", "4708", "3794", "270", "1277", "7311", "4387", "378", "7316", "5228", "6438", "5372", "3492", "5824", "2548", "6132", "6621", "7921", "5731", "9317", "1229", "822", "2745", "5943", "6564", "6110", "2827", "7141", "6630", "485", "7808", "5849", "2847", "2668", "4741", "3270", "283", "7750", "7435", "9752", "3412", "1603", "4467", "1689", "4200", "1111", "5791", "6162", "3481", "1568", "4367", "1424", "9529", "6776", "7337", "8670", "3806", "4879", "4616", "5798", "6893", "3026", "9261", "900", "2407", "5208", "7341", "3031", "6926", "4424", "3176", "7772", "1121", "3812", "4665", "6597", "171", "2502", "3787", "4293", "9665", "9589", "5831", "3123", "2073", "4210", "1728", "7204", "3973", "1627", "6118", "3726", "2652", "9186", "3999", "5204", "2338", "738", "6681", "5021", "4190", "8171", "7232", "5901", "5439", "9254", "839", "8077", "803", "8817", "5550", "1343", "5548", "6530", "4852", "5985", "175", "572", "687", "9946", "4266", "5712", "9751", "6023", "7933", "6999", "8568", "9228", "7310", "5396", "1914", "1004", "4928", "1398", "1537", "6496", "3968", "3198", "8778", "9683", "8800", "8407", "2104", "4791", "5251", "1079", "9343", "8989", "9382", "5955", "6479", "6122", "6548", "2645", "3799", "4347", "7488", "7275", "4045", "6188", "1751", "9514", "747", "1792", "6311", "7803", "9543", "4115", "9020", "1530", "2921", "9995", "1962", "1721", "3271", "8904", "7749", "48", "3409", "8920", "3065", "7023", "3893", "2117", "8782", "9650", "248", "312", "7930", "2683", "3998", "9170", "2784", "376", "5738", "2564", "9376", "8744", "9801", "1324", "6080", "3689", "1443", "2942", "5804", "938", "5679", "3903", "4128", "6579", "5463", "819", "8922", "9638", "4234", "559", "6629", "7397", "5434", "289", "3112", "5401", "915", "4185", "5240", "8380", "417", "9049", "1773", "7127", "239", "8384", "705", "8881", "9671", "4844", "9161", "8243", "177", "7061", "9553", "1772", "7584", "709", "1329", "7030", "4053", "42", "5795", "5539", "5119", "278", "3073", "8741", "6678", "3988", "9866", "1705", "7813", "5175", "3485", "5330", "2044", "2973", "9279", "7826", "8180", "3828", "4186", "8940", "5076", "2937", "6591", "3496", "9031", "7157", "5560", "8700", "2597", "9171", "2750", "8793", "3816", "3436", "1589", "4166", "1468", "6", "7662", "998", "5990", "6637", "6145", "3665", "675", "9726", "5069", "3433", "1798", "7380", "4330", "6766", "2890", "978", "3756", "1255", "8646", "2842", "4666", "1800", "9449", "5595", "4577", "9963", "262", "4510", "2562", "6125", "2208", "3310", "6411", "5293", "392", "2001", "676", "591", "2651", "6537", "7305", "4233", "492", "8304", "4887", "9516", "4350", "678", "3133", "2862", "3604", "6057", "3304", "8587", "502", "8328", "9528", "3796", "7074", "3963", "8947", "6836", "1835", "8394", "5371", "4629", "1746", "4340", "8282", "6260", "3793", "8162", "5733", "2268", "6325", "3185", "6844", "3217", "4876", "8231", "6129", "8371", "8346", "1194", "2228", "1515", "1544", "4581", "3919", "4318", "4793", "3733", "4521", "6635", "1392", "5274", "3332", "6318", "7619", "3955", "3264", "5888", "9713", "9844", "2382", "3339", "7852", "2546", "3556", "5830", "8245", "3461", "7121", "7404", "8609", "2231", "8235", "7031", "3534", "5007", "8828", "5338", "1531", "3167", "7794", "4109", "63", "5071", "7095", "2416", "8141", "7614", "4780", "8550", "6194", "1141", "7527", "2279", "9520", "9600", "8278", "5222", "712", "9534", "7583", "8734", "7139", "6619", "5922", "8883", "4375", "8834", "663", "3092", "907", "5328", "3238", "1830", "8456", "1647", "6211", "924", "35", "9674", "9495", "9146", "2791", "5166", "3158", "1302", "5048", "3789", "8333", "133", "3895", "8434", "3251", "5464", "9908", "6484", "1663", "9498", "3686", "47", "5425", "3761", "4806", "3392", "7993", "5351", "3764", "1753", "3", "272", "2455", "2927", "3751", "8365", "2098", "8351", "4777", "9017", "1952", "973", "1212", "7196", "8610", "5728", "6349", "8004", "1284", "6005", "9274", "1203", "1864", "8298", "9482", "441", "6665", "8392", "5800", "8642", "466", "3878", "6329", "1805", "7739", "8900", "5928", "9160", "2484", "8482", "281", "1667", "5409", "4622", "4769", "780", "7824", "2108", "9414", "7714", "5752", "2311", "5279", "5226", "5444", "6832", "7491", "211", "8751", "4732", "9597", "1089", "4124", "4084", "8854", "4626", "2707", "8743", "1387", "9697", "612", "7874", "9379", "8436", "1910", "727", "2152", "4392", "7635", "7462", "1230", "6175", "9109", "2246", "3368", "292", "8437", "4598", "1077", "7261", "8992", "6670", "6476", "2138", "1655", "5994", "6308", "4645", "7065", "4321", "9935", "6538", "1735", "9121", "2464", "5244", "285", "3441", "7300", "9235", "649", "4914", "6446", "8185", "1670", "2768", "5040", "2658", "1495", "87", "6178", "493", "7005", "6115", "9123", "7038", "748", "9575", "5880", "1052", "5433", "92", "7494", "2418", "6807", "7479", "9128", "7625", "3931", "2958", "4394", "389", "804", "9167", "2551", "5137", "3529", "7270", "8425", "9742", "5949", "3551", "9111", "7276", "5441", "3970", "2703", "2891", "6694", "7693", "6000", "9040", "6171", "2826", "2078", "9468", "1218", "1428", "3817", "8092", "9312", "6350", "5991", "9544", "6239", "5043", "4764", "8062", "2406", "5126", "8919", "3819", "5918", "5783", "5153", "6783", "6022", "7646", "7003", "7378", "11", "1871", "1241", "7795", "9698", "1918", "5415", "4216", "810", "3621", "6133", "7182", "6522", "8655", "5790", "6330", "4587", "8458", "5319", "5272", "6991", "2984", "4787", "9945", "4747", "2681", "3233", "6756", "6231", "261", "7431", "7160", "7207", "5388", "6458", "8321", "845", "9241", "5494", "2129", "9868", "4790", "3561", "4881", "409", "933", "604", "798", "9047", "9843", "1695", "5840", "7346", "1451", "3842", "4707", "4061", "8621", "4716", "8146", "189", "9018", "3232", "7497", "8138", "4582", "4296", "2004", "2478", "9549", "9546", "8387", "1703", "6523", "4730", "3788", "664", "8772", "4635", "7321", "1837", "9308", "2925", "6602", "4971", "2844", "3236", "1712", "512", "8639", "9321", "4929", "8124", "8962", "6955", "1987", "8703", "9836", "6684", "9975", "2274", "5998", "566", "3813", "5227", "3422", "4591", "1499", "8893", "6041", "6875", "5568", "8136", "4877", "1466", "7631", "6234", "3509", "8067", "6764", "4724", "6113", "8701", "5744", "222", "686", "2345", "3429", "1404", "9257", "5809", "7686", "6746", "9364", "5385", "1164", "1247", "6765", "7454", "1481", "1813", "991", "8337", "3352", "3858", "6369", "7726", "7413", "2237", "2896", "8343", "4273", "5613", "6470", "3570", "194", "8846", "3459", "8645", "2963", "208", "7611", "2849", "6720", "6935", "3584", "1917", "8884", "3719", "7722", "401", "6340", "7513", "3843", "2139", "5536", "9354", "8998", "9654", "7279", "4077", "1283", "2987", "7950", "9450", "5220", "9118", "8579", "3218", "530", "5326", "9711", "7650", "5407", "1694", "9265", "457", "8647", "2948", "3588", "5774", "1598", "6412", "7357", "2517", "6763", "1519", "5247", "5324", "7886", "5365", "8558", "5295", "9690", "5353", "25", "7669", "3783", "3853", "8391", "582", "5854", "6950", "6298", "8203", "4814", "9736", "3126", "1697", "1390", "9117", "4875", "6703", "1901", "9662", "7326", "440", "1948", "3691", "5294", "8386", "1297", "6712", "2185", "4223", "3343", "2838", "746", "3300", "939", "9778", "1932", "2578", "735", "7227", "8982", "1233", "3235", "6779", "571", "193", "6521", "5214", "8045", "2664", "7086", "3623", "4712", "8891", "9263", "1060", "3918", "6361", "8954", "1099", "4808", "8504", "172", "6971", "9531", "9816", "6153", "2217", "6725", "7020", "6845", "2012", "8750", "2863", "5359", "1", "4486", "2976", "267", "683", "9252", "8112", "555", "921", "5056", "1899", "174", "6972", "4157", "7949", "1136", "6968", "8980", "691", "5340", "5418", "9791", "5714", "2676", "3275", "7476", "4536", "5513", "7639", "3333", "4029", "7800", "8418", "4235", "7049", "1267", "2423", "2453", "5814", "6625", "265", "6634", "9072", "60", "7966", "3315", "9063", "9398", "5930", "1883", "5256", "8050", "2219", "3964", "9071", "5102", "6455", "9889", "6085", "5690", "7474", "5748", "9392", "8484", "460", "7732", "3447", "9458", "3946", "1359", "6296", "9051", "191", "885", "3703", "3018", "4532", "4941", "6427", "7717", "677", "8874", "8829", "4501", "2632", "4809", "4496", "6449", "3916", "8415", "2728", "2956", "7173", "2271", "5290", "5167", "1965", "3055", "9370", "3060", "3600", "4033", "1069", "4211", "7319", "5650", "9203", "4843", "4559", "93", "2161", "739", "4585", "5211", "616", "1631", "7299", "4363", "8368", "6819", "1021", "9991", "6947", "2585", "8600", "5734", "6534", "539", "2411", "5907", "360", "4995", "2449", "6524", "7032", "4542", "449", "3661", "7293", "6031", "8445", "5413", "467", "5165", "8843", "5282", "8052", "415", "3410", "3517", "2374", "9401", "9224", "9314", "4575", "2153", "3712", "2716", "5518", "6295", "8144", "6409", "4615", "1459", "352", "9692", "9052", "6771", "7546", "6117", "3508", "6942", "4966", "3989", "7636", "1988", "9788", "1160", "5621", "5773", "3435", "9993", "8976", "3684", "3772", "9194", "4480", "8011", "6443", "6743", "2516", "2199", "8666", "5964", "9326", "8296", "6655", "2992", "4801", "1799", "298", "6803", "7736", "2618", "2154", "4311", "6398", "6462", "5380", "1767", "7788", "7853", "3860", "8097", "2058", "8864", "6456", "4085", "5766", "838", "4951", "3982", "7986", "4209", "2128", "7673", "1511", "9853", "909", "8309", "6281", "4950", "6599", "9012", "828", "2719", "103", "7761", "594", "1461", "9594", "3383", "436", "3536", "2808", "8643", "5156", "923", "4518", "6111", "3625", "6721", "4884", "5934", "4948", "8569", "814", "4834", "621", "3738", "8345", "3501", "7445", "714", "8419", "5708", "8623", "135", "4278", "5278", "1732", "7538", "7353", "8017", "8174", "7450", "6596", "9887", "564", "5327", "9664", "3688", "5206", "2003", "7612", "6679", "3905", "8689", "6064", "1133", "5176", "2877", "8577", "2222", "9388", "5886", "9363", "9439", "7421", "2739", "5473", "4133", "4240", "3865", "7798", "1539", "4824", "6198", "3170", "5940", "2583", "3290", "9485", "1513", "8939", "5249", "2880", "2690", "3723", "6371", "8850", "1086", "6249", "4983", "9032", "6800", "5945", "2386", "9082", "15", "7136", "5502", "6657", "199", "1147", "2085", "6447", "8007", "927", "7575", "2383", "6058", "756", "9958", "5265", "9645", "3856", "8486", "9914", "7135", "3402", "477", "7129", "9855", "2429", "6436", "2789", "1878", "3795", "8460", "119", "5520", "26", "1076", "1717", "4009", "7381", "4506", "8293", "9027", "8932", "7774", "3824", "7217", "8977", "9931", "2115", "8886", "2868", "2721", "6668", "139", "2100", "4106", "9316", "4973", "3889", "504", "9236", "8352", "1431", "2660", "2367", "7336", "7382", "2840", "3763", "5459", "1215", "9602", "6963", "1877", "532", "4901", "5287", "8659", "8401", "4922", "2120", "8322", "3836", "2064", "556", "6483", "5416", "9763", "9037", "356", "3564", "5758", "3755", "5780", "6751", "7117", "9440", "6553", "7632", "8472", "1447", "1402", "1464", "1335", "7656", "2278", "9041", "1528", "4492", "8855", "7296", "3698", "783", "1867", "8764", "6299", "7167", "2674", "9694", "1980", "8048", "4197", "5307", "6410", "7307", "6059", "8625", "861", "7345", "947", "1184", "6616", "8341", "507", "771", "4959", "8959", "2735", "3140", "3596", "797", "8965", "2596", "4893", "8942", "9399", "3522", "5336", "5083", "547", "1139", "9309", "7465", "3743", "8719", "5286", "9681", "8644", "7186", "4776", "3468", "4606", "6116", "7325", "8285", "3023", "9391", "1187", "8839", "4107", "8055", "9973", "3431", "1373", "1507", "1214", "5006", "2431", "2567", "9886", "1923", "4638", "1615", "3246", "1843", "28", "446", "4545", "2404", "9297", "4445", "384", "9175", "8541", "4927", "3648", "201", "8752", "8882", "7992", "5333", "8172", "4493", "481", "5926", "8727", "579", "1039", "4652", "7363", "7899", "9237", "6877", "4064", "8984", "2602", "4550", "608", "6310", "8421", "1714", "1661", "4520", "7720", "4263", "5900", "6384", "1692", "5751", "7801", "2323", "4934", "4103", "9164", "2790", "6705", "167", "782", "4431", "7089", "1783", "529", "3046", "3206", "8654", "9979", "6452", "4823", "8547", "1367", "7396", "1691", "8708", "865", "5782", "1571", "2644", "8524", "7536", "4034", "527", "252", "9083", "4736", "5700", "1997", "4982", "9932", "4399", "3877", "6780", "5906", "387", "1929", "2096", "130", "2094", "2430", "4453", "3974", "5343", "4218", "1336", "4419", "8332", "5653", "3746", "8402", "6649", "2483", "9856", "4298", "6727", "7100", "7423", "3765", "8896", "5323", "6674", "7355", "32", "2182", "2167", "8866", "4476", "1904", "657", "2889", "1036", "8535", "3984", "7029", "3103", "3466", "5130", "6146", "1984", "3411", "5402", "2365", "5820", "4891", "3627", "3922", "4600", "2581", "2380", "8720", "3423", "6622", "1795", "5001", "2488", "2433", "2846", "846", "6878", "2127", "6191", "1032", "6677", "1465", "926", "1605", "5127", "9715", "6030", "9744", "7520", "3897", "3809", "2628", "4637", "5996", "405", "3928", "7855", "2521", "7742", "578", "1882", "2282", "2592", "6701", "3780", "1266", "4345", "1091", "3326", "7757", "3276", "8179", "3653", "5600", "7419", "5859", "7410", "7152", "1289", "9329", "9924", "6003", "331", "4604", "8089", "5350", "5902", "8905", "1142", "7674", "2872", "8155", "9430", "3781", "5662", "3724", "8522", "8217", "284", "7508", "6932", "2414", "8151", "4498", "3871", "8428", "6899", "8548", "2730", "8888", "7471", "5025", "6791", "2143", "2670", "8590", "5611", "2508", "7810", "1062", "4954", "227", "8263", "8964", "8044", "563", "5012", "4795", "3854", "625", "7230", "6488", "9585", "8664", "3181", "2903", "5098", "5237", "85", "724", "925", "5664", "4119", "7420", "5190", "1018", "3705", "2357", "3774", "4556", "4735", "3015", "9220", "9938", "3612", "3487", "617", "8813", "335", "7954", "685", "7105", "3798", "4260", "3741", "6072", "4621", "9394", "9952", "2881", "1991", "8604", "1902", "952", "4636", "8120", "102", "4643", "2333", "3558", "9572", "412", "4156", "7935", "4325", "7390", "6480", "9987", "151", "9733", "9446", "4633", "1327", "5145", "3007", "8357", "8788", "3880", "3504", "4811", "2843", "4313", "2744", "5169", "7689", "273", "729", "8663", "2285", "8422", "7268", "4672", "6375", "7398", "8236", "1075", "7847", "2277", "6094", "8783", "6407", "381", "4641", "8819", "9006", "600", "2848", "6834", "2965", "3695", "4407", "3682", "2678", "9567", "7084", "1903", "472", "1833", "7904", "8442", "3090", "920", "61", "9489", "1880", "43", "486", "4733", "824", "1911", "622", "7037", "7490", "5450", "4889", "8403", "8913", "5655", "7743", "4254", "1608", "7340", "3505", "835", "1188", "2806", "1350", "1580", "33", "5235", "5389", "6302", "816", "2725", "1416", "9443", "3641", "38", "1066", "7040", "6224", "6379", "8473", "1529", "7668", "8143", "3498", "8723", "2968", "2684", "1155", "6841", "8771", "6376", "3599", "6928", "1521", "6327", "4690", "180", "2369", "8046", "9162", "7559", "8480", "5821", "4414", "8781", "4089", "4391", "4297", "3470", "2695", "5852", "4215", "1925", "6838", "1378", "640", "6859", "4586", "9367", "6084", "8016", "3888", "6731", "4208", "5212", "4057", "1736", "4048", "1288", "4899", "8474", "1166", "2837", "6704", "9748", "9441", "3760", "4316", "1037", "1759", "6212", "6355", "9320", "9337", "4302", "7718", "2145", "6748", "2264", "7328", "2018", "4307", "7002", "4300", "197", "7821", "182", "7596", "7197", "336", "7863", "1941", "2815", "6831", "2723", "7265", "7344", "7784", "8218", "8238", "1708", "383", "6958", "673", "5506", "569", "4308", "5310", "2366", "2014", "5537", "7973", "1723", "2918", "8355", "1868", "3700", "3204", "7468", "4369", "8648", "2378", "2793", "8187", "3166", "2563", "3467", "1888", "1045", "6343", "5492", "8471", "4561", "3748", "6045", "4322", "3572", "2232", "2614", "6793", "1003", "7715", "4168", "5607", "8117", "118", "7409", "8934", "3805", "6730", "1053", "632", "1826", "8763", "6759", "6011", "4286", "1441", "6641", "9962", "6451", "413", "8973", "7882", "6969", "4567", "362", "9149", "1312", "6818", "5285", "3541", "893", "2554", "6283", "6860", "7245", "5454", "3330", "6504", "9998", "2041", "350", "1394", "8160", "2509", "4192", "1636", "1232", "9275", "6785", "9821", "8510", "8409", "343", "7851", "543", "3097", "6093", "5694", "1509", "2226", "8865", "7124", "2982", "5424", "6490", "6970", "3187", "101", "8108", "7737", "732", "2724", "6527", "9508", "8712", "3114", "5947", "4460", "6754", "2088", "9191", "4417", "3803", "8518", "6840", "1015", "6338", "6586", "3016", "4446", "8192", "3387", "2438", "9397", "8128", "1731", "9899", "4986", "9504", "3581", "2967", "1512", "6240", "9972", "4850", "2399", "5168", "7629", "2295", "6502", "9685", "8672", "7091", "9283", "3668", "1300", "9433", "1270", "3424", "1016", "8208", "5093", "505", "7564", "5606", "2865", "8350", "6217", "5989", "9618", "6642", "5641", "1438", "6247", "6474", "9879", "3844", "7744", "2363", "4715", "2218", "6060", "8184", "725", "3069", "4237", "3519", "847", "4455", "8925", "9501", "5034", "524", "9070", "5129", "4065", "7680", "1186", "3215", "5808", "2565", "2773", "4389", "6594", "8277", "7331", "4932", "2734", "7478", "8498", "5760", "757", "8364", "282", "5337", "3935", "6386", "482", "1395", "3611", "4125", "7393", "2450", "3663", "658", "2920", "1384", "1002", "5275", "6335", "9607", "1453", "6737", "7850", "3241", "1282", "1170", "5489", "4858", "1958", "2758", "2224", "1724", "3151", "8247", "5181", "5470", "4596", "662", "9095", "7193", "5677", "8806", "8544", "2194", "8950", "4523", "6272", "1576", "4290", "6464", "2151", "5666", "8206", "2970", "9659", "6686", "3205", "3863", "96", "9519", "3580", "1707", "3091", "479", "5180", "7011", "5867", "3406", "890", "3484", "1042", "690", "2046", "2650", "9197", "7302", "8259", "826", "1857", "3861", "3681", "3063", "5726", "5299", "5073", "3037", "5705", "5589", "3601", "3042", "266", "851", "8858", "2341", "2535", "228", "246", "6196", "3539", "9039", "4219", "6663", "5769", "3252", "8974", "7343", "8220", "4438", "9486", "7165", "3839", "4264", "1279", "8019", "3006", "9480", "779", "5845", "2623", "8536", "5209", "2929", "9521", "219", "8101", "2851", "1206", "7969", "9967", "4320", "3792", "8127", "6689", "6400", "1982", "2370", "4789", "5750", "5035", "1968", "6321", "648", "9129", "9420", "769", "4373", "4182", "5292", "5863", "2080", "7072", "6733", "349", "3752", "5681", "4194", "7249", "8818", "859", "7473", "8567", "321", "5817", "5112", "9077", "8785", "5966", "6213", "2575", "7892", "5873", "5999", "6418", "7045", "741", "4505", "3448", "8397", "6089", "4910", "1525", "9044", "9182", "4027", "5870", "430", "341", "29", "5975", "408", "6758", "5123", "7493", "3081", "7552", "310", "7873", "9677", "674", "8490", "2144", "1852", "385", "8372", "6357", "1207", "9058", "9506", "7678", "1876", "9427", "200", "4349", "7967", "2339", "1637", "2959", "4252", "1328", "8892", "2923", "643", "1659", "1019", "3677", "7259", "9066", "7235", "1778", "1407", "1832", "2994", "2299", "2995", "8988", "5186", "9686", "5342", "9144", "629", "2352", "897", "5028", "1540", "3196", "5419", "6739", "3875", "8326", "6988", "6377", "8199", "1351", "8948", "6997", "9614", "6532", "5219", "6770", "7620", "3029", "3940", "9735", "8890", "1326", "9286", "1117", "9878", "3345", "3418", "4432", "5735", "1881", "6054", "3381", "6081", "5816", "6660", "3838", "4118", "3851", "8512", "4668", "2084", "5543", "6274", "1396", "4682", "9966", "9152", "7309", "8258", "6186", "3841", "8898", "8261", "7905", "8799", "9500", "277", "4415", "2971", "4565", "9560", "8280", "4019", "3040", "4821", "3488", "8545", "7939", "6549", "9076", "8662", "9148", "9460", "4262", "6517", "7968", "2079", "2954", "7358", "9612", "5037", "9073", "1239", "1675", "873", "5925", "9510", "2953", "8583", "1161", "6658", "6232", "7324", "3711", "9678", "8385", "7327", "6306", "2283", "799", "3421", "8825", "4339", "5889", "73", "65", "7633", "9390", "4996", "5185", "9850", "6173", "7103", "7573", "3208", "4155", "4355", "6717", "4650", "7113", "6804", "902", "7977", "5142", "3506", "3706", "8760", "7597", "7183", "4416", "3670", "3708", "4539", "3211", "5399", "5868", "9470", "3071", "6903", "4225"]], "xtest": ["int", ["4627", "4669", "7487", "5811", "2749", "5770", "2910", "6263", "4306", "9934", "8921", "1049", "3186", "585", "3518", "9819", "9126", "9358", "6070", "937", "6127", "3388", "4921", "4924", "8571", "7470", "2413", "596", "8173", "9212", "9007", "5223", "6275", "7027", "4946", "223", "9383", "1452", "3933", "2620", "1604", "7070", "9423", "549", "5132", "854", "1861", "609", "3807", "5311", "2221", "3609", "4450", "399", "3808", "9992", "2326", "1809", "2351", "7376", "7111", "4245", "4644", "6078", "5026", "668", "1570", "5070", "3282", "5111", "4980", "892", "704", "3512", "5484", "6326", "4038", "1559", "9745", "2328", "8073", "6406", "3457", "490", "2319", "4555", "5234", "4003", "3827", "5449", "6729", "6354", "431", "7367", "6004", "2334", "6440", "4149", "297", "8320", "3549", "3350", "9434", "6090", "9838", "9693", "9199", "3316", "2626", "7096", "9192", "9912", "2962", "5842", "3514", "7979", "6675", "4925", "5787", "4939", "7908", "6718", "6193", "2574", "1949", "7820", "2021", "7362", "6886", "3952", "8836", "6646", "8686", "6636", "9593", "2511", "6656", "2051", "8561", "3638", "9603", "5692", "2999", "5042", "6044", "4174", "7975", "3312", "2434", "3249", "3115", "5483", "4409", "6372", "1510", "8412", "9342", "5266", "4134", "5339", "4175", "3882", "7213", "716", "7071", "8302", "9138", "5090", "4613", "8823", "8953", "8871", "7974", "5914", "3868", "7822", "7190", "5971", "7763", "3628", "7062", "7918", "7295", "3538", "6644", "2672", "4361", "8699", "1849", "4727", "7498", "3416", "6328", "9613", "641", "2698", "5939", "2649", "7610", "5972", "7860", "1397", "9757", "5616", "1873", "5642", "7185", "4766", "874", "9124", "6210", "3785", "552", "655", "4945", "9385", "9524", "9892", "3707", "109", "5173", "3801", "6160", "1977", "1827", "5283", "5105", "3005", "9239", "2329", "7153", "1526", "7211", "4820", "7483", "5357", "4868", "9977", "7366", "1496", "7556", "4999", "3132", "6235", "4285", "7547", "7424", "1727", "1887", "9737", "9679", "733", "8681", "2286", "2263", "4798", "955", "3385", "4867", "942", "9271", "1844", "3480", "1112", "8709", "7568", "4010", "5511", "5730", "3379", "2545", "8080", "2737", "3093", "6536", "8031", "9576", "9721", "2691", "136", "403", "4043", "7373", "1940", "7271", "7134", "3286", "6413", "138", "6531", "367", "4659", "6745", "7594", "2312", "7060", "3452", "2270", "2538", "8317", "5313", "2346", "850", "1414", "4963", "7642", "6520", "7052", "6939", "5864", "1001", "8845", "3463", "9984", "6585", "1821", "1635", "6901", "6322", "6348", "9067", "8875", "1556", "3971", "6515", "6790", "7297", "1894", "1092", "8329", "5721", "8733", "5004", "5704", "2879", "8675", "9968", "5215", "7443", "7951", "881", "2213", "8967", "7987", "152", "2677", "3336", "4242", "3348", "6453", "5467", "9431", "6937", "8022", "75", "1383", "9818", "9921", "5131", "8225", "8366", "9643", "8613", "2233", "4346", "1412", "1597", "2304", "7356", "836", "9120", "5878", "4826", "706", "5709", "9291", "263", "4869", "256", "7603", "9625", "9174", "7907", "3848", "7057", "3351", "1769", "6587", "4122", "4772", "8935", "2193", "1477", "9755", "6473", "3323", "6289", "4123", "3937", "8305", "9455", "946", "2503", "1939", "5827", "6102", "4105", "701", "1562", "7475", "4049", "1828", "7066", "6839", "2156", "9644", "7665", "7076", "1969", "1456", "6383", "9472", "2783", "2030", "4047", "3072", "9981", "834", "2882", "1051", "4385", "9933", "2473", "4628", "956", "5673", "8459", "7887", "5877", "8416", "7287", "107", "7856", "9792", "4767", "4213", "1701", "6592", "6952", "3602", "5074", "6809", "2722", "4778", "3164", "2940", "5191", "204", "9621", "4700", "5722", "2360", "5617", "3847", "5853", "9923", "4055", "1154", "5923", "8015", "7922", "3366", "1353", "3942", "6481", "7926", "2180", "1654", "9302", "6713", "6563", "5920", "8796", "9269", "2401", "3021", "8088", "8370", "9002", "7372", "9069", "4181", "353", "4323", "1956", "9714", "4804", "7841", "1847", "4859", "4434", "9782", "3822", "2616", "541", "6309", "1582", "8879", "9371", "3396", "2181", "5431", "9556", "3790", "9256", "9211", "5807", "7191", "6459", "6424", "2230", "2499", "7655", "6559", "5573", "883", "6465", "4312", "7876", "3925", "2755", "5637", "9719", "7004", "5016", "1543", "575", "1733", "7558", "4571", "3263", "4483", "5291", "4454", "3438", "789", "8847", "8877", "3260", "1810", "317", "4965", "6177", "6887", "4753", "5724", "7164", "3598", "8186", "6555", "1269", "6715", "4231", "1970", "3311", "9649", "5170", "1971", "6267", "8801", "3099", "7403", "4494", "6293", "6825", "5575", "4032", "2126", "5510", "5912", "7963", "4137", "3258", "117", "6998", "3829", "4651", "7615", "6535", "8616", "2198", "394", "5619", "9418", "1641", "2303", "4366", "1699", "6561", "9459", "1706", "5538", "4994", "1168", "5095", "5417", "736", "4276", "327", "20", "8899", "1101", "2347", "4004", "1743", "4294", "1807", "671", "2909", "7587", "3395", "4871", "8740", "7361", "2487", "3074", "7209", "6385", "9001", "7080", "2586", "7168", "790", "4734", "1151", "2914", "3358", "5405", "1737", "275", "9728", "3872", "4896", "7591", "717", "2544", "2646", "8887", "6301", "7189", "1533", "6356", "9841", "6141", "2902", "5544", "6228", "5574", "1179", "3243", "2580", "1380", "9249", "7846", "8495", "3736", "607", "3085", "6993", "6405", "7391", "5629", "3495", "1606", "34", "3953", "2654", "4750", "4466", "1846", "435", "3147", "2123", "7427", "8389", "5482", "7984", "870", "5823", "8201", "8944", "2576", "7206", "9960", "8110", "2986", "762", "5899", "8323", "4851", "7888", "8272", "5590", "6154", "919", "6156", "6285", "9815", "841", "3259", "794", "6711", "7260", "9615", "6123", "4524", "7529", "767", "4142", "1646", "7859", "630", "9765", "7411", "3417", "4471", "3739", "7879", "4356", "3273", "1845", "5230", "5693", "2938", "4553", "5671", "4305", "2996", "9699", "7205", "329", "2427", "934", "5171", "5317", "2566", "8912", "1484", "8470", "8661", "8271", "5645", "7812", "114", "6617", "1275", "3135", "9555", "5887", "9680", "9943", "796", "3145", "2570", "3962", "2717", "5747", "6512", "6869", "1417", "2619", "9024", "5059", "6367", "3926", "6556", "5596", "9738", "6558", "4224", "6362", "832", "2269", "9784", "6911", "8410", "713", "3607", "4354", "8076", "1780", "7441", "1425", "5456", "1379", "5468", "5803", "7616", "2392", "8450", "4694", "1616", "5003", "5625", "8914", "6543", "4686", "5643", "5699", "805", "3701", "3399", "3397", "4058", "7677", "8228", "9300", "3178", "624", "6161", "4017", "7118", "1645", "1506", "9709", "2733", "9795", "5682", "3577", "7510", "7016", "5742", "237", "5411", "5652", "110", "7464", "8511", "9646", "1668", "1113", "2667", "8025", "9881", "4900", "930", "1109", "4430", "6270", "7806", "7081", "7154", "2301", "6342", "5452", "7688", "7578", "5490", "3671", "8211", "198", "8266", "1995", "9832", "7565", "6408", "2555", "2993", "5382", "5869", "7828", "3030", "250", "4646", "8692", "8449", "4944", "3095", "3043", "681", "6652", "7405", "6468", "1501", "6571", "9475", "1747", "8842", "2497", "2742", "5139", "2179", "1202", "7290", "9116", "5676", "395", "8018", "7654", "969", "5570", "9633", "1225", "6700", "9893", "8427", "2960", "5905", "6485", "2342", "7422", "6861", "5599", "4", "5146", "8198", "5344", "7694", "1377", "911", "9897", "4792", "6798", "1555", "4513", "2552", "1033", "9848", "1436", "4100", "3537", "4919", "9901", "9131", "411", "1818", "4697", "9581", "1140", "4543", "428", "9219", "5054", "8191", "9045", "9926", "3924", "5114", "659", "2778", "6416", "5765", "1726", "5717", "5727", "5141", "1472", "4534", "2917", "4169", "4503", "9970", "1041", "7747", "971", "6333", "3960", "3576", "515", "7685", "8777", "1612", "6921", "4093", "5347", "2059", "7592", "6268", "7053", "2621", "9244", "4265", "9631", "4180", "2045", "7166", "6430", "3657", "9038", "210", "1575", "6128", "2876", "4807", "238", "4384", "1274", "2913", "4892", "8176", "9463", "3369", "5872", "3603", "5786", "9034", "8963", "833", "898", "4918", "7179", "1928", "3934", "615", "4608", "3184", "7109", "4711", "6863", "9617", "5429", "3193", "7258", "8032", "4607", "4639", "817", "4663", "4799", "7697", "1047", "2421", "8990", "9749", "5263", "2192", "8671", "483", "2933", "7602", "2595", "2832", "7705", "8657", "7106", "6773", "9285", "7280", "5445", "3477", "7695", "1177", "6238", "1963", "6794", "5475", "5577", "983", "577", "3867", "5818", "5184", "2300", "8909", "8457", "576", "2489", "1217", "4897", "1657", "104", "8131", "8159", "4771", "5802", "2869", "1363", "1817", "8010", "2500", "7162", "1791", "9867", "5491", "6347", "6038", "2613", "5977", "5524", "5813", "976", "5871", "1593", "3256", "5068", "9805", "5113", "9772", "3078", "1065", "2190", "7455", "3364", "6245", "8113", "9351", "6192", "1145", "1020", "6169", "7577", "3908", "7819", "1356", "4683", "8901", "4353", "2685", "1165", "6796", "7518", "3972", "1628", "4758", "2013", "1205", "4890", "7369", "1652", "8452", "9590", "8971", "7012", "414", "6344", "6287", "9142", "4066", "534", "4863", "6393", "2276", "9296", "8325", "698", "3039", "9290", "8936", "1134", "8574", "2446", "339", "3080", "8134", "4280", "5581", "76", "7159", "3687", "7433", "3961", "3113", "158", "7291", "7868", "2692", "4421", "8135", "4214", "8889", "9858", "2266", "8620", "4199", "8505", "7658", "476", "9822", "4488", "5159", "6828", "5633", "5386", "7123", "718", "6884", "4335", "8116", "1674", "5437", "2688", "6769", "9092", "7543", "627", "5555", "3107", "1673", "4880", "6871", "3543", "2817", "4552", "9307", "3331", "1280", "7250", "6359", "8057", "5665", "9994", "3936", "9448", "4802", "5172", "4574", "36", "424", "1609", "3840", "7548", "778", "8880", "1000", "1762", "8669", "141", "5892", "6165", "6492", "2640", "618", "4878", "8129", "7223", "8961", "5826", "2461", "39", "4654", "8123", "3540", "6222", "69", "314", "218", "3478", "279", "4287", "3000", "3390", "105", "4390", "7567", "5507", "8591", "8414", "8060", "9357", "4229", "1891", "1639", "1886", "8145", "5883", "7440", "958", "1085", "3450", "5968", "4135", "7507", "9961", "4832", "6691", "793", "2930", "4131", "9965", "6544", "8210", "3676", "4713", "166", "5081", "1590", "2137", "3229", "5301", "8189", "2767", "9831", "1098", "1551", "8222", "2141", "74", "9666", "3047", "8383", "6043", "526", "6500", "3277", "4039", "9313", "4953", "3992", "3486", "7146", "3583", "5393", "5260", "7894", "5014", "3778", "9112", "7834", "784", "6865", "4329", "912", "1410", "7195", "3852", "2093", "8820", "9408", "8586", "8064", "9250", "6392", "510", "6227", "7221", "3142", "7059", "8081", "9754", "8315", "2322", "4447", "638", "232", "7415", "1106", "3802", "23", "4508", "3902", "9205", "432", "4568", "4025", "8626", "9890", "8706", "4519", "1716", "5032", "1722", "4191", "7796", "7637", "6547", "5702", "4069", "7085", "3106", "9880", "6900", "3201", "8995", "6946", "544", "2184", "3415", "6426", "4481", "554", "5436", "4183", "4187", "2110", "2432", "5623", "8028", "8542", "5635", "6291", "7365", "299", "8554", "1626", "568", "7920", "6472", "8104", "6782", "749", "821", "9207", "5038", "7605", "2495", "9813", "703", "6021", "6849", "6425", "5308", "3634", "7108", "6063", "3692", "8125", "5387", "6786", "4441", "1318", "1905", "81", "4490", "6603", "1819", "3168", "6403", "5487", "9177", "7039", "7511", "9378", "3786", "4094", "6014", "4611", "1176", "9091", "5118", "5011", "8798", "5255", "6098", "230", "583", "9845", "4634", "3247", "9346", "253", "5276", "858", "9232", "465", "7746", "5632", "3876", "2124", "9902", "968", "5023", "656", "6593", "8895", "2111", "7690", "9538", "3083", "7187", "4220", "3110", "2687", "5158", "1022", "4079", "2599", "3941", "7007", "1745", "5458", "5501", "2364", "7469", "904", "763", "5956", "3542", "1779", "217", "8041", "1959", "8118", "427", "8911", "8252", "9710", "112", "750", "1618", "8058", "979", "4709", "5246", "6632", "9684", "3131", "7408", "3475", "393", "224", "3474", "1693", "4315", "3267", "2029", "9101", "397", "8137", "3393", "7144", "241", "1272", "8219", "9596", "2905", "1839", "962", "6026", "7292", "830", "184", "2008", "9639", "1498", "1265", "6250", "4926", "3209", "6201", "525", "2445", "3012", "2171", "7496", "9537", "9494", "2727", "4448", "5075", "8448", "9338", "9410", "4617", "9059", "7582", "9874", "8929", "9632", "9920", "5958", "5039", "1440", "7315", "450", "1285", "5398", "5485", "5529", "3898", "5050", "9980", "2781", "6855", "7943", "3818", "6108", "4833", "1713", "4786", "422", "1516", "3709", "9974", "2245", "9133", "5233", "6491", "3149", "7849", "8713", "5447", "1193", "1007", "1957", "9305", "4281", "1688", "5053", "8981", "4458", "6251", "4163", "6008", "9097", "5080", "1259", "5236", "1146", "7147", "4796", "44", "1573", "6130", "4464", "3544", "8439", "6002", "4958", "6189", "7257", "173", "7169", "1304", "9564", "5013", "9904", "3560", "147", "6442", "7976", "743", "6867", "5160", "8072", "4599", "977", "4830", "1293", "8676", "9569", "2830", "3244", "91", "1989", "458", "7453", "6519", "6513", "3367", "1301", "2609", "4485", "9718", "9190", "6159", "4326", "2011", "2415", "7289", "3885", "3656", "7531", "7350", "2454", "5197", "8566", "3291", "6943", "844", "5718", "7877", "5843", "8441", "6944", "1757", "8082", "5332", "7889", "5193", "7042", "9268", "8684", "4386", "6019", "2878", "6204", "3161", "3320", "1698", "9591", "8521", "3745", "6802", "7090", "6610", "9330", "3530", "3144", "5847", "2819", "4731", "590", "6613", "8773", "5432", "9424", "3349", "6417", "7224", "6732", "9281", "884", "4923", "7026", "2032", "1210", "3087", "9332", "8359", "1718", "354", "5031", "4597", "4070", "6546", "1372", "1491", "8119", "7848", "159", "8026", "6864", "7670", "8826", "7214", "3565", "6699", "5015", "8678", "7867", "478", "7628", "3056", "1854", "6661", "3357", "8581", "4013", "9502", "5739", "6062", "2740", "8424", "97", "7802", "5269", "6821", "8168", "5224", "2747", "9421", "88", "6170", "6956", "3675", "7988", "2412", "9610", "1029", "734", "68", "2381", "2821", "538", "1408", "9328", "1503", "3346", "5078", "8749", "4272", "7429", "1149", "2543", "22", "8085", "5542", "8970", "7172", "5124", "2916", "7458", "1483", "8677", "8190", "8463", "7151", "5549", "1370", "2440", "7910", "5631", "869", "3335", "7240", "1924", "8039", "3911", "9022", "3407", "7338", "1936", "7018", "7827", "6590", "8656", "8233", "8794", "8122", "7793", "9173", "2857", "251", "3049", "294", "9568", "6612", "994", "7017", "8034", "9272", "6824", "6692", "4304", "4275", "9857", "423", "307", "2310", "2249", "3977", "6633", "4717", "3338", "6883", "2647", "4230", "3287", "2355", "1339", "7517", "1788", "4987", "3245", "5182", "7386", "5065", "3124", "8308", "4967", "9583", "1890", "5753", "7934", "5390", "5024", "3900", "1990", "1476", "4478", "9157", "589", "4677", "3120", "1662", "4195", "1601", "3640", "6974", "4970", "6336", "3566", "1700", "3870", "6346", "3831", "0", "2974", "8762", "7911", "2309", "6365", "5678", "1183", "4074", "2039", "3683", "4883", "8705", "1242", "7630", "2485", "9999", "1967", "1182", "9876", "8091", "3148", "5841", "840", "8665", "1943", "4888", "2209", "1664", "6741", "2807", "3770", "8170", "195", "8378", "4102", "2403", "4895", "7255", "672", "5010", "6166", "6795", "4364", "9997", "3182", "6985", "1613", "9586", "7323", "4705", "8367", "9936", "1200", "6135", "9928", "3373", "6077", "9651", "755", "7068", "6598", "7770", "9155", "3472", "6948", "1048", "1584", "1527", "9179", "6282", "1564", "4456", "9042", "4907", "3231", "9660", "9135", "6120", "3713", "4898", "8856", "2076", "3008", "3907", "7710", "9483", "143", "9842", "3469", "4270", "291", "1253", "1884", "6868", "1256", "5974", "315", "7861", "247", "1027", "4755", "1916", "1619", "1163", "3169", "4949", "6264", "959", "2991", "8361", "1709", "4188", "7449", "4706", "9445", "4016", "4360", "6497", "4380", "9870", "8685", "2763", "9578", "6996", "5661", "4917", "9601", "3059", "1009", "90", "1764", "1196", "5614", "8565", "3288", "1549", "1405", "3647", "1311", "8622", "9258", "1123", "2481", "6368", "9796", "6826", "5585", "7269", "1054", "54", "2669", "8812", "8680", "7709", "4452", "8916", "558", "5688", "6989", "2779", "7923", "8519", "6889", "1231", "4870", "2291", "2297", "9764", "6506", "2907", "5479", "5672", "3552", "7067", "1825", "7104", "2007", "3219", "6389", "7228", "484", "5321", "9225", "1271", "3363", "6431", "3128", "4975", "1263", "8978", "2081", "8555", "3309", "1341", "5066", "9122", "2490", "1081", "2550", "3957", "4794", "2776", "8533", "620", "3969", "2259", "7210", "1856", "4120", "2988", "4517", "2325", "1083", "4947", "7102", "988", "4920", "4885", "1262", "128", "4404", "7301", "3947", "2529", "5806", "8682", "5122", "503", "6872", "5320", "19", "4788", "3094", "7200", "6716", "5833", "1488", "4737", "5060", "5381", "3295", "9287", "4477", "3702", "8056", "6214", "5825", "7970", "5741", "2861", "2019", "6332", "8702", "5740", "8726", "3068", "9315", "2422", "1115", "742", "5051", "9753", "1681", "8573", "6888", "7927", "1973", "7385", "3945", "7126", "1116", "719", "9088", "7024", "8242", "6314", "9098", "1167", "2043", "404", "2898", "8563", "4840", "9277", "5660", "5366", "7807", "2265", "3845", "451", "7044", "8167", "1960", "3119", "9233", "3662", "1399", "4310", "1317", "4014", "3614", "8996", "7500", "7609", "1376", "2459", "9675", "2069", "7148", "2017", "1532", "2166", "5965", "8972", "8943", "5044", "4972", "8215", "551", "967", "2915", "7766", "5187", "5903", "7099", "5133", "7313", "3660", "7858", "439", "337", "3391", "3899", "45", "894", "9785", "3849", "3714", "2715", "2158", "684", "364", "8096", "5136", "5848", "8784", "5860", "4670", "6252", "4873", "4671", "1448", "6898", "7842", "8166", "4991", "5862", "5683", "1508", "67", "7940", "3405", "8688", "1366", "2214", "2697", "2294", "2420", "1434", "605", "9053", "5952", "4656", "9435", "9941", "2091", "9609", "9786", "4238", "6787", "8849", "606", "4086", "4549", "8766", "2696", "4383", "1812", "1287", "6381", "8631", "5941", "4468", "7909", "8114", "5297", "986", "3155", "4472", "8038", "2989", "6284", "4756", "6415", "9707", "6581", "1361", "1766", "9669", "3248", "3162", "3440", "8717", "9814", "6152", "7132", "7745", "6833", "3837", "1838", "4974", "2528", "2786", "2805", "4397", "1765", "9214", "8098", "2494", "7789", "8531", "2997", "27", "944", "795", "9565", "6434", "8983", "7481", "9242", "5092", "6273", "447", "5356", "7000", "8814", "8376", "7093", "8596", "7267", "480", "8528", "2769", "4935", "4775", "3011", "9406", "8226", "2541", "8941", "8594", "9571", "4810", "1126", "3920", "8501", "901", "6976", "2513", "3428", "3575", "8683", "2396", "6174", "1334", "402", "5179", "9806", "940", "4800", "8061", "1320", "257", "1487", "4905", "7303", "3453", "1268", "9304", "7446", "6905", "1927", "6121", "6331", "875", "4658", "1286", "2504", "6286", "8348", "2568", "2558", "2436", "3225", "4620", "6526", "511", "8353", "52", "7439", "7643", "3019", "7919", "7563", "7046", "3890", "3939", "7463", "4500", "9159", "5177", "4691", "811", "7379", "1497", "5099", "3207", "3569", "6147", "1955", "4087", "9270", "5649", "6697", "654", "9036", "2189", "8546", "8468", "2159", "1034", "5238", "8516", "7387", "9368", "3978", "6460", "4479", "829", "3545", "5242", "6106", "474", "6482", "4916", "6448", "1983", "5978", "3986", "3002", "7282", "2860", "2486", "5448", "7854", "4104", "4428", "3820", "4359", "9513", "3643", "4178", "4718", "2738", "3525", "6042", "5442", "1306", "905", "1213", "3632", "9193", "3109", "7534", "7406", "3483", "2197", "2173", "2148", "540", "1450", "1024", "2760", "1653", "6207", "7872", "1975", "1273", "7818", "161", "2964", "9278", "6719", "4745", "2201", "9687", "3329", "164", "470", "3307", "6441", "3152", "2321", "3757", "4685", "1169", "3324", "8183", "3319", "6589", "3197", "5096", "5488", "2075", "9474", "8043", "5894", "2901", "5897", "6091", "1442", "4759", "3932", "8582", "2052", "2598", "6297", "7006", "7896", "4443", "3098", "1105", "2362", "6363", "4703", "3769", "2911", "9825", "120", "7712", "9292", "2256", "1055", "3616", "9075", "9604", "2811", "1711", "7580", "2759", "8748", "8986", "3834", "6528", "9919", "3672", "1071", "340", "3550", "1648", "286", "6144", "3586", "2699", "8444", "2255", "7566", "6750", "6638", "7485", "886", "4861", "2631", "9188", "9108", "6890", "7368", "9503", "7506", "5559", "4402", "2792", "5836", "4993", "5057", "2493", "2665", "9454", "3159", "9253", "1393", "7932", "2612", "3227", "4911", "5058", "4162", "1951", "8264", "438", "1236", "7014", "2106", "2160", "6028", "1158", "1162", "1622", "966", "1067", "3427", "3651", "1038", "9011", "8602", "8212", "7069", "7281", "7641", "3636", "1478", "2579", "2666", "5082", "1553", "2060", "3659", "4997", "7898", "4687", "6155", "358", "7645", "2480", "2912", "4838", "8863", "5103", "5743", "613", "7684", "9107", "1851", "302", "7329", "8405", "9381", "6319", "7460", "9125", "9837", "8164", "2591", "812", "3666", "2200", "3375", "7388", "5109", "9851", "3317", "258", "4674", "4171", "9457", "17", "7306", "876", "7140", "8730", "3732", "9676", "751", "2946", "5117", "2633", "8830", "5101", "5086", "473", "4412", "3398", "416", "601", "5997", "7434", "7704", "5832", "4551", "316", "8229", "5144", "5207", "4226", "8", "2121", "6237", "7958", "5865", "2874", "9134", "7579", "7395", "8273", "5879", "6294", "49", "4547", "7174", "3327", "4609", "8946", "8316", "9982", "4227", "2254", "9835", "7844", "8356", "535", "4114", "2337", "4037", "9086", "931", "1748", "5796", "3617", "187", "8997", "2871", "2885", "5362", "5545", "8493", "4657", "4396", "3884", "3605", "4684", "913", "2140", "9008", "1181", "7549", "8254", "7691", "2063", "3650", "8319", "2056", "4721", "6671", "2005", "1219", "8652", "9119", "9335", "1228", "9794", "3088", "4541", "6735", "500", "8506", "9267", "9579", "2335", "2028", "78", "1012", "6973", "7416", "7438", "1869", "4222", "9541", "5594", "3613", "9481", "5523", "2417", "4461", "2657", "2225", "6396", "1898", "770", "1749", "8453", "8033", "8805", "9701", "2726", "2714", "1197", "7576", "9230", "5685", "5258", "8991", "5583", "1189", "8770", "7767", "7570", "6874", "4704", "3202", "5239", "9467", "1011", "8393", "9670", "1599", "5472", "4660", "5268", "528", "5329", "4459", "5512", "2700", "8335", "5289", "6651", "7354", "5757", "7294", "4161", "660", "4154", "4603", "9409", "7528", "5378", "9025", "8994", "1010", "7936", "3239", "5576", "6131", "5497", "2262", "5140", "8362", "2972", "708", "7870", "5893", "5715", "5005", "5988", "1994", "7128", "359", "3305", "6806", "1211", "9696", "5346", "7652", "9491", "2330", "3943", "4538", "2466", "8761", "2708", "79", "7590", "1235", "7158", "5713", "5270", "6097", "692", "3975", "2375", "9628", "9319", "8299", "3499", "127", "190", "7871", "7176", "7033", "5697", "773", "1435", "6801", "4548", "4827", "7817", "6615", "4537", "6566", "3082", "1243", "4783", "5515", "3189", "215", "4803", "3111", "6307", "259", "1853", "8736", "5218", "1238", "765", "2469", "896", "6073", "5587", "3652", "1198", "508", "867", "3637", "8466", "8126", "8924", "7437", "9864", "351", "2894", "2479", "8188", "296", "2936", "7247", "8556", "4630", "9016", "1542", "1389", "4931", "7087", "7869", "8494", "1895", "4853", "2086", "1475", "3944", "1638", "2590", "8111", "7755", "9913", "7698", "5609", "5593", "6620", "2402", "8534", "3382", "5514", "4148", "7753", "8885", "4746", "1119", "801", "4228", "2472", "4757", "5835", "7008", "5348", "9183", "2272", "5921", "163", "3747", "3125", "4376", "9404", "553", "2344", "2520", "8360", "9724", "5421", "3553", "996", "6662", "1651", "1122", "950", "7978", "5875", "2491", "6640", "2023", "3341", "7212", "8966", "2457", "4836", "9218", "689", "7809", "7672", "7644", "5648", "2424", "7937", "965", "2867", "169", "1030", "7428", "2107", "8013", "3716", "4933", "2824", "3850", "2804", "6852", "1870", "8103", "3886", "7723", "6529", "1423", "12", "6205", "3003", "2766", "8608", "1104", "7585", "3414", "918", "680", "2027", "168", "8714", "3585", "9240", "699", "7607", "9523", "4533", "9940", "6195", "9803", "7707", "429", "5771", "6029", "3694", "2492", "8589", "9561", "8489", "6312", "4936", "3014", "4269", "7502", "132", "2150", "3062", "8933", "2318", "3524", "5469", "2561", "1095", "4530", "9580", "2204", "4117", "5810", "5960", "3257", "4497", "8739", "3873", "8078", "7569", "6055", "2035", "4288", "936", "2522", "2594", "2002", "3654", "4570", "9731", "5027", "420", "6866", "1421", "7447", "6136", "1319", "9759", "8196", "1226", "2350", "6984", "4829", "8615", "7895", "9262", "4726", "1427", "1108", "6429", "852", "9688", "1734", "3891", "1486", "9776", "4593", "5029", "1879", "6457", "5498", "887", "7015", "6600", "7994", "808", "3022", "66", "7682", "8109", "1199", "3362", "3077", "1078", "4205", "8314", "1221", "7364", "5553", "1536", "8897", "8872", "4382", "6163", "7627", "4248", "7622", "462", "4856", "6236", "7542", "9426", "7660", "2682", "243", "9373", "3222", "1014", "2452", "8311", "144", "2215", "7996", "2957", "8993", "9141", "9798", "8605", "6885", "7233", "4664", "8707", "5993", "3730", "4129", "2507", "1823", "9505", "7389", "6540", "1909", "4401", "3995", "9894", "7177", "1178", "8790", "4661", "4940", "7623", "7790", "6337", "8230", "8903", "6977", "6103", "1624", "4817", "5077", "4943", "7756", "2257", "6964", "8342", "4406", "4410", "3777", "1457", "4457", "4504", "5221", "7370", "1090", "8649", "7253", "9355", "2169", "744", "3630", "9195", "1074", "1308", "3976", "6609", "1961", "1040", "9172", "4662", "4487", "2926", "4179", "7972", "8618", "7764", "6265", "4429", "6918", "5505", "2175", "5106", "4097", "4495", "9352", "2639", "5178", "9915", "7515", "8968", "1325", "760", "9916", "5689", "1649", "9608", "5915", "4388", "3394", "8804", "2142", "6915", "8597", "3413", "2732", "309", "4610", "6808", "7965", "2371", "2858", "7426", "5521", "5183", "2130", "8130", "7112", "18", "8485", "6951", "6351", "9055", "4969", "3237", "8560", "2752", "5252", "6075", "9909", "57", "650", "1299", "8848", "9704", "1815", "7122", "8071", "4648", "9829", "7903", "5150", "6139", "8303", "700", "8475", "3253", "4675", "6965", "8870", "9535", "7731", "4978", "2839", "9056", "165", "6276", "1816", "9469", "4040", "4160", "1893", "4101", "3296", "8066", "707", "7862", "7734", "8040", "602", "8426", "7699", "4411", "7571", "9734", "9882", "7957", "3038", "3568", "3557", "4813", "7180", "2701", "5686", "2400", "3548", "5591", "711", "5216", "6896", "6744", "2637", "1514", "3051", "116", "5262", "9592", "6183", "3342", "330", "6693", "9026", "301", "4212", "5950", "3195", "3127", "4427", "8221", "4499", "3188", "4068", "3425", "3993", "3228", "124", "9548", "506", "9029", "1415", "3278", "2068", "4261", "3299", "8021", "2313", "2031", "4259", "6577", "2239", "6789", "7394", "2605", "4374", "5457", "5422", "5566", "3906", "6933", "1947", "9661", "2823", "6230", "5428", "669", "7050", "2900", "3203", "13", "1855", "2315", "860", "7229", "1345", "8795", "95", "2736", "5100", "1690", "6013", "6847", "5361", "3639", "4580", "274", "5264", "5331", "14", "1972", "4147", "374", "8768", "3020", "8177", "4333", "3180", "6858", "4781", "2774", "1585", "2092", "6421", "4749", "9846", "2102", "5091", "4649", "7175", "6466", "5322", "3365", "6104", "8765", "2710", "3859", "8525", "8336", "4779", "188", "2062", "3462", "9202", "2731", "5443", "5639", "4076", "6626", "2990", "7601", "4139", "1934", "4569", "448", "3775", "9215", "6618", "4763", "2512", "945", "4314", "8075", "6495", "9852", "1900", "9019", "455", "8133", "2146", "6050", "1671", "3563", "8578", "7254", "7560", "2308", "9779", "5503", "7741", "388", "9626", "3693", "4110", "9828", "2741", "6180", "2178", "1558", "9081", "8029", "214", "5944", "644", "8269", "6280", "2875", "8792", "9911", "6986", "5561", "9323", "9341", "7550", "8388", "9264", "7466", "4317", "777", "9084", "4998", "2952", "1382", "9540", "9656", "6507", "3810", "9030", "4930", "4886", "5785", "2074", "4358", "6341", "2606", "7218", "4051", "3767", "9668", "9411", "6478", "1741", "8627", "3515", "6846", "5610", "464", "9238", "2077", "7225", "2026", "3658", "2248", "8313", "5061", "2477", "1993", "961", "6244", "8006", "633", "4822", "328", "8030", "181", "5937", "3857", "437", "6687", "9691", "1129", "4052", "82", "8617", "7178", "7913", "863", "3183", "3646", "8869", "8465", "5772", "1938", "6167", "8710", "3460", "8694", "220", "6628", "1479", "3136", "8318", "9990", "2267", "318", "9976", "4563", "7574", "4136", "8630", "4138", "4595", "9964", "5995", "8668", "9636", "2164", "1400", "8529", "1814", "6606", "3901", "6627", "377", "1246", "3489", "9653", "1992", "3446", "3451", "2506", "9301", "593", "4060", "3389", "3784", "4268", "8411", "7181", "4689", "2172", "8759", "319", "2343", "4173", "5592", "1517", "8767", "3574", "929", "498", "517", "2468", "9895", "759", "6259", "4831", "4526", "71", "623", "3121", "1061", "6032", "6471", "737", "4433", "8270", "7916", "501", "1586", "3476", "9623", "7647", "9767", "516", "5839", "8214", "7359", "3160", "1642", "3240", "9861", "7351", "5759", "8758", "1771", "9140", "9021", "4172", "7778", "8979", "2470", "5373", "7981", "8660", "3302", "7480", "1981", "4378", "9476", "8307", "2168", "5352", "4140", "7145", "2944", "5723", "7314", "7285", "3130", "592", "9865", "2049", "3067", "6168", "1093", "7237", "7664", "8526", "2787", "7521", "670", "1811", "3597", "5646", "9983", "6114", "5856", "5987", "1249", "6930", "3546", "5799", "8927", "9360", "1338", "3076", "5703", "2174", "7400", "587", "1260", "7278", "1640", "7702", "2870", "3455", "5509", "7215", "6702", "7960", "6277", "5519", "6557", "5049", "1761", "5504", "7840", "1291", "6339", "347", "3527", "5232", "4239", "444", "3137", "9380", "9725", "1046", "154", "2641", "2608", "4092", "4022", "2296", "910", "9616", "1100", "9900", "8324", "8049", "115", "7738", "9345", "8653", "5217", "5674", "8584", "6501", "9647", "2713", "8951", "3959", "580", "2630", "323", "5959", "8063", "2866", "9280", "2542", "7156", "4904", "6373", "9402", "7991", "7815", "8197", "7917", "3322", "6034", "2114", "4842", "8154", "2070", "7823", "8207", "5638", "4590", "6541", "419", "6243", "5579", "9871", "7883", "603", "4059", "9350", "1502", "4952", "2702", "628", "715", "6669", "9361", "8275", "4475", "140", "3154", "9732", "8406", "6902", "5203", "4688", "9530", "7961", "8918", "6257", "2408", "1439", "1634", "6760", "9746", "9100", "4020", "6096", "7461", "809", "9780", "888", "6938", "8745", "6435", "6505", "8807", "7184", "4067", "4035", "1043", "3200", "4710", "8673", "7522", "2361", "3507", "185", "7624", "4251", "6079", "5192", "981", "7477", "5707", "5030", "6182", "8068", "8420", "8446", "6820", "9064", "2463", "1954", "2048", "77", "7516", "1782", "2795", "7626", "2711", "682", "7533", "9595", "9939", "9673", "8861", "6261", "9169", "442", "7676", "31", "9223", "6962", "3804", "954", "9325", "4740", "7663", "3892", "6486", "9869", "8193", "7725", "5345", "2788", "637", "3915", "1252", "7929", "6516", "16", "8469", "3297", "2748", "4818", "9951", "9013", "4605", "3531", "1056", "4024", "7262", "9527", "8815", "6493", "7598", "8576", "7231", "7912", "9790", "9294", "3927", "8377", "254", "2113", "6624", "7519", "8157", "9773", "9797", "9781", "6817", "1566", "4773", "5423", "1629", "64", "1430", "6775", "6572", "2496", "5403", "4111", "4398", "9750", "5651", "6673", "6509", "8557", "6220", "4217", "8413", "7999", "3742", "6891", "5768", "3340", "5493", "4099", "8083", "3618", "7814", "2397", "7251", "7727", "866", "6647", "3685", "5541", "6067", "906", "7252", "7034", "9947", "8715", "3633", "4422", "1541", "1620", "1035", "7486", "1482", "8150", "8002", "2207", "459", "7719", "8810", "1872", "2743", "9854", "8330", "679", "6292", "5793", "4036", "8240", "7759", "5933", "2753", "3655", "522", "4960", "5335", "878", "4046", "4121", "8147", "597", "1937", "4319", "7952", "3308", "5476", "4588", "7666", "2373", "4752", "9333", "3337", "3740", "6197", "7881", "206", "1298", "3731", "1185", "8607", "2050", "3710", "6047", "3894", "5779", "5046", "3419", "255", "1750", "391", "8695", "6992", "6910", "7771", "5325", "6695", "6853", "205", "3268", "4201", "5628", "6278", "8824", "9178", "5477", "3134", "8334", "7417", "521", "6157", "843", "3674", "9891", "4679", "6039", "4602", "4469", "6562", "9050", "8003", "2939", "7430", "9667", "581", "2157", "3464", "269", "1804", "5363", "7077", "8507", "1355", "4395", "9347", "1125", "4400", "8737", "6648", "7782", "9168", "8202", "721", "2797", "574", "5898", "1131", "2831", "9573", "5866", "8952", "1858", "5017", "2149", "5334", "4141", "2515", "5851", "696", "3100", "7047", "8023", "4894", "355", "1471", "226", "4405", "2306", "1808", "786", "9629", "8268", "5298", "1309", "1362", "3064", "2897", "468", "6569", "2409", "4681", "5426", "7342", "4165", "2356", "4207", "2533", "9222", "8838", "1730", "1931", "9611", "5257", "2935", "3762", "6076", "1485", "1896", "4912", "4988", "5698", "2573", "1388", "2087", "9552", "9652", "1907", "1314", "4624", "7266", "6255", "5154", "5253", "1220", "5982", "2969", "7248", "3904", "2332", "7833", "4594", "9000", "2331", "2236", "2634", "9288", "6851", "2919", "2922", "3306", "4743", "7897", "1715", "4673", "9582", "2072", "4623", "2112", "787", "2864", "903", "6623", "6912", "518", "9339", "7194", "8223", "9165", "2931", "8331", "8570", "5684", "9910", "550", "5754", "6767", "8181", "4592", "4562", "985", "1996", "8462", "3403", "825", "8780", "537", "6511", "2340", "53", "8358", "5597", "5895", "7013", "8149", "1754", "5711", "2314", "5992", "2813", "6953", "5453", "3966", "4860", "9809", "7982", "9104", "9103", "5953", "9918", "9720", "6040", "4257", "1305", "8297", "6931", "3950", "2932", "6187", "8105", "3821", "6583", "1489", "1964", "5885", "3811", "7274", "3725", "9587", "4695", "8722", "5532", "820", "9622", "2751", "9153", "1017", "5608", "588", "3490", "2241", "9200", "8822", "4576", "4554", "5946", "1678", "8878", "3479", "6664", "4720", "7308", "7687", "9896", "6904", "7078", "2980", "3768", "4968", "2465", "6053", "3325", "9340", "8344", "1365", "9484", "1469", "6685", "7661", "9473", "2656", "5957", "879", "5394", "6290", "9054", "4992", "8481", "2442", "6253", "5973", "3105", "5148", "9331", "348", "9048", "5792", "1915", "1374", "999", "3669", "4098", "5861", "6908", "9132", "5701", "6762", "5460", "1237", "9542", "6242", "5245", "5213", "567", "4957", "7101", "5927", "4377", "8289", "2451", "4015", "935", "5522", "5435", "8300", "3032", "1120", "1644", "4159", "7028", "5618", "2510", "9396", "5097", "9466", "9605", "4091", "1908", "6539", "7618", "1617", "8614", "8629", "1729", "8697", "5155", "2186", "8262", "4255", "1545", "7540", "1802", "9657", "9217", "9061", "6202", "3449", "3334", "6119", "9948", "4449", "2458", "647", "7073", "2822", "4696", "5557", "1493", "695", "463", "5162", "9147", "8094", "2855", "7019", "6148", "9584", "889", "6422", "8115", "3096", "6200", "5280", "4632", "1375", "5341", "3503", "8514", "5367", "1860", "2116", "3116", "7043", "9760", "9181", "5908", "4761", "5041", "3606", "5986", "6324", "4337", "8742", "9166", "4784", "3289", "2302", "8095", "8239", "1316", "631", "2251", "932", "4760", "598", "7143", "6433", "3754", "1740", "3025", "4653", "2055", "4444", "1650", "3456", "322", "9550", "6707", "6140", "2444", "4158", "1834", "9487", "697", "4072", "4267", "1245", "6401", "8005", "1082", "9403", "5412", "6980", "4023", "928", "9009", "7838", "3833", "9422", "2761", "9234", "8757", "9276", "4348", "5369", "9988", "7063", "8027", "895", "3298", "9386", "899", "5828", "1790", "1574", "2526", "5002", "1224", "4258", "2556", "5755", "5306", "2531", "1368", "8949", "3471", "5360", "1614", "3360", "5962", "2405", "6391", "7865", "8065", "9547", "842", "2706", "3138", "960", "6882", "3513", "5565", "7805", "3444", "8862", "1467", "6087", "1143", "5890", "2419", "1919", "5654", "8651", "4008", "9074", "2720", "5932", "9507", "3454", "5303", "1897", "3473", "9377", "407", "4351", "9705", "3141", "3869", "494", "4170", "2242", "4379", "9344", "6643", "3013", "1658", "5500", "7554", "7692", "768", "9210", "1347", "1865", "7944", "2482", "7758", "8395", "5636", "987", "5530", "9", "5471", "4857", "5640", "7383", "2177", "5151", "2845", "1127", "7199", "8213", "5486", "3696", "9028", "666", "2663", "7532", "8216", "3265", "4676", "9559", "1944", "8937", "4011", "7683", "5571", "5410", "2359", "2275", "1998", "4474", "94", "1950", "5634", "3058", "2135", "8811", "1803", "2260", "4976", "8087", "3036", "5088", "1446", "5764", "5474", "1720", "7503", "7022", "2109", "4655", "2066", "306", "4198", "9545", "8431", "2090", "5572", "573", "1292", "8479", "781", "9872", "639", "7375", "2671", "848", "8195", "8559", "2103", "7192", "8894", "7360", "3272", "6683", "3562", "8674", "2694", "5288", "3715", "8000", "1418", "5535", "2349", "4640", "5349", "1057", "6772", "5064", "1330", "9804", "7025", "5250", "5695", "8575", "8593", "373", "3749", "5658", "3737", "3179", "943", "752", "9807", "8438", "4742", "9106", "1672", "4283", "5970", "5089", "1026", "7945", "8867", "5710", "5737", "5598", "957", "7304", "8152", "1460", "5231", "7241", "2549", "4962", "6799", "5305", "2211", "984", "4573", "3516", "7706", "4544", "5881", "5198", "4108", "6680", "2293", "963", "2625", "7138", "212", "9648", "9456", "9415", "9208", "5696", "5552", "5379", "4002", "9497", "6747", "914", "7432", "7875", "8301", "1535", "9216", "2850", "7959", "176", "7349", "398", "4075", "8520", "9163", "5582", "8640", "2020", "6611", "1150", "2589", "2410", "9154", "6382", "7208", "5756", "9839", "7995", "8086", "2771", "2895", "8312", "1786", "4511", "1569", "162", "766", "7613", "3494", "8156", "1591", "3157", "4256", "1600", "5147", "7115", "8100", "7137", "2655", "4509", "5019", "6913", "3620", "1518", "9295", "5461", "1831", "3432", "2348", "2746", "3664", "3949", "1391", "1504", "9949", "802", "774", "1524", "5797", "9511", "1583", "2884", "3108", "6714", "4825", "8637", "9416", "7900", "7713", "9478", "9512", "3967", "40", "3832", "4126", "1874", "693", "513", "4979", "6876", "178", "7414", "7775", "7149", "5983", "6017", "2258", "4184", "7971", "5143", "7946", "8747", "3283", "7671", "519", "7843", "1758", "5047", "974", "9562", "7914", "2801", "9989", "2601", "1976", "8132", "5984", "1784", "1455", "7425", "5384", "2775", "5896", "3194", "8956", "7891", "8910", "6994", "5838", "245", "9226", "311", "5020", "6387", "7501", "396", "6134", "8432", "6378", "3254", "4324", "9884", "1563", "9248", "469", "2398", "3985", "7130", "2384", "108", "6967", "3753", "1770", "2617", "2125", "7985", "4206", "9471", "3589", "4362", "7352", "792", "2648", "4560", "2476", "882", "5687", "5578", "8635", "3826", "6777", "6856", "2284", "7983", "1660", "1682", "3027", "2034", "9184", "7133", "8835", "9689", "4365", "3313", "9282", "7621", "9978", "5980", "1130", "1088", "9907", "2244", "2288", "5273", "9014", "9156", "1966", "5225", "6919", "1702", "2061", "5135", "8349", "6218", "7330", "3165", "9811", "55", "3673", "5205", "2518", "7752", "2853", "4151", "6690", "1875", "7163", "880", "8754", "980", "9522", "1276", "2934", "5882", "8106", "728", "3929", "6071", "5767", "9768", "2501", "1794", "9090", "6742", "665", "1437", "2210", "1058", "6388", "1385", "4516", "1787", "5072", "371", "8926", "3226", "2679", "5659", "9574", "7110", "6209", "776", "4572", "3420", "4336", "3129", "9577", "6892", "4699", "1602", "3990", "9789", "7436", "3079"]]} \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist.config b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist.config new file mode 100644 index 0000000..44c29d8 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/mnist.config @@ -0,0 +1,13 @@ +{ + "scheduler": ["str", "cos"], + "eta_min" : ["float", "0.0"], + "epochs" : ["int", "50"], + "warmup" : ["int", "0"], + "optim" : ["str", "SGD"], + "LR" : ["float", "0.1"], + "decay" : ["float", "0.0005"], + "momentum" : ["float", "0.9"], + "nesterov" : ["bool", "1"], + "criterion": ["str", "Softmax"], + "batch_size": ["int", "256"] +} diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets-split.txt b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets-split.txt new file mode 100644 index 0000000..9fe861c --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets-split.txt @@ -0,0 +1 @@ +{"train": ["int", ["5470", "3295", "4905", "5837", "4853", "3074", "2927", "2081", "4554", "4171", "4888", "4802", "3257", "4932", "558", "468", "2944", "1332", "97", "5759", "2293", "2086", "3518", "2031", "5505", "4948", "1182", "4418", "111", "2149", "4166", "1451", "1673", "2646", "5307", "705", "3374", "2090", "3914", "1413", "1350", "5066", "5212", "3148", "4543", "1970", "943", "1511", "5125", "1203", "2822", "937", "2529", "5970", "3879", "4231", "2104", "2175", "2954", "5919", "4658", "3601", "5774", "418", "2515", "4338", "6227", "5805", "488", "2857", "1600", "3758", "420", "3762", "761", "5880", "1864", "3248", "1520", "796", "2403", "3784", "1009", "635", "2148", "4743", "2263", "299", "3189", "2511", "1870", "1467", "2087", "5118", "4728", "923", "127", "696", "5659", "3949", "6007", "5180", "5817", "4060", "2065", "1190", "5101", "4391", "1780", "2919", "5463", "2766", "3785", "5756", "36", "5860", "344", "868", "3674", "3508", "4984", "3739", "314", "2539", "3145", "5249", "720", "5055", "2648", "676", "3805", "1119", "5967", "814", "599", "4145", "5751", "5521", "3542", "5926", "144", "1898", "4925", "4075", "2962", "2238", "4487", "1766", "5017", "3534", "3401", "463", "3579", "663", "1471", "5796", "2302", "5883", "4329", "5778", "1821", "5742", "6", "1393", "5343", "4406", "3957", "860", "703", "999", "3041", "4400", "1476", "3721", "1307", "4813", "4445", "5409", "674", "5794", "1352", "2902", "1259", "4427", "5995", "5615", "3494", "2685", "2466", "5811", "3848", "916", "4548", "4141", "1293", "2963", "3070", "5625", "5223", "4011", "2726", "3444", "1681", "4700", "5858", "2122", "4398", "2817", "4987", "636", "4721", "5818", "685", "409", "1168", "4557", "3884", "3013", "4268", "293", "2045", "4525", "5730", "92", "839", "3440", "5985", "5681", "4408", "4250", "1670", "2953", "1528", "5303", "1254", "25", "2340", "3889", "4881", "3268", "4111", "3025", "3027", "3997", "2725", "2564", "3701", "1233", "1058", "2152", "6179", "659", "540", "5110", "592", "1154", "3525", "266", "6024", "638", "4381", "1811", "2814", "4529", "2199", "2794", "4249", "3535", "3635", "1191", "6234", "1491", "5677", "5384", "1441", "3612", "1268", "4127", "124", "5953", "4958", "2774", "4819", "821", "5386", "3326", "3180", "4148", "597", "4863", "4074", "1850", "573", "3970", "3794", "3611", "3962", "4794", "5142", "70", "54", "3920", "4118", "3092", "3215", "3091", "4014", "5575", "2169", "4892", "2974", "1628", "3331", "835", "1577", "5389", "1440", "773", "1575", "688", "5666", "2051", "5120", "44", "1704", "3956", "4530", "2231", "4770", "5457", "472", "901", "5261", "1822", "3507", "390", "5515", "5297", "5918", "629", "5693", "3850", "1019", "4327", "4120", "5026", "1938", "4895", "4506", "4624", "979", "3621", "2635", "1668", "2694", "5565", "6213", "115", "2578", "2061", "5595", "990", "1700", "2483", "4002", "5525", "4220", "3050", "5993", "1430", "6272", "5930", "3124", "2889", "1193", "2076", "933", "1599", "3244", "3280", "2385", "5958", "1640", "3436", "3652", "3839", "2206", "1614", "4562", "2958", "1513", "5912", "4670", "2760", "3355", "5861", "1849", "388", "2147", "738", "590", "4045", "3207", "5671", "2421", "227", "5450", "5903", "5029", "1942", "4135", "6202", "283", "5510", "3551", "526", "2830", "3430", "1897", "4540", "4980", "4489", "2159", "1998", "1030", "1038", "3143", "5831", "5700", "3691", "2443", "5834", "6261", "1349", "1696", "4221", "4224", "1286", "5074", "3161", "4542", "2003", "5310", "2353", "4370", "3054", "2304", "4512", "1627", "4366", "3834", "1275", "5552", "1691", "4776", "4649", "737", "5405", "3277", "137", "116", "5488", "3220", "4481", "2769", "464", "3582", "1241", "1358", "3172", "4200", "433", "2956", "5884", "2134", "2803", "3622", "1016", "1195", "1370", "1063", "2729", "5863", "5495", "4595", "4827", "1956", "1664", "6057", "4281", "2575", "1025", "4125", "5037", "5763", "4996", "5452", "3577", "1380", "4898", "1946", "4488", "1969", "4063", "2856", "1622", "3782", "29", "1889", "1887", "1319", "3638", "2341", "2358", "3445", "5094", "6004", "4638", "2790", "1013", "4300", "2038", "989", "5197", "3442", "1610", "1420", "2239", "3883", "159", "914", "5813", "4509", "3361", "2546", "166", "98", "3188", "202", "1947", "415", "1591", "907", "4834", "2367", "2210", "4968", "2492", "5185", "3989", "5999", "5787", "661", "3866", "2359", "2936", "533", "4399", "3425", "3122", "82", "3709", "1959", "6136", "1164", "2226", "6131", "5028", "2557", "3645", "3669", "5155", "2113", "785", "809", "5873", "717", "2510", "5103", "147", "3446", "5732", "442", "1741", "4998", "2029", "2062", "2246", "1076", "690", "6062", "5088", "1853", "2544", "1201", "3323", "306", "2708", "2609", "459", "5679", "985", "4211", "2819", "5702", "527", "3740", "3514", "3297", "2217", "3099", "755", "4692", "4151", "1206", "3648", "5870", "1472", "1554", "683", "4566", "1812", "757", "6170", "1302", "3922", "1156", "6098", "1940", "2850", "3998", "2335", "5444", "1958", "3196", "5451", "5506", "1161", "864", "677", "4337", "2472", "5825", "4493", "4228", "1461", "1125", "4769", "698", "3702", "190", "4126", "4096", "5601", "6183", "4628", "4349", "5327", "3386", "151", "1361", "5215", "4038", "4685", "2964", "3811", "4029", "3906", "4033", "5964", "4947", "5331", "1772", "3472", "3166", "5240", "5098", "992", "2350", "1751", "1605", "300", "5181", "2495", "2015", "5676", "4515", "3861", "5040", "1065", "3135", "4150", "6224", "2080", "5925", "1540", "5388", "2861", "274", "1483", "2185", "2396", "1062", "1978", "2100", "4313", "601", "4590", "1296", "5724", "2596", "72", "4199", "5224", "45", "325", "5899", "4901", "1953", "1903", "2739", "3214", "6109", "1545", "3787", "3926", "2487", "3130", "5042", "1086", "1235", "3917", "6138", "3409", "4927", "243", "5001", "831", "892", "998", "6011", "2221", "2227", "4739", "1636", "5067", "1239", "2281", "3427", "2990", "5537", "5626", "5061", "2489", "1138", "3637", "3617", "26", "6235", "3675", "3324", "2129", "550", "5954", "832", "2566", "2541", "4967", "253", "5853", "225", "1725", "3790", "4640", "2410", "3126", "2257", "3845", "5684", "3349", "3538", "2617", "3605", "2179", "2338", "3217", "3817", "4724", "74", "5871", "3100", "2662", "1059", "2981", "5348", "5847", "375", "357", "621", "3060", "2296", "4706", "6243", "2615", "42", "4876", "4009", "21", "4433", "898", "2016", "4461", "3627", "585", "187", "4142", "4326", "4379", "4401", "4444", "3363", "3987", "4783", "980", "2204", "2606", "4865", "1612", "3424", "5797", "4389", "4716", "3833", "4133", "5056", "2298", "6071", "5781", "780", "3984", "5342", "4080", "4431", "3947", "4662", "3644", "5141", "1899", "4950", "3985", "982", "4851", "4762", "80", "4130", "317", "904", "4866", "647", "3213", "5167", "4653", "3197", "1922", "802", "4701", "3628", "1720", "2327", "2446", "1316", "2659", "2137", "4752", "877", "499", "6049", "2042", "1951", "4791", "4287", "2207", "3405", "5358", "31", "2317", "201", "3813", "5578", "2616", "6002", "73", "4615", "3403", "2422", "632", "354", "4363", "4815", "1265", "939", "2674", "3823", "4880", "410", "4889", "2167", "1641", "4714", "4465", "5572", "3808", "4321", "4594", "362", "3362", "5300", "3774", "1738", "2740", "3314", "3294", "4351", "826", "4255", "4633", "4081", "3066", "5465", "2424", "897", "2570", "4490", "3202", "2952", "3804", "5828", "874", "5403", "5849", "2286", "55", "3413", "4912", "5504", "6043", "691", "3036", "1036", "3102", "3452", "5000", "1072", "3140", "5960", "2225", "657", "942", "4157", "103", "4235", "6029", "1862", "27", "434", "565", "5063", "1633", "6190", "5593", "5865", "4687", "1100", "936", "2827", "2554", "4959", "867", "4340", "2121", "1624", "2130", "1192", "4962", "2940", "126", "2468", "5283", "4858", "4972", "30", "4191", "2885", "3059", "2143", "4521", "234", "4040", "173", "113", "3098", "4718", "5216", "4215", "1395", "1331", "2097", "5974", "5966", "2505", "4003", "2145", "3227", "3742", "2501", "3654", "5165", "2146", "3653", "1536", "2111", "2888", "4513", "1151", "1043", "2567", "6099", "2890", "5337", "2585", "4568", "539", "5630", "1697", "3569", "1021", "5673", "2680", "5211", "5009", "6155", "371", "381", "4829", "5462", "2916", "1745", "2117", "5226", "3418", "3733", "2579", "405", "6026", "4571", "5377", "1051", "1915", "2069", "5560", "5900", "3952", "2628", "4707", "156", "1436", "2862", "4729", "2268", "5425", "3506", "2131", "5603", "3640", "1855", "6199", "370", "5989", "799", "146", "5178", "2759", "2552", "3504", "5720", "754", "947", "6271", "2083", "5100", "4732", "2258", "425", "6005", "3337", "1368", "2054", "1755", "6186", "5020", "1066", "5421", "2088", "6121", "2675", "4143", "4119", "5158", "2319", "218", "1158", "4808", "6111", "4066", "4476", "5675", "2005", "3937", "4223", "5780", "4989", "1635", "200", "2096", "5116", "941", "5400", "1541", "5963", "3152", "4688", "2959", "2677", "1188", "1481", "3936", "3888", "2451", "578", "340", "3259", "1389", "4335", "5201", "191", "3581", "3307", "2821", "4887", "948", "338", "4471", "4360", "1568", "1465", "1570", "3792", "3222", "2299", "1684", "1717", "1364", "580", "5168", "4092", "3626", "6197", "4862", "5516", "5247", "5982", "5317", "700", "5013", "944", "6178", "3744", "2048", "4414", "5186", "1199", "5915", "1762", "2698", "648", "862", "1468", "2623", "4930", "2743", "4267", "3340", "5394", "3613", "3513", "3296", "100", "2997", "5309", "4916", "1163", "1764", "2871", "719", "6228", "4593", "5422", "1226", "1234", "2736", "3316", "6000", "2840", "339", "1310", "3435", "4396", "2809", "5585", "1868", "4833", "2587", "321", "4759", "1522", "4680", "1455", "392", "5461", "5743", "1651", "4173", "5150", "3106", "1715", "4008", "2154", "5766", "4956", "1113", "5866", "1480", "4364", "63", "5391", "3779", "3343", "1620", "477", "3667", "3049", "2351", "4270", "59", "1843", "1505", "4879", "559", "2530", "5447", "3870", "1798", "1001", "724", "5779", "4085", "4971", "4735", "5210", "6267", "739", "3821", "5217", "4484", "1228", "4945", "1706", "3288", "205", "1828", "4831", "1960", "104", "1464", "5412", "1689", "402", "858", "4785", "4755", "6162", "5005", "2918", "2670", "2205", "3094", "5687", "4944", "1576", "625", "1558", "4605", "2756", "3862", "236", "6077", "1816", "3865", "1795", "1049", "1584", "1311", "1068", "5613", "4302", "2481", "2690", "3651", "949", "1997", "2613", "1117", "4323", "4606", "6221", "6252", "4645", "3778", "5407", "4761", "2722", "4782", "153", "2881", "3875", "675", "2259", "3073", "4434", "257", "5367", "6053", "624", "5864", "5646", "1746", "1957", "3919", "2793", "2277", "3897", "1323", "2714", "5608", "2333", "4276", "6113", "3006", "5807", "5845", "2562", "5890", "546", "78", "4306", "6176", "2542", "3372", "813", "910", "2270", "766", "3142", "270", "1503", "1433", "260", "3081", "3284", "1734", "3370", "1968", "259", "3305", "1242", "5735", "623", "95", "1507", "5725", "2520", "885", "4810", "4144", "4138", "326", "1243", "1469", "3749", "714", "48", "5177", "1775", "4188", "2044", "6058", "6079", "4778", "178", "1060", "3672", "562", "3965", "1655", "1514", "4884", "2397", "5236", "541", "3015", "5584", "5667", "4600", "896", "2110", "870", "3429", "2671", "3771", "5653", "16", "2658", "2309", "5614", "64", "1758", "2604", "5710", "5769", "5726", "563", "66", "4678", "4", "84", "2297", "404", "6204", "2892", "5419", "3151", "1305", "5219", "1427", "3988", "2930", "743", "2678", "5547", "3910", "4024", "3961", "2208", "5788", "4665", "2688", "3887", "4061", "1609", "5571", "2203", "437", "2176", "1621", "732", "5932", "1753", "3539", "455", "38", "1925", "1962", "3002", "2951", "4072", "6245", "1179", "1824", "3206", "2398", "3707", "5889", "4576", "1866", "3131", "4404", "1835", "2314", "5301", "5152", "3457", "2354", "861", "3747", "5888", "3321", "3146", "2526", "4502", "3365", "2265", "162", "4565", "4870", "5904", "5640", "3589", "3315", "1396", "3505", "2034", "367", "4129", "1220", "903", "5938", "1174", "4098", "1934", "5800", "3088", "745", "4570", "542", "1478", "3512", "6237", "5765", "5109", "993", "4044", "2727", "2713", "4286", "4271", "5475", "4043", "1589", "6276", "4918", "2279", "905", "681", "5190", "1539", "4459", "5852", "2377", "5174", "1909", "5382", "3097", "4312", "4754", "2710", "4395", "4671", "5558", "3058", "1761", "2877", "5044", "43", "3468", "4993", "422", "3996", "2164", "5652", "3407", "3274", "4637", "452", "5840", "5976", "3312", "2284", "4382", "4377", "2195", "2506", "1421", "5582", "3878", "3874", "238", "2521", "1693", "4629", "246", "1728", "5869", "298", "1543", "4773", "628", "3369", "5198", "139", "279", "5434", "6142", "2538", "804", "2875", "5140", "5551", "1726", "5321", "4832", "3385", "1238", "3591", "4410", "1927", "4602", "591", "1806", "4334", "1719", "5791", "1041", "1475", "6144", "4897", "765", "1883", "520", "3855", "1407", "125", "4609", "4786", "3441", "871", "3940", "2128", "2209", "2342", "2339", "5375", "2448", "795", "2574", "3944", "4064", "5996", "2189", "3428", "1410", "5529", "649", "4644", "311", "3698", "4805", "5986", "1653", "3553", "3335", "5526", "4052", "5723", "3633", "102", "5576", "5620", "5191", "6019", "612", "1532", "775", "1649", "2150", "3354", "250", "6177", "4443", "6120", "4019", "5822", "1157", "1419", "846", "4669", "5489", "4147", "2716", "704", "5146", "3459", "5833", "4291", "2414", "5749", "2020", "3192", "447", "5549", "4068", "6038", "430", "3796", "4868", "2555", "529", "2565", "5025", "4903", "3000", "954", "4763", "2757", "1081", "1074", "379", "693", "5359", "1022", "917", "1162", "6021", "3209", "4793", "3991", "1439", "3225", "5263", "4451", "5076", "951", "6193", "3825", "768", "5494", "4713", "6257", "6246", "1053", "851", "4620", "3903", "4768", "4070", "1851", "2158", "3011", "3931", "5654", "4028", "295", "2960", "2928", "4386", "5034", "1381", "1124", "1040", "4664", "2455", "3695", "2824", "1677", "6198", "5973", "188", "1132", "3798", "3990", "2135", "240", "1385", "2047", "2366", "1848", "5115", "5468", "6226", "2440", "2106", "5347", "1121", "2050", "4203", "679", "6274", "1109", "667", "1340", "5349", "3971", "6080", "5032", "593", "2282", "2305", "1524", "1184", "1087", "1460", "3267", "1549", "840", "5123", "2893", "220", "3298", "2464", "6241", "1047", "4486", "605", "3880", "5862", "181", "350", "4727", "511", "3696", "2166", "382", "5099", "2545", "4842", "2994", "3241", "3461", "3147", "5192", "2412", "3176", "2222", "3356", "2126", "1000", "2493", "3641", "4904", "1874", "4042", "400", "5939", "957", "1200", "1329", "5808", "4877", "5823", "4243", "930", "510", "6059", "5438", "2836", "5568", "1644", "5517", "3397", "3520", "5624", "3711", "4198", "2408", "2860", "5387", "3270", "2039", "1886", "2837", "6174", "622", "4303", "3484", "5581", "1637", "5597", "6157", "1091", "3023", "1784", "931", "4923", "3680", "504", "805", "4523", "853", "5775", "1878", "5278", "5207", "2858", "3421", "2815", "3847", "1453", "1373", "1312", "5637", "3051", "2838", "2883", "3710", "4574", "5225", "2975", "1985", "2665", "5162", "4090", "5314", "2549", "668", "2343", "3201", "454", "4561", "2219", "2630", "3114", "4405", "4146", "5143", "2816", "5476", "3198", "5277", "4635", "1744", "3448", "849", "6030", "3153", "6060", "5698", "5981", "5812", "4558", "2977", "5994", "3760", "3482", "2859", "3493", "2491", "4964", "5166", "4841", "2389", "2656", "1993", "4608", "4981", "5784", "4647", "3921", "4387", "1271", "4314", "5455", "5798", "1819", "5752", "5496", "2779", "342", "1710", "4599", "3458", "6075", "3986", "3044", "1935", "1473", "5621", "5122", "5738", "453", "4578", "5906", "5439", "4202", "2676", "549", "216", "6047", "5232", "1071", "3563", "1177", "2519", "2469", "1587", "4290", "1403", "5334", "2571", "733", "4137", "3925", "2095", "3830", "3224", "1564", "2324", "2654", "5202", "3432", "3832", "1891", "1562", "3464", "1426", "2672", "5737", "5728", "2750", "5754", "5956", "2832", "4941", "2693", "5031", "3752", "725", "4711", "2056", "7", "4910", "589", "2660", "996", "4544", "5472", "983", "2948", "4965", "3105", "4859", "5896", "2568", "470", "3026", "2504", "5977", "2190", "329", "2496", "4591", "3797", "4006", "1366", "150", "1977", "5703", "2770", "2387", "3586", "5762", "1176", "303", "1723", "3930", "4260", "2313", "478", "1551", "4994", "1255", "3113", "806", "310", "1017", "5440", "1921", "4547", "4726", "3226", "2349", "3584", "182", "3831", "4766", "2882", "1052", "2433", "4757", "662", "749", "4677", "1140", "2807", "3069", "2650", "2943", "2618", "4811", "5821", "5479", "4113", "2910", "4828", "493", "4261", "1793", "4517", "4882", "93", "3046", "2734", "204", "1836", "4421", "3327", "4592", "5338", "1024", "1388", "1248", "5411", "1122", "2023", "5062", "2488", "6096", "4083", "5801", "3496", "3101", "673", "1008", "3485", "3089", "5544", "1929", "5642", "1044", "927", "2434", "2639", "1077", "3527", "1894", "3184", "3045", "5627", "643", "5473", "1042", "2778", "3235", "2183", "2261", "1263", "1357", "5508", "5147", "3892", "3588", "3713", "2467", "1965", "5090", "518", "1692", "1033", "2089", "5616", "5454", "1508", "4588", "1906", "4455", "2507", "2573", "1134", "587", "3923", "2782", "4656", "521", "5096", "2887", "4942", "3818", "3247", "716", "4843", "6106", "2292", "1688", "2776", "4974", "1773", "4504", "6118", "1774", "2283", "2115", "3660", "1205", "6129", "3982", "2328", "1674", "1542", "5672", "1814", "3076", "3755", "1786", "751", "4510", "3239", "4738", "3406", "99", "4214", "4020", "561", "5628", "3974", "3604", "2561", "2055", "2010", "3994", "2915", "2805", "4426", "921", "1770", "364", "2684", "5091", "3783", "2524", "1295", "5481", "224", "4309", "607", "6045", "5015", "2153", "4385", "1447", "4573", "4394", "3107", "710", "532", "4478", "6217", "4031", "1569", "976", "913", "5302", "5325", "4368", "633", "3171", "2360", "2825", "3249", "2437", "3858", "233", "2717", "5707", "3631", "1933", "239", "5951", "3104", "3705", "3756", "5287", "5097", "4035", "5573", "5187", "5329", "801", "604", "1274", "4409", "294", "6009", "4960", "1262", "1550", "5874", "2355", "4238", "4084", "5638", "4496", "3999", "1114", "1863", "1180", "4279", "2933", "1705", "4623", "5990", "4099", "5718", "189", "3566", "2718", "5649", "2172", "414", "4598", "5433", "2663", "3541", "3859", "1318", "263", "5771", "1297", "3683", "3829", "4280", "5945", "5520", "2586", "4100", "3467", "1988", "6187", "3688", "746", "5456", "5816", "6249", "2160", "5071", "4328", "1703", "1172", "2066", "2267", "5156", "929", "5077", "3978", "4940", "1917", "2834", "4750", "5949", "1315", "978", "6211", "348", "4392", "4369", "5376", "2651", "5295", "5235", "2699", "4149", "5802", "5179", "2407", "5172", "3992", "1046", "3193", "462", "2748", "4450", "5786", "2976", "1525", "1752", "5380", "6145", "1930", "4041", "2450", "5139", "5969", "5480", "2771", "2486", "2368", "709", "3064", "918", "4466", "2732", "2937", "1185", "4077", "5806", "2196", "1144", "4436", "3918", "2804", "1639", "5312", "114", "3420", "2929", "2234", "5514", "2187", "2027", "2133", "5708", "3911", "6023", "940", "4089", "651", "2285", "3716", "269", "1787", "1208", "6256", "1827", "3835", "1990", "3938", "5316", "4372", "1749", "3689", "615", "6192", "333", "5161", "3826", "2046", "1377", "1792", "3057", "2595", "3560", "4936", "2597", "3765", "2631", "5144", "3530", "646", "830", "1098", "963", "2441", "5285", "6260", "6242", "1061", "3080", "4425", "2085", "479", "2686", "4473", "1485", "5057", "1210", "782", "1881", "1777", "1678", "1722", "3292", "4204", "1135", "2823", "1450", "6159", "4422", "1067", "6126", "5136", "484", "5133", "3399", "5795", "5021", "6090", "5844", "2608", "1209", "2428", "419", "4241", "1724", "2006", "767", "1495", "304", "971", "1865", "1351", "1701", "5004", "5819", "1779", "5662", "4966", "4983", "2073", "1632", "3009", "4115", "3384", "1981", "5273", "4325", "1084", "5135", "1645", "4177", "1443", "4541", "5522", "3692", "4549", "5978", "3253", "3028", "1045", "413", "3769", "1129", "5559", "3264", "818", "2711", "4693", "4536", "2909", "4470", "3687", "1007", "1056", "1593", "3676", "3650", "5016", "2839", "4684", "869", "4109", "133", "3052", "231", "5931", "6231", "3396", "3574", "4723", "4262", "4269", "3136", "3328", "320", "3170", "6225", "1607", "120", "5997", "1404", "9", "644", "3789", "3546", "2093", "3913", "3629", "1841", "4816", "701", "794", "4176", "2071", "1488", "1781", "4322", "3730", "6072", "106", "3898", "2109", "2318", "3250", "1687", "1794", "6212", "1813", "4482", "2640", "6116", "1136", "2738", "332", "5041", "4630", "5757", "1246", "3477", "1867", "47", "5891", "1538", "2444", "5965", "974", "984", "5602", "5129", "3393", "256", "431", "1698", "4694", "3634", "3993", "5230", "1070", "4467", "439", "1531", "3075", "5711", "1735", "6191", "4209", "3873", "3609", "3981", "4824", "1535", "5936", "617", "1252", "5318", "2118", "5089", "5199", "1731", "1994", "1604", "3275", "1418", "4850", "1290", "4437", "1750", "1003", "5014", "4617", "5979", "4767", "5848", "4538", "4737", "5789", "226", "3178", "0", "1283", "678", "3243", "2569", "783", "5138", "6041", "524", "967", "1996", "894", "2768", "3665", "571", "3272", "2478", "6018", "4373", "3281", "616", "2155", "2213", "5591", "6196", "5617", "4336", "6201", "1660", "1606", "841", "1085", "5634", "1225", "2924", "6151", "4758", "1890", "2745", "6083", "3599", "79", "5785", "2528", "5750", "3561", "2236", "3332", "28", "880", "5937", "1737", "4055", "730", "2141", "2657", "4218", "1877", "5747", "2865", "1187", "4000", "316", "4639", "5543", "1181", "2465", "1401", "950", "3886", "1207", "3500", "2525", "3483", "3671", "4095", "1115", "5491", "4627", "938", "2322", "3585", "6054", "845", "5048", "2593", "1739", "4417", "2848", "2028", "838", "6175", "6110", "4854", "4519", "3545", "2499", "772", "4339", "83", "2142", "2957", "2383", "4062", "5922", "2162", "1064", "53", "568", "3748", "6188", "5609", "3775", "3110", "5396", "5362", "600", "1309", "2242", "3437", "5308", "6219", "5368", "491", "6040", "4208", "6001", "3411", "5184", "4520", "5332", "5692", "1414", "3183", "4681", "3763", "3003", "3182", "1264", "6025", "952", "1884", "174", "1376", "4781", "890", "3720", "2078", "4826", "134", "1278", "4265", "2178", "760", "4806", "1398", "4163", "4812", "1031", "4526", "5562", "1326", "4603", "3149", "5035", "5539", "1299", "318", "2655", "1995", "6124", "4435", "2632", "4058", "2423", "791", "2704", "2905", "1699", "2621", "1386", "1829", "5856", "2513", "873", "2523", "4934", "3916", "4156", "2123", "2011", "4991", "5282", "3899", "1328", "6143", "428", "6012", "3600", "2019", "5050", "3233", "4359", "572", "2320", "6215", "5484", "4247", "1617", "3024", "5941", "180", "3410", "197", "5669", "4285", "1285", "4625", "1516", "3200", "1466", "4500", "3806", "934", "52", "1367", "4909", "4039", "1444", "1005", "4182", "4288", "3639", "280", "105", "3211", "972", "3111", "5008", "2900", "4246", "779", "1911", "842", "6259", "2746", "1756", "432", "2715", "4419", "2652", "2993", "970", "4777", "4821", "1150", "3377", "5553", "4442", "4537", "3603", "2393", "4251", "2091", "4304", "1321", "6240", "3746", "346", "1435", "852", "4153", "1667", "5354", "450", "3186", "376", "820", "5835", "2668", "508", "2581", "995", "172", "5790", "1902", "2795", "2592", "3572", "17", "237", "2649", "1791", "1148", "5306", "1654", "2763", "217", "1082", "473", "2633", "1955", "4921", "4691", "2425", "3838", "3358", "2473", "932", "2370", "4951", "1534", "154", "141", "3877", "6051", "3236", "1602", "1344", "3881", "778", "117", "2233", "4552", "3262", "2784", "1080", "876", "4282", "6063", "3223", "512", "4240", "4295", "2751", "6173", "6161", "3404", "2463", "4634", "5471", "2818", "581", "1229", "3843", "1768", "5373", "706", "5371", "3492", "731", "1313", "2874", "4581", "2438", "2527", "2806", "507", "5920", "169", "6069", "6013", "4734", "3271", "5293", "2057", "5719", "2260", "6133", "2938", "5002", "543", "2749", "5586", "4420", "429", "4741", "2012", "4626", "4054", "1131", "3449", "3544", "5204", "2913", "640", "3456", "3969", "2787", "3103", "3154", "2835", "1709", "961", "5988", "3950", "1626", "2878", "4311", "1415", "2707", "2418", "3851", "1236", "262", "5345", "2503", "3313", "5286", "68", "3820", "3273", "5929", "2534", "3341", "1371", "1277", "5427", "1411", "531", "2194", "878", "2851", "136", "5842", "1648", "1506", "6050", "6087", "5502", "3400", "1647", "4086", "5694", "4988", "2374", "3034", "2330", "5428", "2703", "502", "4717", "4992", "3480", "1871", "4155", "2119", "866", "37", "3466", "650", "5690", "1876", "4277", "4799", "728", "6056"]], "valid": ["int", ["4342", "2416", "3712", "179", "5770", "4636", "2000", "2289", "4411", "4528", "3706", "3179", "4123", "212", "3256", "3043", "2955", "2171", "1817", "6230", "3231", "35", "1266", "3743", "3503", "345", "1826", "1547", "4239", "3625", "3786", "1658", "5252", "4178", "5946", "2084", "167", "2904", "2449", "2201", "4893", "5346", "5518", "436", "8", "4784", "2897", "2308", "2558", "5566", "4736", "1583", "5406", "412", "3348", "4237", "5881", "3014", "752", "474", "5655", "4935", "1048", "3673", "2880", "3012", "5767", "2181", "5699", "5940", "5151", "322", "2518", "1101", "576", "1652", "2474", "5850", "196", "4139", "1800", "3344", "2191", "609", "138", "5435", "1529", "1597", "3373", "1490", "6117", "1167", "2831", "5234", "843", "3592", "4015", "5803", "1104", "195", "909", "5992", "2230", "5727", "1619", "1740", "5052", "2371", "5492", "5464", "4610", "5947", "3718", "3391", "5372", "3708", "2855", "23", "2583", "5229", "4104", "4661", "1500", "5924", "5497", "2594", "1", "4619", "2485", "5739", "3564", "5933", "2744", "5436", "5121", "3402", "2879", "4076", "5298", "2719", "3286", "6065", "1020", "4192", "61", "6147", "2413", "3234", "177", "4253", "4452", "1322", "5390", "1142", "4704", "3767", "2537", "5987", "660", "2509", "516", "545", "1231", "3681", "4495", "4453", "3526", "6156", "1742", "5364", "626", "3824", "327", "235", "3662", "1356", "5656", "4978", "3388", "3983", "2577", "2533", "4913", "4017", "2363", "5404", "1273", "6022", "5106", "2973", "96", "4575", "5426", "4162", "3469", "2869", "3346", "4682", "3129", "4673", "2969", "5905", "5154", "3731", "5357", "547", "1446", "2796", "2291", "4651", "1919", "3836", "711", "2170", "4353", "330", "1983", "5910", "6266", "4036", "3345", "5741", "5051", "194", "2301", "5570", "12", "2245", "6262", "2547", "5352", "3392", "34", "5453", "3590", "3726", "5242", "5258", "4136", "110", "2761", "3610", "4472", "3062", "3431", "1544", "3383", "5493", "1665", "170", "3800", "1108", "2456", "1918", "1694", "438", "2563", "2721", "5588", "694", "1279", "6123", "2645", "2679", "2174", "3814", "5478", "3160", "4790", "2375", "2682", "5193", "4449", "2730", "2551", "926", "1029", "5556", "5127", "5429", "3929", "5196", "1854", "4937", "1055", "5395", "2002", "40", "1799", "5130", "1288", "4986", "689", "3230", "902", "5753", "3684", "5350", "2508", "5500", "665", "4845", "2272", "3237", "5319", "4131", "3375", "85", "2689", "5921", "5437", "3729", "3736", "645", "5901", "4438", "451", "2780", "5717", "2791", "1669", "1718", "6247", "847", "1797", "4087", "1629", "1546", "637", "2683", "5173", "5712", "5189", "1989", "4023", "1601", "5119", "977", "158", "1372", "5632", "5341", "3322", "5950", "4584", "5607", "2480", "1474", "5948", "1790", "3334", "4837", "4021", "3266", "4108", "365", "5291", "3565", "3735", "2917", "3159", "3815", "421", "286", "494", "4165", "4491", "3019", "2898", "3540", "1057", "4775", "1284", "3519", "3619", "4878", "4367", "2758", "2978", "41", "544", "4689", "3722", "3090", "1489", "387", "4197", "4193", "4774", "6064", "2427", "5340", "3593", "3750", "4582", "50", "2626", "1416", "6039", "5715", "2945", "1143", "3141", "4347", "1347", "800", "4977", "5894", "3567", "4205", "4900", "2967", "5761", "1281", "5639", "2802", "4855", "1258", "3453", "6128", "656", "3802", "4733", "1498", "1857", "3907", "458", "5446", "2894", "1823", "4885", "2921", "2576", "3854", "2378", "4946", "2156", "863", "3018", "2731", "895", "1936", "1018", "4307", "740", "2276", "351", "2591", "210", "4753", "3181", "824", "551", "148", "4675", "121", "2409", "1803", "3382", "3083", "6263", "315", "4698", "5648", "654", "1967", "3728", "6164", "3995", "5682", "1830", "911", "1145", "4621", "5289", "3523", "1224", "1494", "385", "5772", "4708", "1566", "6066", "2400", "5023", "4121", "19", "335", "3912", "1680", "199", "1327", "1232", "786", "3927", "3138", "3287", "4567", "214", "4091", "3434", "2021", "1280", "296", "973", "3125", "3465", "2058", "556", "1837", "1409", "4871", "1931", "2107", "4699", "3727", "4933", "3486", "5689", "2980", "669", "3719", "2454", "2661", "2332", "1502", "4660", "1556", "4674", "3902", "2808", "3417", "2901", "6107", "5378", "1219", "3856", "2040", "215", "1872", "4185", "4297", "3139", "5171", "5644", "119", "3556", "5683", "4505", "4424", "2157", "817", "2077", "2116", "5416", "2139", "3770", "1611", "4583", "5011", "2312", "2828", "815", "1261", "2334", "3364", "4920", "5112", "2849", "4838", "3955", "5113", "2696", "6027", "2232", "1980", "5829", "486", "5485", "2004", "2627", "567", "736", "3490", "4839", "4213", "1341", "5206", "1166", "4440", "5736", "1314", "3522", "3768", "1729", "2112", "492", "1170", "900", "6006", "3173", "1560", "2599", "3156", "859", "4210", "3636", "2867", "884", "2024", "3632", "1523", "6093", "3351", "3300", "2252", "3087", "386", "756", "264", "4158", "3317", "5214", "3670", "4631", "3416", "3803", "4999", "5943", "4292", "4107", "457", "2345", "3042", "5875", "5975", "3118", "4278", "888", "4601", "443", "4801", "3700", "3408", "4348", "6035", "3963", "5339", "1027", "4535", "6248", "3795", "5622", "5507", "4836", "6122", "2841", "2382", "5563", "1497", "4051", "3973", "4236", "517", "5872", "109", "456", "3187", "6073", "4883", "5998", "630", "5163", "3004", "2911", "1438", "1105", "3837", "6232", "2395", "4463", "2775", "4047", "2666", "2798", "2987", "3309", "2461", "4720", "1630", "1580", "1765", "727", "2", "10", "836", "5183", "4264", "4683", "2611", "3204", "4643", "2614", "5501", "2636", "3379", "75", "276", "481", "1405", "1169", "1173", "991", "6108", "5670", "2724", "251", "3071", "2531", "4646", "4516", "671", "5668", "3212", "1116", "4807", "4551", "2995", "2642", "3319", "2447", "1325", "2811", "5868", "4346", "5245", "4747", "3132", "3077", "4354", "2402", "1618", "282", "1171", "5688", "6082", "3791", "1240", "5897", "5012", "577", "4012", "569", "355", "5511", "4201", "5641", "5984", "2843", "6149", "1093", "4222", "2872", "4501", "3455", "1949", "1459", "108", "816", "3531", "5019", "3067", "4556", "4333", "5335", "3134", "953", "4533", "2931", "2310", "5745", "5744", "5231", "328", "2500", "2197", "5942", "2966", "3846", "5523", "3751", "3167", "3117", "712", "208", "2453", "3953", "5083", "2102", "3261", "5678", "435", "3242", "268", "2459", "899", "4057", "2290", "3016", "4078", "5820", "3079", "5176", "4856", "1833", "449", "4352", "3251", "1623", "4597", "5841", "3557", "5721", "3123", "5205", "3578", "4825", "2979", "1308", "3433", "3020", "4464", "964", "3398", "3647", "4846", "1345", "5542", "608", "5483", "4652", "2783", "764", "2347", "5366", "4492", "1932", "2323", "163", "3228", "4970", "1146", "4161", "5401", "4082", "5916", "90", "1634", "857", "3895", "5085", "3219", "4355", "3933", "5276", "4365", "2307", "5467", "4046", "2972", "446", "5583", "2381", "5729", "292", "4686", "1815", "602", "252", "4497", "3524", "3649", "2754", "4167", "3939", "5557", "2025", "614", "5532", "1552", "1548", "744", "1561", "6270", "4817", "3550", "5643", "1428", "3548", "5895", "2092", "4390", "2240", "5385", "1186", "3909", "4485", "3573", "1785", "5149", "185", "5782", "267", "5047", "4795", "5054", "4454", "2180", "4460", "4804", "2220", "4446", "1861", "337", "1118", "2479", "5108", "1343", "4350", "406", "6273", "3084", "3620", "1530", "6209", "1776", "3664", "4324", "4183", "5827", "2346", "6251", "828", "2244", "2866", "1073", "789", "1028", "211", "1979", "4244", "5251", "5374", "2582", "1943", "5706", "4456", "4416", "2271", "4803", "5661", "5250", "596", "2186", "401", "4580", "3928", "1971", "5908", "6014", "228", "2598", "1253", "1574", "4132", "4749", "3968", "5548", "4744", "6222", "6036", "4030", "2311", "1183", "5629", "4531", "6172", "5379", "4001", "5336", "4296", "3568", "5280", "6046", "3844", "5663", "4230", "6115", "6279", "2925", "6184", "925", "3290", "1189", "2471", "1482", "2022", "1245", "4462", "1335", "721", "219", "2105", "3657", "856", "2647", "6067", "2653", "1078", "670", "3529", "3450", "4441", "3157", "313", "297", "4284", "353", "4097", "2165", "2294", "5968", "2009", "3175", "5", "2767", "1097", "2777", "4907", "4511", "1106", "3764", "2030", "1257", "3946", "3366", "5311", "4310", "1527", "3357", "3935", "4049", "1004", "5182", "872", "3499", "5253", "5417", "247", "1458", "3972", "5612", "1982", "2912", "4931", "469", "5080", "3304", "5660", "3127", "1457", "309", "3891", "1838", "3030", "4764", "4225", "1382", "39", "619", "1204", "278", "6238", "535", "2215", "2988", "2607", "4587", "2773", "363", "3419", "5269", "5111", "5243", "222", "5246", "4374", "1904", "2845", "4315", "2372", "3005", "2177", "1014", "3781", "2249", "1324", "4499", "3533", "1754", "3325", "2376", "3353", "4949", "5631", "3598", "750", "3827", "4116", "1858", "6275", "2742", "3371", "692", "2842", "2287", "2348", "3115", "5431", "5503", "2445", "2914", "3964", "687", "3478", "4194", "6189", "4679", "5647", "3828", "4283", "4760", "2269", "2522", "290", "1214", "3924", "908", "360", "3336", "4114", "5351", "5878", "1598", "1010", "3801", "20", "2026", "255", "5777", "6092", "2280", "2218", "3376", "1384", "373", "3128", "3699", "734", "4432", "5686", "3532", "3571", "1801", "1079", "1424", "915", "594", "5957", "5024", "5399", "4257", "3864", "5228", "1510", "4393", "579", "4975", "792", "2477", "4896", "128", "4112", "1818", "2214", "4650", "254", "1337", "6094", "3311", "1160", "1582", "397", "5972", "2331", "1137", "3853", "4056", "5068", "574", "2634", "4973", "1778", "1708", "2998", "444", "2550", "331", "1675", "5259", "3460", "1808", "5882", "2394", "827", "4383", "4730", "3246", "4254", "1987", "5606", "1747", "2765", "4345", "1348", "1306", "4969", "968", "4069", "5420", "3085", "2644", "1012", "6097", "2223", "4985", "4667", "1155", "32", "3438", "5078", "4559", "5731", "4641", "2873", "5046", "723", "4771", "4522", "1434", "5294", "2935", "5087", "1213", "4886", "89", "2369", "788", "498", "1573", "4403", "2876", "886", "2907", "5914", "4869", "4860", "1730", "4319", "1567", "1642", "2295", "4010", "1807", "928", "4289", "2991", "2306", "2253", "5902", "14", "33", "5685", "2060", "1683", "1895", "3715", "3112", "5043", "5962", "4206", "2895", "1032", "3318", "6255", "4402", "2810", "770", "2068", "3738", "5746", "2664", "2494", "5363", "5322", "5221", "2248", "776", "5132", "682", "480", "5007", "3238", "69", "6135", "2989", "5674", "347", "762", "2864", "2391", "5836", "424", "5033", "308", "3065", "4299", "697", "3714", "5045", "1111", "4318", "3761", "4124", "6146", "1565", "2502", "2101", "5596", "1127", "2484", "1999", "383", "555", "3656", "1625", "5260", "2224", "4016", "3150", "2692", "6010", "3807", "4226", "5598", "4007", "2723", "4564", "922", "1178", "4553", "5574", "3203", "2344", "1406", "4384", "2516", "5360", "1112", "1094", "3602", "1397", "554", "171", "865", "3061", "5733", "2996", "3488", "5153", "5851", "906", "1287", "1227", "5474", "2540", "2643", "1452", "3554", "1893", "2482", "4169", "4294", "2436", "242", "2188", "3229", "2108", "213", "4320", "5713", "2625", "6134", "4867", "3977", "1736", "1966", "2352", "122", "265", "1250", "3479", "960", "1445", "6089", "4929", "5983", "3278", "1354", "5799", "4745", "3737", "4375", "4572", "4899", "4524", "6020", "5534", "5843", "5911", "3724", "4266", "5369", "5268", "4788", "4273", "672", "3788", "4666", "6104", "4356", "13", "2961", "3208", "4154", "2325", "4207", "1581", "6152", "699", "2922", "5680", "1809", "176", "4005", "4376", "2846", "2379", "2288", "1392", "3471", "482", "1110", "460", "1907", "5927", "920", "5705", "2326", "5095", "1218", "2462", "2854", "3867", "2211", "5546", "525", "2373", "6141", "1832", "2124", "2059", "4498", "4789", "966", "2262", "1842", "4740", "2965", "5134", "962", "4184", "2985", "378", "3516", "5442", "2720", "2490", "1336", "2949", "1088", "3852", "277", "2886", "658", "2457", "5541", "3368", "5370", "5114", "1339", "1695", "6037", "3447", "5892", "2316", "5619", "3776", "1272", "3606", "5830", "132", "1006", "2728", "4820", "6074", "5018", "3381", "4429", "812", "6268", "2590", "1390", "735", "3310", "1924", "1879", "5657", "2852", "5036", "3646", "2229", "5272", "1330", "5611", "3690", "4212", "1300", "1291", "4672", "3439", "3412", "86", "3038", "4616", "1521", "4397", "2799", "4695", "1294", "1712", "3095", "5160", "5758", "3894", "959", "261", "3849", "4995", "1973", "1721", "5633", "1840", "6269", "1912", "403", "3871", "3240", "3021", "639", "4690", "4875", "1557", "497", "1035", "164", "4219", "1015", "6044", "6033", "4065", "4844", "741", "5913", "3517", "6148", "2772", "1492", "2737", "3842", "6220", "3945", "5072", "2163", "1470", "6103", "4256", "2629", "3265", "3659", "416", "3", "5768", "759", "4216", "2681", "1992", "1782", "4915", "4105", "87", "4798", "981", "5082", "855", "1412", "4245", "879", "3308", "49", "3329", "2386", "5284", "1223", "2329", "4607", "248", "1615", "1844", "1555", "4676", "112", "1733", "4818", "5793", "4330", "988", "3967", "1845", "161", "4479", "6081", "165", "742", "4894", "3116", "3489", "4545", "5188", "819", "2536", "4179", "5944", "5748", "245", "1579", "4358", "4106", "440", "4088", "4654", "6127", "6088", "1896", "5701", "6171", "2041", "829", "919", "6130", "2812", "3809", "5304", "1885", "1748", "528", "3245", "924", "3032", "1484", "5299", "5320", "2735", "3347", "5893", "586", "3096", "642", "5288", "2572", "3677", "3597", "2099", "3010", "3595", "5448", "4469", "5696", "1431", "5244", "1197", "5734", "4026", "3462", "2098", "2250", "4928", "2439", "1149", "1846", "3029", "2144", "3137", "6052", "1685", "5137", "2941", "183", "5579", "875", "323", "1537", "123", "514", "5605", "4152", "135", "3491", "3301", "2254", "5664", "854", "1875", "707", "5128", "4697", "4048", "3812", "4423", "3068", "758", "5934", "3608", "1346", "2891", "5971", "5361", "6114", "1034", "5886", "4589", "1592", "1039", "1269", "2667", "3451", "4976", "5755", "2968", "4613", "273", "2700", "5117", "249", "891", "1141", "570", "5487", "4702", "1270", "6200", "2673", "1448", "553", "394", "1769", "557", "4719", "2923", "3547", "2764", "302", "4569", "2430", "3623", "5898", "5519", "2833", "5079", "1120", "4217", "6253", "3255", "5814", "1362", "2800", "5535", "1603", "4160", "2950", "1860", "2548", "5213", "5126", "4259", "4447", "2942", "3885", "2624", "787", "3958", "284", "4772", "2600", "1657", "6216", "5482", "476", "275", "2781", "287", "6158", "2067", "5423", "5776", "5867", "2701", "5928", "3120", "3008", "4722", "4917", "4073", "5069", "3254", "784", "380", "3047", "1359", "3177", "5760", "515", "1463", "5513", "889", "3616", "5564", "1212", "2603", "5209", "5854", "4233", "5248", "1486", "22", "6180", "1276", "1517", "2401", "5961", "4079", "4614", "1759", "2241", "2200", "3723", "3549", "3666", "1963", "627", "3502", "844", "5804", "1856", "2362", "4906", "3741", "1437", "1810", "6207", "4252", "5645", "2136", "6031", "5740", "2792", "1477", "2017", "6205", "2417", "1442", "3643", "6068", "1939", "6140", "729", "94", "2813", "4361", "2475", "6101", "4648", "4873", "3509", "4103", "2605", "6015", "3169", "4874", "203", "3031", "2986", "718", "5459", "3570", "2826", "4025", "1479", "5935", "4891", "6055", "130", "4982", "4388", "1571", "748", "6181", "2476", "1317", "2497", "2193", "3302", "1588", "6150", "3882", "341", "358", "4622", "140", "1679", "312", "3476", "4961", "3048", "3615", "3773", "5315", "969", "5527", "5279", "2384", "4508", "4710", "2052", "1360", "4957", "1839", "445", "448", "5073", "530", "389", "4027", "4532", "5885", "4655", "4474", "209", "6032", "5569", "797", "5107", "3799", "523", "3162", "3443", "6182", "4293", "3306", "5239", "2120", "1984", "3498", "3473", "2741", "6034", "811", "1859", "5783", "1090", "4742", "1952", "986", "2140", "5275", "6028", "2709", "3876", "1002", "1586", "5402", "994", "1888", "1666", "67", "143", "4938", "5550", "1075", "584", "3966", "5907", "131", "3511", "5610", "3387", "258", "603", "4990", "3624", "5540", "1986", "2691", "5499", "2202", "1249", "4004", "4779", "4696", "4196", "4032", "1202", "4248", "1914", "3276", "4514", "5195", "223", "1267", "1882", "2899", "3232", "4272", "71", "3001", "5486", "2797", "369", "2556", "1244", "2420", "4926", "1194", "893", "2337", "4908", "2074", "1901", "4187", "3303", "1083", "5587", "4413", "4093", "24", "3655", "5238", "5533", "3668", "1320", "1711", "3703", "3562", "1825", "595", "1175", "2755", "6016", "4181", "4731", "1650", "1519", "4071", "5324", "4195", "1222", "3868", "408", "618", "3537", "564", "4748", "1948", "2336", "232", "307", "4604", "2458", "471", "6154", "2712", "4924", "2637", "3753", "2553", "1714", "107", "441", "4430", "6280", "4560", "2971", "4756", "5344", "4013", "4765", "3943", "3463", "423", "5084", "1920", "4357", "4953", "1408", "2053", "5955", "912", "583", "2695", "1834", "702", "3536", "5618", "3263", "1103", "6208", "1037", "5665", "3390", "753", "5274", "1713", "1892", "769", "5604", "3022", "2788", "519", "4507", "3289", "3840", "500", "1512", "411", "5691", "5857", "5220", "3039", "1089", "467", "4274", "2934", "4067", "5392", "2435", "4180", "2884", "1928", "3596", "5716", "391", "5356", "4159", "4954", "5567", "5636", "4848", "3521", "1743", "4902", "2419", "3195", "5418", "3819", "496", "5887", "5695", "4134", "157", "1423", "407", "5305", "3777", "2399", "3293", "5561", "2075", "2266", "2033", "2114", "3904", "487", "2785", "3772", "2752", "241", "5952", "3618", "3282", "5290", "336", "1251", "5271", "5081", "1916", "4914", "4800", "4022", "3164", "5233", "4797", "1152", "3380", "955", "5266", "1387", "3896", "2356", "653", "1417", "1820", "2063", "3320", "5839", "3979", "834", "1590", "6265", "3663", "1585", "4703", "229", "3732", "2470", "395", "485", "3395", "6105", "5075", "3863", "2906", "377", "1926", "1107", "652", "5365", "398", "5445", "684", "3260", "3869", "722", "1374", "2406", "4415", "2517", "5524", "1230", "1595", "3975", "2273", "5064", "145", "6229", "230", "2070", "6078", "1023", "4263", "4168", "324", "2127", "1159", "2452", "6167", "1133", "6008", "1682", "6017", "1672", "1646", "2357", "5599", "206", "5536", "1771", "77", "1643", "4371", "2932", "798", "1941", "6250", "560", "3515", "2669", "3793", "5531", "244", "3035", "2702", "4709", "2013", "4457", "833", "3475", "2132", "4830", "2999", "1831", "4943", "1954", "288", "3108", "4494", "11", "1298", "1304", "5577", "4186", "2278", "3056", "1732", "1518", "5175", "4258", "781", "4546", "3144", "3158", "3908", "426", "774", "5398", "2275", "793", "4378", "2896", "5267", "4725", "1594", "3960", "5432", "4117", "3495", "3580", "5635", "1578", "5441", "2018", "6277", "3414", "3422", "1880", "1391", "1757", "6163", "2300", "4174", "1217", "6132", "118", "1365", "2390", "81", "3780", "582", "1702", "5093", "3258", "823", "1092", "2705", "4642", "2411", "3426", "6070", "2243", "3121", "6214", "3694", "18", "2984", "3510", "5980", "5397", "3040", "3155", "46", "505", "2442", "5059", "6042", "1690", "2251", "1900", "352", "5449", "155", "5102", "4963", "3915", "3299", "1661", "5292", "3954", "825", "1501", "427", "506", "987", "1572", "2037", "207", "5859", "2868", "1616", "4955", "3053", "5006", "1760", "2908", "5554", "5555", "2237", "946", "1596", "1659", "4659", "3594", "2560", "3210", "4611", "4534", "2161", "3007", "3252", "6239", "3205", "1456", "2947", "2747", "3474", "4864", "2580", "4037", "3037", "4344", "5580", "5460", "4555", "6203", "5714", "1763", "3658", "1944", "3704", "4172", "4122", "536", "1333", "3959", "5722", "1196", "1096", "5545", "4751", "4668", "4939", "2610", "4997", "3678", "15", "883", "6102", "2460", "4911", "2601", "4563", "4332", "6168", "1054", "2638", "359", "285", "3934", "3661", "3163", "1663", "3063", "3218", "4094", "1462", "6278", "4480", "4663", "6048", "3185", "6169", "935", "3697", "1139", "5469", "2970", "5203", "1383", "5477", "3717", "1153", "4275", "2255", "193", "975", "5208", "2687", "4170", "3269", "1937", "3976", "3679", "3948", "837", "2043", "5509", "5709", "726", "1422", "881", "6137", "5270", "2404", "2946", "4316", "3614", "149", "2801", "3078", "5131", "4164", "4317", "6076", "272", "4823", "3367", "5038", "2138", "1363", "186", "4852", "1099", "3165", "5773", "1256", "2920", "5383", "3168", "5623", "4847", "6003", "5105", "3394", "4175", "631", "537", "366", "4059", "5222", "4861", "2264", "1608", "850", "4439", "1454", "2216", "3905", "2532", "4596", "2151", "1559", "3559", "5846", "2983", "334", "3423", "1011", "3941", "466", "3872", "2589", "1496", "4840", "4227", "1128", "2939", "3359", "5959", "1303", "168", "4849", "958", "4034", "3119", "1638", "4809", "641", "289", "1950", "175", "5027", "4612", "1504", "810", "3330", "2926", "4746", "4586", "490", "1847", "2762", "1289", "1945", "4979", "3017", "4458", "5810", "5792", "356", "5658", "5594", "1976", "51", "2429", "715", "2064", "1282", "1334", "822", "1796", "384", "1338", "91", "680", "3342", "5650", "5512", "501", "2619", "4657", "5227", "4919", "2543", "5815", "3810", "1379", "1292", "3481", "2535", "5086", "3333", "4922", "1260", "777", "4101", "465", "5410", "221", "5124", "848", "1216", "1449", "5809", "129", "4102", "6061", "1429", "4380", "6236", "6185", "1147", "1788", "192", "5424", "305", "5333", "4475", "4792", "4110", "3194", "6100", "2380", "552", "695", "2008", "1487", "4242", "3191", "790", "1515", "5826", "2198", "5876", "1908", "4518", "1613", "5323", "4234", "548", "3285", "503", "5353", "6194", "1069", "2820", "5328", "5498", "1991", "2559", "1425", "1237", "4527", "5170", "5589", "771", "65", "1165", "4190", "6244", "598", "184", "6086", "3497", "2364", "3378", "4412", "4705", "372", "611", "5600", "3630", "4579", "620", "5824", "2584", "5393", "4362", "5257", "1526", "5254", "483", "101", "6264", "4050", "1355", "4301", "2361", "3558", "5832", "4585", "6091", "2032", "2182", "5194", "396", "3822", "6218", "4468", "5764", "1974", "152", "6139", "3575", "2082", "522", "2844", "5330", "566", "4428", "3082", "5264", "393", "6153", "2847", "5281", "1783", "3552", "5592", "2228", "1671", "2014", "2903", "6160", "5909", "5355", "4477", "4712", "2426", "2405", "4503", "708", "5838", "4539", "2125", "3086", "2036", "1873", "6125", "3725", "664", "5148", "3555", "1375", "5022", "3528", "489", "4618", "319", "1130", "4550", "3055", "5877", "3816", "1402", "62", "4483", "1400", "5408", "1215", "3501", "2035", "2247", "1369", "5159", "2173", "3980", "56", "3389", "76", "5414", "1923", "4814", "2870", "4343", "5145", "513", "2103", "5530", "291", "634", "945", "3576", "3890", "5917", "997", "6210", "1026", "2588", "5458", "4341", "2641", "3901", "5991", "4835", "1102", "538", "2612", "1961", "2786", "343", "5490", "2001", "3932", "2512", "2706", "3682", "60", "2697", "3279", "1910", "1342", "6195", "3543", "6206", "3686", "713", "3454", "3841", "3951", "1707", "1247", "58", "965", "803", "2303", "6085", "5528", "3642", "3757", "5313", "1563", "5157", "475", "3360", "5538", "2602", "4229", "4189", "1905", "3338", "4822", "2388", "2315", "3587", "5241", "5415", "399", "882", "4787", "417", "2514", "1727", "2365", "6095", "3221", "5923", "5443", "2431", "5381", "534", "2415", "1221", "5003", "3734", "3415", "4053", "5060", "1533", "1399", "2072", "4232", "3607", "3942", "2982", "281", "1869", "4140", "666", "6166", "2753", "2829", "3291", "5058", "5590", "1353", "613", "1394", "5265", "588", "5070", "3759", "1805", "5651", "2622", "655", "6165", "3470", "6119", "3350", "3283", "887", "3072", "2184", "1964", "686", "575", "2256", "3745", "5466", "2212", "4448", "1050", "461", "5049", "3093", "2392", "1095", "2992", "1716", "1767", "4857", "3190", "1631", "2733", "5413", "6112", "3693", "5010", "368", "2853", "5296", "1913", "301", "4715", "3339", "5704", "5030", "1972", "3487", "4407", "3857", "1509", "3109", "142", "5065", "2620", "3583", "3033", "5092", "1662", "1852", "1123", "3174", "5255", "5855", "1975", "3133", "2235", "495", "1676", "1789", "4796", "4331", "3685", "807", "2168", "3900", "1432", "2192", "88", "1301", "3893", "361", "1804", "4305", "3199", "2432", "5879", "606", "3216", "1656", "57", "1493", "956", "6258", "4780", "5200", "2863", "2007", "1553", "747", "4128", "5039", "5262", "4872", "6233", "1211", "160", "4298", "5218", "5169", "763", "271", "374", "1802", "2321", "2498", "6084", "509", "2049", "5326", "4018", "349", "1198", "6254", "1378", "4632", "3860", "4308", "5430", "2274", "5104", "610", "5164", "1686", "4577", "3766", "4890", "5697", "5053", "6223", "4952", "3352", "1126", "2789", "198", "808", "1499", "5256", "5237", "3754", "2094", "2079"]]} \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets-test-split.txt b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets-test-split.txt new file mode 100644 index 0000000..3792a10 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets-test-split.txt @@ -0,0 +1 @@ +{"xvalid": ["int", ["253", "834", "737", "45", "373", "390", "961", "592", "41", "217", "245", "1039", "67", "3", "121", "71", "407", "786", "511", "504", "514", "659", "521", "524", "173", "611", "1023", "636", "113", "148", "283", "174", "1074", "176", "387", "285", "915", "213", "699", "916", "382", "66", "550", "496", "692", "732", "709", "633", "538", "911", "735", "1058", "980", "923", "795", "238", "769", "75", "225", "107", "200", "763", "1046", "264", "315", "838", "109", "259", "427", "74", "221", "275", "1022", "436", "596", "586", "456", "805", "568", "989", "949", "139", "231", "557", "237", "352", "499", "994", "877", "268", "531", "269", "672", "476", "68", "2", "1052", "330", "828", "56", "188", "371", "487", "548", "497", "710", "501", "1040", "912", "314", "651", "728", "25", "1069", "969", "17", "284", "554", "816", "272", "809", "1078", "904", "997", "578", "662", "1037", "270", "664", "933", "1071", "823", "761", "945", "723", "631", "687", "306", "1000", "60", "953", "30", "190", "693", "725", "1094", "972", "372", "775", "469", "897", "785", "119", "178", "122", "526", "442", "304", "396", "566", "438", "963", "444", "581", "1017", "235", "905", "1106", "33", "132", "463", "398", "564", "787", "1056", "547", "579", "655", "942", "136", "558", "485", "772", "451", "717", "1065", "567", "830", "6", "676", "607", "776", "12", "991", "508", "807", "529", "599", "832", "1009", "509", "517", "484", "116", "756", "334", "936", "695", "661", "827", "288", "891", "652", "301", "629", "992", "311", "171", "203", "44", "919", "433", "464", "189", "958", "343", "197", "495", "180", "884", "500", "1042", "353", "612", "572", "748", "298", "588", "903", "215", "713", "1007", "808", "560", "694", "921", "266", "902", "929", "532", "159", "1079", "533", "715", "783", "752", "906", "701", "1", "1103", "917", "94", "482", "289", "38", "543", "1081", "404", "434", "309", "129", "854", "341", "418", "474", "999", "962", "342", "815", "143", "262", "313", "47", "424", "62", "803", "546", "1031", "97", "751", "489", "893", "328", "105", "307", "677", "123", "733", "552", "523", "369", "324", "1108", "1041", "479", "1033", "439", "773", "872", "46", "971", "978", "103", "257", "843", "1002", "1076", "700", "615", "868", "55", "1092", "817", "849", "458", "106", "767", "571", "594", "755", "683", "593", "93", "605", "19", "951", "117", "134", "426", "169", "300", "928", "437", "535", "987", "29", "401", "448", "399", "973", "707", "147", "595", "126", "839", "825", "862", "1098", "758", "336", "684", "539", "757", "124", "52", "580", "177", "1088", "377", "616", "696", "689", "505", "858", "940", "455", "234", "214", "81", "865", "1021", "880", "818", "551", "966", "39", "1096", "1003", "506", "42", "201", "127", "412", "591", "393", "1029", "114", "996", "671", "410", "467", "645", "965", "765", "943", "470", "831", "254", "250", "453", "762", "840", "429", "914", "462", "950", "859", "770", "913", "674", "583", "1101", "974", "69", "286", "861", "883", "525", "445", "1030", "175", "70", "1044", "43", "967", "211", "386", "556", "450", "954", "779", "811", "416", "1035", "98", "549", "425", "187", "61", "754", "486", "918", "447", "461", "368", "908", "603", "364", "440", "959", "381", "920", "598", "204", "582", "22", "1015", "394", "120", "305", "419", "553", "54", "310", "277", "281", "16", "627", "415", "1026", "1085", "488", "88", "856", "888", "635", "857", "657", "243", "15", "1047", "1012", "988", "1063", "513", "379", "13", "273", "1018", "79", "507", "584", "829", "166", "92", "423", "1005", "1010", "14", "870", "620", "845", "57", "741", "976", "291", "146", "796", "673", "680", "609", "851", "252", "1075", "494", "293", "822", "545", "714", "360", "80", "937", "647", "51", "774", "167", "384", "800", "338", "1008", "1011", "320", "938", "738", "990", "90", "388", "925", "844", "472", "49", "1064", "590", "722"]], "xtest": ["int", ["934", "577", "926", "760", "297", "383", "1004", "931", "939", "1067", "229", "1019", "984", "628", "1072", "84", "361", "73", "332", "824", "316", "909", "292", "736", "894", "216", "794", "716", "375", "866", "184", "983", "179", "255", "348", "335", "744", "194", "901", "191", "280", "65", "730", "417", "791", "536", "871", "512", "806", "948", "59", "363", "492", "7", "156", "402", "327", "624", "18", "4", "378", "302", "960", "638", "640", "955", "750", "565", "370", "483", "970", "354", "747", "76", "345", "530", "1050", "667", "907", "435", "21", "1054", "688", "630", "478", "267", "898", "1049", "812", "118", "1084", "1001", "261", "814", "230", "321", "648", "705", "819", "1104", "702", "36", "63", "128", "347", "274", "561", "782", "781", "232", "703", "452", "1014", "248", "359", "104", "138", "739", "947", "125", "172", "1034", "137", "473", "986", "932", "276", "83", "642", "205", "385", "679", "678", "130", "520", "643", "704", "96", "1086", "28", "323", "797", "27", "220", "35", "869", "623", "58", "663", "23", "77", "731", "344", "544", "1077", "185", "764", "226", "326", "742", "977", "685", "813", "165", "621", "101", "115", "793", "162", "892", "576", "802", "924", "218", "639", "365", "244", "601", "542", "585", "192", "196", "613", "600", "100", "493", "534", "350", "930", "1093", "879", "975", "376", "656", "183", "1027", "675", "874", "711", "863", "31", "153", "89", "968", "279", "271", "430", "608", "618", "522", "510", "540", "465", "789", "537", "131", "778", "208", "133", "669", "686", "158", "570", "527", "362", "1013", "964", "299", "563", "956", "331", "222", "1016", "227", "706", "698", "468", "199", "206", "163", "1053", "867", "836", "263", "294", "575", "10", "1089", "397", "935", "847", "374", "233", "574", "569", "150", "768", "161", "885", "351", "666", "168", "108", "422", "641", "413", "941", "9", "198", "24", "1102", "207", "26", "247", "708", "518", "278", "290", "406", "164", "349", "841", "366", "637", "287", "910", "339", "0", "622", "889", "295", "195", "649", "876", "48", "236", "927", "626", "792", "993", "258", "1061", "820", "1095", "878", "357", "431", "1059", "799", "753", "855", "810", "900", "219", "614", "256", "1100", "20", "860", "712", "503", "333", "587", "668", "589", "597", "1105", "346", "317", "155", "606", "864", "40", "471", "170", "466", "1073", "160", "985", "788", "421", "392", "632", "875", "1028", "541", "562", "318", "850", "790", "239", "995", "1082", "653", "826", "240", "682", "78", "1060", "142", "457", "112", "981", "395", "833", "718", "658", "743", "265", "319", "95", "409", "1038", "690", "91", "477", "459", "691", "1032", "355", "670", "719", "246", "228", "1025", "853", "5", "490", "420", "340", "881", "1024", "610", "145", "1107", "480", "428", "835", "475", "86", "882", "650", "952", "282", "181", "1080", "260", "411", "209", "296", "798", "1090", "391", "37", "400", "982", "777", "727", "251", "99", "1070", "102", "491", "241", "979", "720", "922", "766", "443", "367", "654", "848", "34", "182", "202", "8", "852", "481", "1097", "224", "729", "998", "780", "502", "734", "625", "784", "515", "141", "644", "210", "1087", "740", "1051", "746", "414", "890", "660", "446", "555", "837", "87", "681", "110", "842", "380", "449", "724", "1091", "157", "50", "193", "405", "498", "887", "619", "886", "873", "358", "135", "403", "617", "242", "846", "329", "140", "821", "759", "249", "1006", "559", "1055", "441", "899", "697", "957", "745", "454", "1066", "1043", "646", "64", "528", "1045", "85", "1068", "32", "111", "634", "771", "749", "322", "82", "946", "944", "896", "149", "212", "721", "516", "804", "72", "602", "432", "308", "356", "186", "11", "152", "1020", "1083", "1099", "389", "312", "801", "325", "144", "460", "726", "1062", "53", "1048", "303", "1036", "1057", "604", "337", "151", "154", "223", "573", "665", "519", "895", "408"]]} \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets.config b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets.config new file mode 100644 index 0000000..b681784 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/pets.config @@ -0,0 +1,13 @@ +{ + "scheduler": ["str", "cos"], + "eta_min" : ["float", "0.0"], + "epochs" : ["int", "200"], + "warmup" : ["int", "0"], + "optim" : ["str", "SGD"], + "LR" : ["float", "0.1"], + "decay" : ["float", "0.0005"], + "momentum" : ["float", "0.9"], + "nesterov" : ["bool", "1"], + "criterion": ["str", "Softmax"], + "batch_size": ["int", "256"] +} diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn-split.txt b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn-split.txt new file mode 100644 index 0000000..900f9ca --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn-split.txt @@ -0,0 +1 @@ +{"train": ["int", ["46692", "63322", "22278", "34546", "51016", "65663", "32396", "61118", "67588", "61927", "41653", "34445", "45341", "34474", "30044", "65543", "64296", "24247", "28223", "54951", "7086", "3344", "65544", "57959", "2793", "13348", "38852", "38973", "61390", "38433", "42827", "37751", "21130", "29843", "33870", "38905", "1112", "53200", "69372", "6617", "50871", "9639", "5537", "36575", "39983", "31938", "35264", "30473", "16072", "39486", "27091", "61447", "68934", "51813", "73159", "2990", "33773", "67028", "41033", "71838", "38365", "2531", "23573", "65200", "42751", "46358", "35692", "51932", "36771", "63597", "32707", "70888", "27849", "39975", "46571", "52978", "26162", "12439", "43351", "45507", "15878", "12087", "41316", "25375", "42809", "5520", "38007", "32421", "37915", "44742", "843", "27355", "32407", "66276", "46841", "49864", "45413", "41687", "45777", "23920", "56825", "53115", "23124", "57713", "22823", "27785", "5995", "39110", "42860", "26561", "68430", "54362", "36096", "48619", "11777", "52806", "72878", "12459", "36695", "7924", "787", "29375", "49860", "27857", "67157", "21354", "29585", "50875", "52253", "67390", "53346", "2832", "31120", "45994", "18487", "48012", "61885", "72106", "42253", "71651", "44334", "45376", "68682", "2957", "35104", "45681", "49753", "4232", "6022", "16280", "22740", "30986", "63643", "13517", "23660", "60462", "16047", "27819", "19865", "6681", "16714", "59934", "63756", "42871", "32026", "69854", "26154", "41131", "49196", "48877", "30720", "69456", "14456", "30178", "59507", "40378", "47023", "52844", "19954", "67633", "5515", "41974", "42306", "13192", "25377", "37653", "18058", "61268", "14164", "4572", "20789", "39163", "23488", "33995", "48417", "40818", "40752", "37699", "54186", "13715", "51493", "50888", "72498", "17134", "59776", "12306", "63031", "13373", "42085", "65492", "47141", "16953", "27799", "47433", "44285", "44278", "15009", "65007", "15460", "12727", "63060", "16730", "400", "53580", "57297", "55130", "68971", "40375", "13748", "60862", "4178", "17720", "41268", "40732", "31306", "71488", "29469", "17157", "11947", "30877", "47302", "68945", "16202", "34821", "18598", "33670", "12078", "58131", "73082", "10476", "46435", "67653", "49187", "46883", "53096", "23351", "1239", "63132", "43214", "53662", "47915", "3233", "4811", "37110", "69588", "25782", "6999", "30270", "61073", "62485", "17421", "9600", "67476", "53051", "17305", "4553", "26833", "6640", "22686", "19541", "26735", "70051", "6505", "26777", "11398", "16199", "52216", "71280", "62717", "37532", "71774", "48359", "47289", "30402", "39123", "5309", "19676", "8058", "29044", "61148", "38281", "38398", "39962", "42124", "30805", "51482", "51568", "9349", "51960", "56409", "31278", "7139", "44671", "43058", "60330", "48902", "39047", "55359", "15480", "72553", "64583", "28073", "3395", "34918", "42355", "35902", "24567", "53936", "30528", "66491", "8755", "3011", "70723", "9012", "25549", "19661", "28925", "72608", "59749", "38961", "53287", "24344", "56977", "475", "42137", "56089", "30797", "54692", "1114", "14004", "73184", "7699", "30759", "70519", "34020", "57654", "67210", "25244", "16568", "23276", "64680", "56906", "57032", "45530", "19231", "22647", "40398", "56176", "37253", "2477", "17790", "62674", "1603", "16956", "5940", "16788", "34473", "16892", "60684", "34884", "25555", "7044", "60912", "32591", "49178", "45064", "33335", "72536", "51979", "34100", "29301", "60363", "1875", "49206", "11938", "13261", "27875", "14561", "28646", "60488", "34706", "9655", "55050", "37403", "24523", "25627", "13194", "21780", "12298", "41591", "15935", "21542", "22458", "56380", "11981", "73185", "46537", "49730", "56848", "61933", "2860", "71951", "56467", "15161", "55607", "67239", "55624", "70384", "54736", "32823", "63082", "64249", "47652", "8075", "66714", "50224", "63411", "48042", "49209", "50338", "45066", "18064", "67745", "4373", "32819", "16054", "815", "18558", "37590", "59169", "64078", "2813", "28701", "29293", "20608", "10915", "66265", "72607", "47168", "35143", "41560", "22275", "29778", "38621", "56201", "1230", "71432", "54660", "55544", "59608", "12584", "43293", "51807", "11160", "70845", "6233", "5631", "27924", "16014", "11839", "15711", "66711", "2995", "35911", "62344", "6461", "42256", "19194", "57748", "39021", "69531", "21725", "7281", "10337", "53825", "66743", "35718", "50288", "41796", "7527", "39318", "47438", "37506", "22813", "40249", "5420", "42703", "37100", "13330", "7757", "7106", "28895", "21105", "5756", "62963", "6097", "22051", "7503", "6420", "22341", "28889", "31655", "24824", "11314", "73144", "7903", "16299", "35931", "28920", "53502", "8095", "50384", "2953", "23000", "67815", "31504", "12848", "7764", "18272", "68295", "841", "69277", "40198", "55650", "44581", "59466", "56607", "39815", "26182", "8438", "30816", "23599", "68692", "30201", "26856", "59709", "43506", "72237", "31927", "32127", "15355", "56416", "55283", "59016", "39953", "23054", "61568", "31818", "7767", "64136", "14728", "47300", "15975", "30951", "31688", "25684", "72881", "54094", "55178", "67265", "32838", "2500", "34913", "67575", "41516", "23589", "31182", "59621", "66025", "71462", "40134", "17366", "10805", "45836", "61264", "28116", "2008", "31437", "47376", "35543", "20317", "36146", "6037", "29021", "21667", "39317", "36964", "44377", "870", "56205", "37334", "28446", "15897", "7229", "19728", "12589", "50730", "55157", "4945", "63710", "12933", "24715", "40662", "24613", "8855", "62896", "13805", "54869", "66290", "24108", "24501", "7249", "50075", "39190", "37872", "27534", "34902", "65791", "19366", "4149", "44201", "31522", "21233", "63629", "70204", "51516", "71768", "53429", "27151", "20260", "3162", "4323", "38192", "13035", "47133", "67080", "61808", "72746", "26440", "7128", "59945", "50706", "55214", "72542", "39314", "66039", "58867", "67736", "13913", "10124", "72156", "35436", "56471", "68478", "10455", "53727", "4783", "51741", "34233", "20268", "64300", "48451", "30087", "34216", "66960", "12605", "1590", "32375", "9115", "44882", "72258", "3509", "54328", "50113", "51642", "60924", "34058", "49108", "35945", "18363", "54426", "37667", "26483", "48938", "17107", "4668", "41574", "49939", "20022", "22889", "69823", "31659", "50615", "62045", "7726", "47213", "30753", "3007", "60713", "31061", "13620", "60300", "36140", "15444", "57849", "28374", "28955", "21704", "22918", "41303", "39003", "46437", "32496", "65904", "38393", "72471", "26275", "62215", "57516", "62104", "4147", "47976", "11333", "16672", "61381", "2004", "67810", "51156", "42628", "23267", "15320", "1716", "64566", "36133", "35932", "42445", "50879", "16978", "68815", "55369", "247", "35496", "35409", "7272", "54273", "12323", "21103", "10892", "69593", "27589", "19868", "1379", "29161", "16857", "10959", "17586", "3771", "16277", "44168", "898", "73117", "12054", "65866", "41192", "48480", "53622", "57644", "1705", "72370", "18063", "26444", "5828", "23268", "66494", "41022", "61649", "32595", "2563", "36753", "43140", "47732", "29170", "71205", "54304", "2298", "22104", "18548", "2234", "22074", "22213", "53990", "4917", "3608", "71037", "30575", "39557", "55151", "42655", "19164", "32928", "10784", "35460", "45135", "32244", "7508", "70596", "33395", "57128", "923", "52699", "64774", "19131", "64008", "61805", "34248", "45849", "60571", "18361", "56797", "11735", "2423", "37966", "2774", "34885", "14988", "17272", "61799", "63692", "9589", "3617", "63919", "5920", "54875", "21504", "40508", "19549", "72899", "66140", "12600", "37839", "44018", "8178", "33095", "43830", "16609", "64967", "18239", "45325", "62617", "49738", "63704", "9433", "12374", "71152", "26015", "38513", "58323", "50053", "26372", "11270", "57565", "1868", "18876", "25239", "71881", "50367", "41878", "35236", "43927", "30807", "547", "66765", "26988", "25560", "2472", "2485", "63507", "8837", "63989", "24431", "4872", "21969", "60097", "59355", "12703", "70226", "47402", "2652", "32093", "41925", "64988", "11656", "71629", "57320", "44663", "34476", "67420", "28174", "37148", "23354", "72134", "54400", "32269", "47445", "49616", "65487", "21453", "61270", "11833", "3701", "24641", "54254", "15719", "27281", "3045", "66399", "64277", "38467", "2273", "24268", "53003", "7214", "44383", "46372", "16538", "5492", "8676", "1935", "41160", "29114", "57187", "50548", "54615", "50312", "44567", "5917", "17548", "10821", "55346", "24494", "40625", "64067", "11281", "49139", "13468", "67467", "69066", "22017", "13610", "15489", "46747", "23398", "56696", "20117", "47690", "49710", "64563", "48382", "52225", "27992", "20372", "16192", "2969", "23018", "62251", "19276", "673", "1021", "32411", "7162", "56275", "1780", "58321", "49412", "25703", "24650", "29401", "43155", "33634", "60110", "44069", "33895", "67812", "52441", "71538", "67933", "1127", "3392", "30704", "66302", "20831", "30345", "41787", "7372", "53706", "28352", "64023", "18654", "2708", "37387", "58654", "36151", "46706", "72598", "63446", "20221", "61556", "69421", "46308", "19089", "72844", "28826", "65304", "13793", "7424", "12129", "9320", "28274", "54599", "20845", "22882", "17912", "47983", "70371", "35093", "18565", "47984", "2370", "63", "2925", "38140", "47096", "4556", "53494", "66725", "10431", "16882", "17971", "23687", "25760", "25357", "16593", "17802", "69446", "54622", "36618", "65452", "14614", "5927", "70341", "48922", "50806", "49225", "43671", "5477", "18156", "22415", "2983", "25482", "48230", "61559", "51903", "21930", "18385", "27915", "56756", "25988", "64186", "5732", "16026", "60168", "54424", "65625", "2455", "18376", "42571", "28355", "14717", "1969", "16416", "48069", "17044", "29019", "26377", "68995", "67468", "12534", "9256", "57916", "54553", "46931", "18849", "8618", "34296", "50060", "53516", "50545", "56407", "42743", "60865", "46878", "28111", "62143", "51018", "45757", "45006", "70822", "59314", "44794", "9156", "58262", "54729", "8684", "64747", "33611", "62356", "35758", "13760", "35111", "3484", "18600", "57730", "22586", "4016", "66219", "72976", "65031", "43704", "53924", "15112", "61688", "10897", "56872", "62812", "38229", "11060", "25462", "59832", "43628", "70443", "60991", "41534", "59536", "8742", "19896", "62586", "21568", "55971", "24254", "1001", "61804", "68668", "51319", "67505", "15122", "62521", "37549", "61208", "4454", "39961", "49720", "1234", "35986", "52932", "64099", "1368", "27409", "70295", "43794", "65601", "53985", "28185", "63183", "23693", "16598", "54893", "60401", "25599", "20907", "13059", "60461", "54040", "46780", "68856", "24149", "24476", "14843", "37247", "49347", "18022", "20793", "37133", "11388", "36540", "46752", "9443", "7682", "6078", "53093", "30006", "51528", "34661", "62891", "10857", "18686", "22774", "52072", "70935", "43150", "15961", "61332", "14036", "69577", "41825", "15500", "61335", "38066", "2989", "34929", "57890", "16935", "43587", "53178", "3224", "10078", "16729", "22131", "53213", "52468", "15828", "21953", "45027", "60173", "43581", "63508", "64542", "17414", "18299", "15988", "19382", "22643", "26193", "14145", "56863", "72095", "39886", "52065", "33379", "21642", "5680", "25839", "49451", "15721", "70182", "16313", "40979", "69863", "27832", "39492", "35592", "39521", "72296", "36944", "18266", "58825", "29131", "18013", "60174", "28296", "24724", "20646", "15588", "60551", "14485", "9654", "13520", "65985", "14769", "58347", "1480", "56752", "22966", "70057", "29514", "8266", "35646", "17734", "29271", "2130", "2745", "30600", "16343", "70465", "7400", "60151", "20761", "14981", "7359", "56027", "28096", "33176", "24450", "16664", "21235", "42766", "20629", "26259", "19311", "51423", "58181", "15433", "72983", "35272", "34140", "55195", "34407", "59105", "28812", "39379", "3248", "34723", "18504", "3297", "64416", "68528", "27807", "5768", "52254", "84", "61666", "62522", "53000", "1171", "31431", "73169", "57926", "34341", "34154", "45767", "12971", "15170", "58873", "17304", "39548", "20961", "24178", "578", "48927", "34682", "3939", "69628", "56527", "1639", "56884", "22572", "71869", "24340", "485", "20434", "4634", "68397", "19387", "29635", "62571", "72125", "15118", "9714", "70495", "62071", "62444", "46094", "53382", "52525", "22339", "7799", "70275", "63267", "57173", "50853", "19030", "69193", "43111", "51248", "59615", "25444", "34726", "170", "3559", "8877", "8220", "33401", "19692", "30428", "59373", "44224", "52545", "46256", "33902", "33389", "65324", "71970", "63024", "42596", "57012", "19239", "24604", "48284", "4461", "60918", "28312", "44649", "63440", "5674", "57745", "9869", "1970", "55685", "24016", "11944", "39705", "70223", "63113", "45739", "17291", "72045", "42226", "13600", "40073", "10007", "69133", "20327", "1623", "52150", "37448", "19582", "40536", "17969", "66794", "1349", "29042", "5431", "47070", "41946", "23711", "33422", "15181", "57016", "44293", "9766", "47618", "20056", "57122", "15371", "13293", "790", "48646", "66210", "8943", "64888", "20154", "71878", "68721", "47194", "70762", "28885", "2400", "34418", "49778", "59142", "28973", "27377", "30100", "23572", "39503", "6489", "61862", "24436", "28913", "48460", "13253", "69952", "6531", "14954", "60288", "9539", "15434", "23709", "15255", "19883", "60897", "19168", "65569", "65605", "65057", "24105", "25913", "68660", "12562", "27303", "5475", "6299", "45816", "60514", "45668", "4321", "52523", "53815", "18554", "66216", "10963", "29222", "41528", "29523", "61306", "71609", "62032", "51133", "31755", "7782", "39330", "44543", "39157", "5541", "72288", "34493", "48009", "71378", "43418", "6689", "52406", "27081", "25727", "58356", "33765", "73254", "3404", "60543", "47752", "25284", "16287", "15524", "21236", "73066", "25814", "41421", "59148", "43290", "57225", "42385", "67142", "10146", "48893", "61530", "61408", "2557", "2733", "22492", "50183", "20797", "10044", "66418", "20296", "15398", "12661", "6867", "10424", "42847", "12959", "26056", "62132", "25362", "35418", "41366", "5579", "31654", "45109", "44610", "34120", "16058", "15478", "69991", "9086", "39233", "18885", "8000", "9458", "46052", "20443", "30148", "43410", "45098", "66769", "69096", "2920", "13265", "65330", "35240", "18005", "1902", "17201", "62037", "50707", "69986", "25862", "37505", "26714", "32083", "46039", "17071", "66413", "4794", "43330", "27095", "16293", "62691", "30858", "53457", "40944", "52075", "67621", "52889", "23630", "55563", "55264", "28326", "64390", "34687", "37346", "37196", "72587", "42814", "1802", "39790", "38046", "4112", "69299", "59153", "25741", "296", "4986", "58267", "43596", "33185", "58806", "31411", "38431", "5911", "19445", "39276", "15482", "48051", "59285", "60485", "2065", "59386", "14168", "19875", "62652", "50034", "48014", "72322", "23285", "63030", "66181", "27231", "6044", "29230", "3746", "13465", "31021", "26819", "41670", "20358", "22806", "22324", "53967", "28683", "19623", "23786", "60302", "27880", "55184", "2200", "17210", "11292", "73205", "5715", "31807", "42148", "3536", "63021", "30621", "7740", "48244", "228", "27901", "68404", "35375", "16120", "36727", "33400", "56316", "36679", "8127", "70500", "39994", "15847", "25330", "59042", "34000", "54700", "37720", "53119", "68847", "65738", "5531", "35404", "65146", "55670", "72050", "72252", "24583", "2188", "5507", "46767", "42872", "41338", "66060", "51970", "31176", "6718", "39028", "27251", "22759", "51690", "71105", "61838", "50315", "34562", "36389", "8175", "26317", "42673", "9598", "47530", "22171", "67662", "7436", "56487", "54205", "45923", "34958", "70643", "13936", "49796", "42689", "37244", "5454", "33325", "14147", "19127", "38925", "53731", "32000", "6436", "15158", "49306", "46544", "7333", "52815", "6083", "20756", "49223", "24258", "33165", "294", "54384", "28523", "32582", "68412", "55915", "33684", "1013", "48420", "12564", "14481", "53088", "14259", "37241", "32660", "44446", "58846", "14627", "20009", "71456", "42682", "51388", "53845", "73160", "55744", "42523", "49941", "11743", "68502", "71188", "40691", "12556", "6948", "52998", "29071", "47547", "40151", "64849", "12212", "60", "48137", "3365", "32978", "43931", "60641", "35139", "58688", "27858", "7010", "35067", "3604", "17187", "19437", "59976", "27981", "60180", "46567", "23254", "22518", "11584", "30739", "8879", "32238", "10402", "16195", "7127", "18675", "12353", "10196", "46652", "69477", "55604", "7329", "4803", "66805", "35010", "325", "6015", "30395", "14853", "66496", "70966", "4140", "56141", "58390", "5667", "17346", "2634", "4797", "36063", "53710", "44983", "32557", "36990", "33031", "48755", "8983", "3642", "71269", "46743", "29903", "9141", "6378", "53159", "33834", "62425", "19613", "23249", "62015", "8669", "19252", "60150", "22260", "1797", "57739", "1329", "3829", "5297", "42377", "57203", "55816", "28103", "6950", "58967", "8166", "41280", "29601", "37565", "38220", "53958", "983", "61946", "31673", "3500", "52136", "5744", "36155", "22559", "15271", "625", "62783", "18351", "30822", "4327", "60216", "21815", "57981", "63116", "19192", "50633", "62050", "39824", "65857", "73124", "50713", "56983", "69568", "67073", "28742", "22129", "39525", "65519", "39191", "28133", "17930", "64235", "35689", "48876", "39703", "43947", "37393", "24935", "20335", "19106", "232", "22378", "69044", "55924", "32332", "10029", "17299", "62666", "1883", "52743", "46569", "53319", "38964", "60754", "58723", "42151", "29120", "10057", "24869", "47860", "33537", "17344", "27828", "25625", "22688", "65016", "1252", "10300", "51361", "60992", "22554", "67776", "26390", "64077", "44434", "20250", "26490", "32635", "60285", "45176", "63470", "41059", "58538", "33132", "41885", "24404", "34275", "54570", "53660", "48948", "13979", "9994", "16197", "6047", "38155", "69402", "20133", "65225", "22351", "66528", "71193", "67639", "50372", "72862", "45514", "65343", "61966", "49854", "65161", "25730", "45879", "11872", "22123", "69657", "10561", "65552", "34919", "676", "58896", "29050", "7089", "59327", "9560", "17462", "55708", "11263", "5262", "59372", "14935", "64017", "53765", "65269", "38160", "39918", "23831", "49655", "66923", "70631", "30182", "24754", "34317", "42888", "43848", "40358", "3747", "28", "66259", "3501", "48054", "46495", "6035", "39391", "33002", "50213", "23361", "15079", "42927", "4696", "29346", "51554", "4581", "11862", "2103", "71638", "28876", "7794", "58383", "50352", "59993", "51656", "25909", "4119", "37768", "48778", "58824", "54663", "2869", "6485", "30992", "40708", "51616", "46808", "40179", "52550", "63967", "48199", "71237", "67085", "47514", "21609", "54852", "34469", "2444", "57727", "51720", "31304", "17943", "27644", "55165", "5106", "11140", "67333", "33192", "68120", "66635", "38863", "31130", "13394", "25118", "33822", "60786", "36988", "36963", "57377", "46934", "8993", "24852", "17140", "845", "205", "17997", "63250", "5388", "60127", "35027", "8008", "73084", "45747", "2450", "60554", "35526", "54367", "15542", "22206", "80", "26717", "8916", "52213", "36435", "30784", "26227", "4879", "3288", "44328", "50740", "765", "57828", "38570", "38294", "45304", "39780", "4378", "35005", "66026", "49736", "44407", "51650", "64968", "20967", "48159", "24388", "40899", "57098", "19988", "14924", "45756", "65798", "20123", "68992", "37080", "31257", "31970", "403", "11581", "3111", "25131", "50323", "57039", "20677", "990", "9297", "73058", "30157", "6549", "15400", "61814", "23395", "40471", "48817", "69416", "17986", "62206", "24609", "50469", "9834", "68950", "47330", "49276", "42993", "65570", "26593", "8829", "12368", "57092", "41423", "28077", "55297", "64286", "52574", "11572", "17325", "67921", "31976", "6700", "14046", "56995", "50791", "1852", "14960", "1986", "1055", "65272", "40213", "66112", "69636", "41510", "34076", "63940", "21298", "56359", "12408", "58429", "73116", "63995", "14310", "45664", "62774", "39888", "42915", "40792", "55383", "4359", "38919", "62046", "56948", "55780", "19925", "57680", "35833", "4897", "45139", "10766", "35926", "63532", "54468", "66739", "22754", "60265", "23326", "94", "55516", "39382", "3", "71039", "55730", "38014", "29083", "42732", "68325", "11611", "58767", "36657", "40972", "9474", "56665", "13159", "64222", "41778", "15368", "28125", "63094", "65653", "62861", "45345", "19007", "5772", "29036", "36188", "22268", "56911", "49964", "63510", "3903", "37266", "69685", "65690", "41064", "45979", "72937", "46641", "4793", "21330", "25618", "19744", "65640", "59232", "44618", "60277", "54818", "46262", "53790", "42264", "60252", "235", "31822", "39529", "60805", "72804", "37478", "47166", "24323", "65709", "65189", "7710", "50051", "32260", "8589", "72691", "16040", "22293", "68881", "59030", "63505", "5665", "62550", "36380", "20784", "39045", "23194", "34586", "6756", "59797", "18582", "68734", "46273", "46974", "43820", "4528", "51477", "40257", "10144", "71506", "5340", "72634", "60982", "20069", "9944", "7497", "52342", "38846", "55934", "39844", "64370", "47923", "48298", "69268", "17216", "35101", "50835", "58919", "61599", "68460", "6923", "64002", "56174", "37791", "31057", "43051", "13364", "22141", "70680", "45025", "12932", "51332", "1128", "10210", "72869", "28996", "8303", "2536", "56521", "72599", "41948", "70066", "48028", "68459", "45424", "68324", "24032", "17509", "30427", "61058", "32135", "72697", "61671", "56954", "60034", "26093", "4109", "26542", "5648", "27087", "54837", "2643", "11940", "60925", "49271", "43173", "36287", "34145", "8967", "49868", "57439", "7663", "58375", "42121", "36520", "55933", "20694", "67489", "7972", "16889", "43461", "4245", "22084", "42005", "68256", "33166", "28634", "26581", "73099", "12965", "58683", "51772", "21289", "48986", "32003", "14643", "11499", "12573", "47421", "41762", "44973", "61811", "4701", "25260", "69598", "20812", "32569", "41438", "44598", "60512", "50235", "72641", "70975", "48034", "47227", "71060", "61573", "1309", "52275", "62498", "41522", "65186", "41770", "49831", "48729", "37281", "2949", "27604", "57425", "45569", "22879", "11505", "51918", "70023", "55646", "46199", "57969", "26501", "71874", "71215", "27213", "42944", "38650", "6887", "38986", "51830", "13395", "3357", "7080", "29841", "71880", "58052", "72177", "36610", "50852", "52944", "62743", "21889", "67135", "53759", "50514", "40175", "5419", "56276", "53470", "52678", "67542", "72813", "10765", "39641", "35135", "17037", "42321", "69053", "1494", "7824", "32246", "38966", "46891", "30522", "48898", "9664", "23450", "60622", "36414", "46119", "59123", "61618", "43831", "54077", "64091", "3207", "4015", "48208", "15796", "3459", "53618", "23173", "40465", "50226", "71220", "59509", "5052", "15402", "19280", "35655", "37097", "2837", "21333", "57030", "32518", "29147", "42185", "1701", "4345", "69487", "13267", "7695", "35454", "34365", "36176", "31273", "65268", "9665", "43460", "70092", "47730", "23512", "54707", "15819", "44556", "56223", "42359", "59092", "62250", "43756", "63423", "37842", "72070", "54477", "40417", "11525", "14207", "14768", "4157", "62529", "56478", "26874", "35147", "53339", "16337", "43422", "61740", "14102", "73076", "62061", "69091", "68722", "56803", "21003", "43666", "36090", "31506", "39904", "18416", "12380", "30593", "30310", "69009", "34389", "41317", "48676", "10620", "9999", "15795", "72447", "45853", "67556", "43139", "42079", "37172", "41079", "46345", "4326", "72950", "64140", "14315", "16055", "25069", "5342", "16832", "4192", "50076", "25443", "18761", "19584", "5345", "35357", "11790", "10111", "70801", "65909", "62317", "44406", "54886", "73240", "60397", "37296", "22527", "17251", "1904", "39506", "60220", "26846", "47354", "26618", "2389", "31169", "39720", "39766", "54341", "62656", "36136", "2097", "52997", "64564", "44309", "33840", "73186", "38462", "14789", "19640", "20183", "39717", "14289", "65119", "48006", "41819", "34505", "2353", "18928", "25318", "35128", "11063", "9261", "8889", "42554", "41919", "25567", "32036", "41956", "16460", "27564", "35867", "72200", "27860", "20974", "69654", "1461", "55391", "39861", "65291", "20181", "7835", "57661", "62791", "9153", "20610", "12373", "42239", "1346", "2559", "72442", "932", "67173", "4725", "4633", "17683", "38653", "17", "43806", "46617", "72272", "66147", "54855", "12862", "29984", "71322", "45905", "20673", "44474", "10824", "36979", "72832", "39205", "49615", "14465", "52245", "35411", "45766", "42957", "57511", "25053", "44301", "73177", "22071", "39016", "43308", "6007", "20926", "53932", "67181", "73081", "8857", "30872", "48704", "72452", "55916", "3129", "18702", "72817", "56429", "21152", "14434", "69313", "25461", "45171", "66333", "71289", "60426", "49294", "8432", "20211", "5730", "50971", "17515", "10242", "4813", "22247", "34611", "7398", "2993", "13405", "37600", "2706", "55107", "20362", "61294", "4088", "66764", "71156", "71367", "41576", "31225", "62910", "50928", "14552", "8788", "7175", "6860", "2436", "9418", "72391", "12048", "42044", "2679", "13705", "14743", "36152", "21692", "56699", "21229", "47081", "55091", "68279", "42356", "7926", "40234", "16060", "64042", "10783", "41558", "58774", "31959", "44066", "24769", "66667", "31984", "23805", "73252", "56491", "13655", "30882", "43084", "26759", "1401", "41737", "23774", "56381", "26808", "12924", "18899", "14752", "57946", "32900", "60759", "71109", "30841", "36012", "12155", "19921", "54385", "3951", "21714", "34730", "64605", "51390", "20323", "52235", "50041", "6297", "47601", "7416", "34323", "39813", "42213", "11679", "41861", "29758", "43414", "36973", "69367", "34766", "12779", "12894", "57575", "19335", "43182", "9878", "22613", "66791", "11338", "64322", "6419", "10662", "17887", "51193", "22717", "64045", "18423", "24414", "45463", "48155", "24670", "63143", "40242", "42510", "6858", "29865", "53810", "38606", "70979", "25622", "71504", "11470", "60632", "47955", "10834", "52022", "47199", "11178", "65475", "5747", "56386", "66581", "72225", "13642", "37231", "25353", "22507", "46029", "29717", "54393", "55990", "14253", "46089", "66106", "57417", "22616", "36483", "18523", "40148", "65750", "39505", "73222", "10112", "38733", "21537", "47304", "16209", "13058", "34337", "40363", "71421", "1131", "25395", "2744", "26692", "24829", "5519", "10748", "14416", "25437", "58413", "63090", "10340", "8263", "38980", "31051", "16717", "26880", "60895", "57587", "35972", "28311", "68241", "57399", "66483", "40806", "33075", "66395", "26258", "44337", "39424", "23111", "34168", "19702", "18896", "26232", "41081", "42091", "45958", "62584", "14519", "4800", "2252", "57687", "21813", "59351", "5096", "4288", "69961", "28281", "67385", "22204", "51342", "39669", "68023", "21177", "9419", "18150", "6862", "42709", "18151", "41077", "30289", "35929", "65730", "23140", "70363", "73168", "17886", "44909", "11587", "65886", "18177", "71344", "55578", "70786", "39877", "17556", "36912", "35305", "58831", "33951", "37000", "35165", "55170", "49482", "13972", "17999", "59371", "783", "71194", "3679", "16389", "58668", "55577", "12443", "46061", "11907", "61221", "1847", "14808", "70750", "25225", "10671", "8176", "39952", "71401", "17821", "13085", "72403", "3383", "1286", "21034", "68562", "65011", "35018", "20723", "40948", "17500", "62626", "49800", "14512", "25343", "52746", "624", "22438", "66545", "41230", "1073", "33180", "59671", "73091", "51891", "51905", "2425", "15343", "17812", "52900", "26704", "26768", "64018", "60770", "42131", "72559", "17713", "63273", "50369", "2419", "42463", "23725", "24649", "190", "21162", "69721", "9238", "52871", "72386", "72571", "30717", "56387", "16466", "38298", "38731", "37676", "48183", "13378", "54128", "30134", "31816", "60284", "9507", "61430", "73062", "57308", "29465", "50881", "57389", "39794", "10251", "46681", "14931", "69051", "23502", "39033", "58178", "4810", "23882", "71884", "51599", "20705", "71458", "2725", "45154", "31985", "8769", "862", "51169", "47474", "49801", "68738", "12557", "36088", "15597", "68002", "13649", "14135", "13834", "59150", "17520", "29182", "38363", "57345", "66793", "11664", "60189", "25864", "47792", "16208", "35953", "51922", "60903", "31395", "19441", "51010", "48593", "20578", "38348", "56912", "71475", "50766", "46744", "58595", "264", "49307", "52460", "58130", "51886", "59738", "58761", "18940", "45087", "26241", "37305", "10668", "7045", "1739", "18346", "72419", "24918", "29105", "51021", "18668", "49438", "20231", "19648", "65327", "7020", "16089", "12077", "14115", "2036", "65383", "5857", "31657", "36465", "47325", "17579", "56829", "41877", "67427", "52646", "23879", "51355", "43521", "48947", "18563", "31317", "33953", "46187", "14152", "61874", "48951", "8249", "6092", "72259", "33458", "71235", "34128", "14509", "57733", "39359", "22886", "47424", "52070", "9395", "7261", "58162", "11537", "63540", "56781", "90", "6091", "55621", "62053", "57330", "25663", "37276", "10190", "32857", "19214", "67754", "71279", "5019", "59602", "19578", "36121", "71368", "41173", "64028", "70442", "26435", "60810", "54558", "45914", "30331", "65728", "42379", "7181", "17516", "2137", "61081", "9189", "30824", "3673", "64866", "25210", "13216", "13199", "3822", "13325", "44499", "39384", "42128", "22190", "70513", "63175", "56307", "46313", "44261", "59650", "66132", "3969", "14286", "60568", "68179", "58575", "36588", "70100", "8659", "34394", "20131", "55060", "10004", "37674", "10572", "11458", "24189", "36316", "48520", "24098", "2555", "15613", "70196", "5269", "37783", "63213", "61609", "59405", "29976", "61138", "37169", "49579", "23211", "64240", "37409", "3697", "17468", "50260", "39946", "35116", "8377", "54930", "23145", "52086", "63827", "5845", "45913", "38323", "21778", "27965", "29938", "42082", "13665", "36589", "69813", "16582", "2060", "13077", "50944", "56923", "15924", "42093", "55810", "51413", "13475", "57283", "56689", "23474", "39199", "64111", "59813", "3666", "57628", "59398", "29406", "8579", "13566", "68307", "71478", "39541", "44083", "61610", "69271", "30868", "22434", "44172", "35163", "42263", "66373", "38053", "49869", "41250", "37675", "43526", "20896", "36779", "17377", "16349", "13341", "54840", "67149", "14819", "30983", "68559", "6813", "34054", "57934", "51724", "39481", "44886", "28824", "37777", "49493", "39868", "62526", "55556", "22302", "20607", "62226", "7874", "24470", "3529", "39175", "55912", "33409", "67009", "46734", "7353", "66541", "1313", "8655", "48429", "18736", "38607", "1337", "26828", "19282", "14195", "31935", "45168", "63087", "13132", "59854", "55721", "44998", "30476", "6029", "59770", "5786", "60659", "33511", "21508", "39849", "7529", "52210", "20479", "8026", "47852", "20591", "67851", "58724", "30313", "38453", "32252", "29452", "32868", "28199", "70968", "46302", "49722", "70735", "23744", "67896", "16274", "16214", "29792", "5992", "43646", "30823", "71191", "12271", "34877", "2158", "24626", "66487", "44249", "40212", "36182", "5849", "9887", "43855", "2426", "33596", "19287", "46979", "67631", "33376", "43682", "23332", "22403", "64672", "2294", "30495", "13155", "59029", "45050", "21545", "60175", "28853", "33322", "38545", "33347", "63972", "69699", "9346", "38396", "14458", "30862", "8506", "29077", "24014", "18763", "53136", "72845", "57721", "35087", "35855", "3310", "3433", "66084", "70743", "15068", "45551", "60790", "46893", "34320", "73120", "23486", "62236", "15155", "45260", "19143", "17948", "1682", "55608", "67236", "63343", "26072", "32362", "23670", "21681", "63896", "38596", "98", "51564", "15774", "23741", "36336", "1403", "65753", "44226", "19765", "4050", "25392", "8887", "30425", "67358", "24504", "7902", "16465", "69407", "30859", "35709", "67042", "45140", "61831", "12733", "36992", "39062", "58708", "66994", "70849", "30871", "16016", "1213", "38364", "61183", "5177", "51445", "63954", "35217", "6185", "9360", "1032", "42186", "7337", "12430", "50990", "40840", "2034", "1121", "54963", "3765", "53919", "73189", "24717", "36091", "45321", "40572", "50125", "57879", "13165", "67958", "51519", "56887", "38911", "22513", "44540", "58589", "2672", "31649", "40183", "41324", "69812", "24397", "65736", "17032", "22236", "62837", "36864", "51271", "62734", "14150", "51196", "45212", "60690", "40794", "56662", "30035", "47255", "3575", "21307", "23969", "37911", "15518", "23130", "50222", "18320", "29576", "24439", "3875", "4478", "51928", "35943", "29556", "70744", "5010", "33024", "35609", "5594", "616", "25399", "72287", "20405", "23128", "13722", "23456", "29503", "32204", "28959", "262", "40935", "52788", "33257", "50838", "31553", "64159", "37459", "52104", "28822", "70774", "38787", "69872", "49147", "3715", "47502", "12224", "15657", "20446", "43708", "22859", "68079", "17358", "4595", "58439", "67545", "15567", "7190", "59733", "62655", "64943", "65676", "5412", "3301", "20341", "13274", "20587", "70940", "44636", "45770", "29479", "1877", "37164", "32087", "3596", "62899", "50516", "23726", "33119", "12754", "2991", "65587", "36941", "36648", "43360", "64202", "5084", "17235", "25418", "20632", "33449", "63744", "24048", "42134", "66891", "51298", "64103", "31286", "60431", "9635", "24950", "67373", "41233", "26040", "53185", "26787", "12037", "45934", "55559", "38704", "41827", "12390", "54549", "34511", "42607", "60439", "14809", "59784", "10104", "40429", "59044", "13289", "16533", "55108", "44391", "56904", "12866", "44459", "3837", "52898", "9817", "50720", "773", "23081", "60247", "9980", "72910", "10392", "7109", "62709", "55463", "3351", "22387", "28663", "39336", "51471", "35076", "47803", "45886", "12039", "70551", "26877", "34793", "28534", "39042", "64248", "38040", "10346", "71328", "8475", "67954", "60587", "65682", "45429", "35458", "48492", "5100", "69888", "2878", "64031", "65996", "58885", "18104", "4837", "64095", "1709", "1392", "27544", "32956", "3742", "59269", "36813", "26953", "54022", "36615", "16793", "28811", "64206", "6569", "24486", "21192", "25535", "51431", "30235", "38461", "56971", "35928", "22532", "12468", "20476", "33912", "54122", "44471", "30870", "44538", "27145", "29885", "17927", "70173", "39881", "68627", "25687", "49791", "38612", "39673", "67581", "15701", "9625", "47461", "61833", "37061", "35841", "29728", "19460", "19498", "66262", "19590", "41348", "690", "53290", "42811", "32416", "40299", "7480", "48056", "52295", "18602", "31619", "916", "4165", "31702", "4944", "31409", "51855", "50293", "10324", "6507", "64474", "54457", "31802", "61635", "64678", "53652", "35020", "70030", "57158", "42466", "26504", "13774", "22673", "25337", "63900", "2270", "46962", "43985", "30598", "4277", "38076", "46884", "71509", "26822", "47357", "41307", "72949", "46998", "18116", "19466", "25930", "10213", "5118", "53907", "63964", "11990", "60578", "41740", "47400", "17256", "25527", "29698", "53441", "2879", "59644", "56998", "20026", "3562", "69862", "45485", "8221", "14396", "39480", "43434", "65842", "59294", "8602", "26716", "73176", "55617", "6908", "50889", "27884", "44468", "53242", "35835", "46161", "32219", "46990", "21403", "5156", "23876", "35916", "71253", "71782", "55330", "59221", "64369", "57886", "54667", "5581", "11460", "30896", "21365", "67796", "68262", "36358", "24060", "17068", "39067", "63500", "27395", "58442", "12204", "19701", "69564", "25407", "39134", "53311", "53905", "50748", "12236", "22348", "35412", "57474", "53409", "48537", "60687", "65685", "29192", "27246", "25581", "44300", "62406", "71727", "30908", "39658", "48377", "56247", "69410", "14443", "58168", "14091", "65278", "61490", "15043", "63383", "13521", "62780", "15101", "62615", "8457", "4320", "12849", "4052", "50456", "15393", "56566", "18956", "57423", "30066", "12375", "24328", "15822", "16995", "34057", "40041", "30394", "41595", "42756", "32125", "15955", "62654", "27545", "40466", "41981", "62246", "147", "51227", "54471", "65611", "62043", "19619", "6366", "33949", "50146", "2422", "16398", "23626", "31024", "15743", "59686", "50593", "19241", "21524", "4421", "21283", "2427", "22579", "47895", "42744", "34679", "28797", "70709", "33502", "23870", "10945", "28160", "33419", "59343", "20659", "55565", "72459", "25204", "59892", "2834", "56742", "24355", "25149", "35888", "18489", "68063", "4705", "9187", "1421", "72956", "45532", "3835", "7122", "64806", "31982", "12602", "41312", "40130", "65437", "30676", "14266", "69055", "42610", "62600", "63924", "69079", "26316", "38425", "66073", "10256", "65614", "31091", "24112", "56789", "9313", "33816", "58010", "4370", "62152", "61723", "57741", "72387", "7019", "29975", "70309", "68226", "11805", "29660", "52488", "19323", "20669", "28820", "50700", "28063", "58552", "61924", "1148", "39732", "4612", "56707", "1657", "63668", "33958", "37537", "69815", "64652", "67248", "63466", "24945", "45877", "32445", "57286", "4083", "5838", "67070", "31690", "28645", "56593", "16081", "10312", "66781", "28803", "55776", "38538", "32389", "25122", "47587", "27554", "60157", "18510", "71500", "45636", "6333", "38488", "43973", "10580", "60442", "70125", "48494", "43090", "31315", "8656", "43288", "55978", "59975", "61040", "70211", "61470", "509", "13089", "36023", "46697", "70350", "33734", "41334", "35651", "20596", "13396", "29790", "15338", "23901", "14997", "7273", "32374", "4076", "18204", "4937", "57717", "23062", "16624", "42577", "30503", "56333", "66425", "41523", "63372", "40133", "33975", "10650", "4964", "21440", "15753", "11147", "15788", "39287", "1322", "24257", "18982", "3062", "35345", "14329", "30654", "58882", "4541", "25934", "21160", "10989", "21718", "1736", "14442", "59532", "33738", "22577", "68417", "10454", "60772", "37233", "36325", "36915", "69174", "33727", "65784", "4286", "53227", "67499", "3194", "41023", "28082", "20927", "40449", "64376", "27336", "55084", "65883", "53366", "37878", "54388", "57163", "37192", "9686", "72269", "20955", "61669", "34603", "39200", "69134", "42443", "62947", "55795", "26888", "59616", "70213", "65128", "34167", "69275", "50014", "431", "47407", "30811", "25228", "3267", "10720", "41167", "68051", "2018", "40831", "68509", "28697", "29623", "71829", "47634", "1605", "45796", "56295", "59875", "31374", "59664", "31076", "57083", "24694", "65378", "64927", "63003", "36128", "641", "69489", "12905", "498", "40688", "31062", "40470", "52695", "2792", "24618", "24364", "56443", "49798", "67436", "7364", "37306", "43838", "2827", "44587", "16608", "2484", "37364", "50123", "68843", "45179", "9666", "48983", "26865", "7842", "13381", "30952", "24027", "23792", "50259", "44813", "60735", "19129", "9958", "62573", "2259", "17627", "70337", "15034", "1931", "8872", "2271", "7818", "46770", "40102", "27514", "71079", "13334", "18315", "49333", "40900", "61613", "22261", "41628", "39140", "30981", "71591", "41820", "43582", "39976", "68073", "44265", "56701", "34004", "6822", "45227", "5425", "24853", "9424", "67380", "58853", "18927", "56517", "32164", "41278", "19729", "63779", "39381", "63163", "24126", "48762", "28010", "48048", "5245", "38245", "36044", "30651", "56149", "35221", "36976", "6912", "28065", "6014", "2369", "67626", "17789", "32567", "19535", "66251", "57522", "62547", "35507", "31456", "38056", "7779", "62513", "32462", "15698", "55752", "10428", "4393", "18757", "38530", "12377", "27088", "25335", "33557", "40674", "43647", "46914", "28643", "54446", "45446", "51998", "12118", "15257", "21805", "56706", "66433", "44437", "1519", "32118", "63428", "45995", "30610", "1649", "43112", "2720", "43331", "17792", "55218", "35896", "39782", "7563", "15354", "23033", "36486", "36006", "11082", "43850", "55450", "42656", "33924", "10086", "19073", "38991", "35315", "63728", "23905", "54524", "60715", "48623", "52644", "5523", "30131", "40609", "41901", "18818", "44451", "19140", "24412", "49452", "43343", "32137", "26816", "68862", "53300", "35386", "38387", "16031", "11468", "55600", "55811", "55304", "63151", "34572", "47196", "51241", "27531", "53925", "16238", "25806", "64552", "47087", "9949", "40582", "42236", "44140", "54588", "31773", "34298", "61665", "21016", "62366", "45578", "70715", "34489", "7147", "24339", "28525", "22360", "39354", "48309", "12797", "61087", "38279", "15549", "59933", "58584", "67620", "52000", "32985", "71982", "50079", "31695", "25117", "5810", "59283", "36289", "22512", "45930", "22332", "14919", "54016", "50084", "43660", "49653", "50712", "51035", "45926", "71584", "17667", "58680", "3163", "23610", "29125", "22249", "64311", "9633", "11009", "53902", "53080", "26009", "65591", "33571", "39696", "56166", "57295", "14676", "72313", "29814", "60411", "64653", "24990", "4099", "2525", "7391", "63185", "20707", "48786", "60594", "36962", "14529", "18838", "69793", "16829", "67394", "704", "57065", "33914", "72008", "17258", "38142", "39677", "64464", "67678", "35006", "39297", "28393", "563", "34751", "71565", "27451", "21918", "48251", "3860", "30254", "54656", "24743", "33565", "13414", "25193", "53943", "59460", "47204", "2532", "20126", "60528", "55788", "3114", "69920", "22142", "17588", "13012", "58956", "37645", "16136", "6190", "5109", "9826", "22470", "38848", "11067", "65758", "50951", "13687", "53703", "62500", "66396", "31234", "35117", "63599", "45395", "6286", "35910", "19643", "26153", "15431", "15911", "49426", "28796", "44964", "44518", "25602", "63390", "69683", "11952", "38503", "28101", "2038", "40100", "47201", "29109", "52166", "71054", "58679", "65445", "47167", "65737", "47998", "17881", "68075", "586", "66096", "59490", "5670", "2096", "17455", "974", "39204", "64793", "25521", "36037", "52330", "63074", "12835", "49284", "9089", "21820", "54871", "391", "12138", "68978", "43228", "62011", "14668", "4425", "34846", "19641", "5256", "59815", "37418", "10797", "24895", "51062", "70460", "14823", "11031", "64437", "28298", "16981", "29742", "29063", "4844", "69074", "71920", "52198", "70953", "22819", "34641", "26646", "12587", "29142", "37677", "44762", "39872", "67090", "12756", "71150", "14986", "27943", "70331", "71936", "52322", "29971", "55163", "26992", "34113", "72257", "54794", "54910", "20124", "53329", "44199", "55464", "3847", "22397", "6313", "57010", "27489", "2012", "23445", "66766", "36882", "13212", "46456", "31668", "17437", "41351", "71465", "8111", "22892", "72140", "39896", "49318", "47150", "3692", "66718", "55863", "12676", "13043", "26582", "41396", "16017", "48660", "49550", "9376", "54807", "34520", "6808", "62168", "14500", "29197", "52369", "12516", "5802", "4223", "63686", "39926", "70613", "36460", "28119", "48061", "51701", "14523", "45680", "58025", "47918", "50867", "19932", "38148", "27328", "45808", "68537", "12322", "56313", "31153", "28922", "12005", "73113", "23881", "35812", "45751", "27352", "62432", "63337", "35350", "71948", "19830", "38854", "40847", "1214", "7502", "70359", "63853", "41157", "21349", "71159", "31365", "28462", "37870", "42850", "38038", "71337", "2799", "69239", "16236", "61436", "4671", "25018", "44347", "19673", "39840", "48840", "52934", "38938", "47145", "25607", "57103", "1950", "36511", "45971", "1220", "14883", "35372", "30625", "7277", "57872", "6622", "47670", "32382", "15997", "45956", "67600", "20951", "49198", "17841", "18240", "27483", "35215", "48826", "44667", "65267", "34274", "4707", "57437", "13902", "46051", "3570", "45120", "23636", "5405", "47355", "38987", "7100", "47332", "72519", "57303", "7427", "55176", "68223", "62530", "62283", "68683", "45240", "14877", "6805", "29830", "51256", "13731", "20795", "68740", "30161", "9616", "464", "7759", "35786", "16904", "20516", "69963", "18575", "31780", "72129", "52148", "23789", "49757", "1816", "70527", "19169", "41315", "45845", "51938", "51541", "59554", "44902", "37866", "64721", "65951", "70177", "27831", "21076", "62636", "64024", "27388", "47528", "54703", "72483", "12838", "46168", "36892", "15598", "41368", "25346", "268", "22943", "60203", "2497", "70394", "27695", "21256", "50415", "25327", "44245", "4206", "8724", "53118", "51157", "21461", "67832", "69328", "50758", "13352", "21292", "18122", "26997", "25044", "2514", "69545", "9647", "36422", "55864", "19690", "35866", "54427", "39367", "41399", "1079", "9491", "23026", "65830", "16715", "29253", "42828", "62197", "69597", "28247", "5945", "844", "4687", "13328", "13233", "16281", "59689", "16997", "58003", "50227", "11869", "26907", "15973", "25009", "8478", "40581", "8683", "3088", "204", "16354", "25475", "55110", "68574", "26985", "29955", "15115", "3629", "20881", "22116", "35427", "15705", "70880", "63737", "62857", "39497", "28526", "23266", "48514", "4737", "40012", "60183", "5650", "70099", "12952", "8390", "53656", "70652", "28262", "71240", "28407", "50952", "17748", "26418", "28409", "51307", "64930", "2151", "26516", "21175", "49564", "46144", "48133", "48300", "221", "50507", "37436", "26329", "14264", "20903", "68219", "55511", "15475", "10502", "5602", "16066", "66658", "2185", "33462", "65392", "18995", "41904", "72083", "54013", "3074", "17188", "44690", "18441", "41475", "14610", "42662", "67601", "14941", "35771", "11048", "28161", "291", "46977", "11988", "38689", "59480", "32477", "49833", "10056", "39049", "54359", "42739", "38218", "56926", "33547", "4536", "47057", "24218", "61622", "57386", "18646", "35890", "49031", "40958", "28198", "19924", "42824", "68920", "1664", "12072", "49810", "23082", "23446", "72500", "51380", "72518", "13599", "61882", "35798", "46772", "24417", "1331", "23428", "53550", "69720", "8881", "20197", "73228", "9473", "9780", "1635", "47576", "16454", "46792", "66857", "26120", "31897", "20077", "49388", "4772", "435", "26395", "1157", "9738", "12499", "71461", "70171", "13104", "18126", "58117", "58662", "23667", "55209", "24955", "132", "1495", "21138", "42008", "11868", "72142", "70758", "55099", "44115", "22607", "45634", "8032", "53698", "28881", "58098", "19503", "10871", "3989", "8968", "41459", "21561", "26797", "31685", "38703", "20864", "19101", "18278", "52940", "58315", "50702", "21109", "49787", "11332", "448", "47529", "4787", "21092", "12818", "8326", "35049", "7703", "12758", "41158", "39316", "55445", "25725", "15733", "20777", "72624", "18991", "70285", "45738", "44838", "35346", "7524", "58338", "45008", "1434", "39398", "32326", "59595", "49151", "46846", "44233", "46007", "29782", "63375", "24959", "48613", "34634", "9757", "59364", "25711", "20037", "43134", "58516", "40912", "1466", "59124", "46531", "17148", "12804", "40180", "55388", "54078", "777", "10750", "31618", "60457", "59360", "50972", "39637", "60934", "50460", "46037", "35269", "20490", "62921", "29670", "36911", "47886", "37269", "14735", "29573", "44217", "8041", "35023", "21818", "13514", "65408", "43354", "41931", "1517", "58504", "26279", "69369", "66594", "27296", "60033", "48620", "12437", "14094", "31880", "32001", "48023", "25660", "11687", "62923", "34554", "32231", "10965", "22311", "24949", "30123", "670", "6510", "70562", "65224", "61631", "60317", "48859", "26233", "4956", "57144", "39413", "3227", "15948", "23918", "3209", "72416", "64651", "65248", "9728", "39622", "26379", "65865", "15718", "65072", "28664", "43124", "19264", "70664", "3218", "7819", "65013", "45383", "55009", "50717", "41894", "54098", "73233", "2274", "70669", "11808", "43813", "52053", "55737", "35755", "63022", "24459", "65476", "33865", "14056", "68439", "277", "62300", "42022", "51142", "59323", "48148", "25600", "27356", "25588", "56562", "15038", "30555", "48115", "56877", "44876", "72373", "37283", "38382", "28751", "39366", "66790", "59228", "10971", "12158", "66595", "47901", "19811", "18933", "68925", "7713", "13609", "2262", "55941", "42796", "33601", "67267", "68401", "10666", "11475", "47744", "44976", "73134", "62633", "55891", "13300", "56642", "12454", "25895", "24069", "55586", "2091", "70693", "71539", "44758", "24936", "34424", "68162", "72431", "10958", "49541", "9267", "28501", "3516", "5981", "40349", "43247", "30200", "40108", "18811", "18704", "48079", "67529", "5999", "39757", "17853", "7056", "16308", "2491", "37800", "58982", "10186", "17489", "64775", "29729", "62971", "56236", "71917", "15454", "34654", "71585", "62419", "38276", "62697", "10254", "47140", "24457", "29423", "31843", "53439", "12107", "57807", "47533", "51901", "2288", "72584", "32442", "38497", "38672", "62877", "61391", "28042", "41991", "25824", "31325", "51030", "46709", "36968", "18146", "58256", "38028", "18894", "46603", "72015", "21737", "65917", "37309", "65695", "53889", "22592", "64141", "70049", "62591", "14744", "4977", "45221", "13213", "1257", "69129", "15916", "65173", "70074", "43750", "44494", "53356", "49697", "53939", "67779", "7851", "73150", "26489", "22118", "40296", "57317", "66323", "27302", "68777", "37482", "27953", "6875", "21380", "54207", "67043", "42771", "3571", "40976", "56759", "70003", "49449", "11932", "42924", "22333", "66884", "54756", "69684", "69351", "286", "65451", "63880", "38048", "49289", "5455", "14470", "37659", "11389", "11360", "24818", "34839", "34717", "69473", "22290", "54604", "14887", "68072", "37776", "30309", "19443", "20953", "69357", "12916", "61147", "65174", "12948", "14404", "26321", "6758", "37711", "6737", "60316", "26183", "33733", "29177", "26167", "10604", "45409", "49888", "22313", "45100", "15520", "71429", "43675", "25655", "10425", "28794", "19843", "19308", "43219", "48015", "65932", "19650", "64795", "39787", "34245", "47256", "52606", "14904", "21837", "14383", "53969", "17715", "30685", "65017", "69359", "62404", "16064", "44770", "50403", "48197", "40930", "37297", "73020", "44102", "66898", "38030", "58852", "2143", "36565", "2696", "49553", "35568", "6787", "48510", "24925", "63245", "55658", "19224", "2582", "19184", "49061", "61847", "34670", "14537", "9986", "57632", "7374", "21123", "38214", "9114", "27929", "52209", "61116", "62563", "48659", "17302", "26554", "25024", "27851", "13652", "59603", "5489", "60546", "31002", "41856", "13122", "13124", "40453", "40730", "36253", "42890", "8018", "58430", "45125", "36704", "44396", "2630", "7324", "27961", "46208", "6267", "34978", "44502", "30229", "52029", "17284", "54929", "5128", "35303", "28552", "34657", "53141", "726", "20293", "64415", "62712", "69105", "20028", "60329", "44986", "23402", "59728", "44100", "4475", "25526", "9685", "69289", "32592", "51366", "18989", "29685", "55225", "60986", "25819", "70339", "22930", "10120", "31466", "44822", "5040", "2190", "33697", "14613", "25595", "22677", "37780", "24074", "60695", "41569", "52247", "19534", "4118", "43171", "18734", "46826", "626", "30826", "16388", "13430", "57139", "47838", "68221", "56004", "70573", "592", "59023", "59467", "62689", "14706", "22534", "30748", "63125", "37803", "47156", "26995", "27130", "26762", "12510", "19992", "12164", "18119", "26981", "49140", "2690", "16649", "55016", "32738", "14414", "30245", "18949", "14913", "32335", "49840", "31184", "45203", "33090", "61750", "4305", "30153", "27597", "33804", "20305", "10701", "33345", "42747", "62570", "62029", "42193", "48366", "36327", "40297", "71840", "23045", "10285", "19313", "35487", "56469", "71333", "26825", "68012", "64226", "7954", "47666", "19829", "21059", "22676", "54302", "10079", "37453", "3200", "52062", "7572", "63986", "6235", "41943", "51995", "32211", "71639", "63925", "38995", "6643", "41582", "21498", "46835", "60242", "28668", "653", "59663", "35509", "20098", "31895", "25929", "43667", "15467", "20184", "50400", "29711", "68970", "38185", "42884", "6722", "6564", "61007", "14445", "31971", "48677", "69375", "12462", "3803", "3264", "2293", "50686", "60422", "37420", "71499", "30761", "57400", "31768", "1944", "9554", "37454", "1468", "66229", "28231", "31178", "2691", "16903", "23415", "38942", "5433", "26428", "29751", "3890", "27844", "49769", "25366", "24221", "21705", "39742", "57531", "41705", "70296", "8431", "16908", "67228", "36070", "2899", "41987", "6843", "61284", "64864", "40723", "10777", "37105", "72220", "70958", "70585", "20023", "3272", "29665", "33137", "50196", "38726", "42561", "67853", "63059", "8373", "31723", "8963", "49277", "4545", "50998", "16144", "2129", "27464", "53379", "37203", "21286", "38278", "51190", "4017", "12531", "25624", "20330", "17067", "62397", "25744", "57577", "14153", "1459", "12528", "55943", "70329", "11678", "32460", "53576", "25237", "48633", "32387", "10048", "58293", "60307", "53097", "10518", "19628", "57605", "54988", "72205", "68696", "66432", "30650", "38058", "46872", "62527", "40782", "20230", "2946", "1824", "70544", "58340", "31073", "70207", "69076", "31414", "38724", "15395", "28904", "53992", "18267", "59777", "26597", "55991", "41646", "69891", "51448", "66614", "20487", "31172", "8623", "72713", "14078", "38472", "7878", "54838", "73104", "68102", "13551", "44292", "8901", "11903", "29421", "29998", "47298", "40447", "46673", "2331", "21892", "44773", "54276", "52782", "26139", "18129", "124", "66083", "23839", "54885", "7936", "32885", "5606", "65689", "21482", "19243", "41089", "34625", "11555", "18705", "47205", "4203", "18801", "54594", "33160", "22949", "36785", "72985", "10884", "22872", "67162", "68144", "66779", "70812", "23291", "35629", "36852", "61517", "25029", "40643", "51754", "38660", "58797", "72383", "6181", "70636", "47123", "61844", "37589", "52987", "49804", "33630", "6546", "7526", "51119", "10786", "57", "27741", "53869", "65930", "64458", "53483", "69846", "41414", "41541", "56631", "60699", "11895", "18802", "61205", "36698", "60444", "2835", "37964", "53982", "23796", "71093", "68550", "57938", "42923", "28438", "50958", "5072", "2483", "52829", "40246", "67422", "55593", "30919", "46515", "11467", "29207", "33238", "56677", "5876", "22805", "46099", "36861", "44507", "21911", "68952", "31943", "51853", "1389", "40872", "13091", "62755", "2145", "57285", "60762", "69307", "18145", "35691", "56787", "29205", "42936", "4630", "9199", "12624", "15214", "45214", "22266", "48802", "12953", "1833", "1989", "8256", "42099", "2868", "57126", "62569", "25553", "14775", "31733", "58380", "56457", "40789", "68031", "18524", "68455", "8577", "56700", "55503", "52858", "12046", "64196", "71781", "30956", "19768", "55686", "11519", "39213", "36210", "35940", "65788", "9700", "53513", "31622", "3945", "46725", "26891", "4331", "50595", "47740", "31210", "56020", "32516", "26470", "57157", "53523", "62190", "22593", "29252", "26771", "70560", "21846", "61062", "42558", "46558", "24406", "3857", "34183", "27150", "4108", "41100", "18091", "46043", "25643", "26922", "45079", "4603", "26498", "62422", "69881", "6816", "62099", "32399", "10717", "11420", "30063", "3854", "38550", "41498", "7224", "910", "24419", "71078", "15217", "68145", "45392", "34580", "34680", "22557", "7771", "33899", "41831", "18543", "36706", "12749", "61820", "23753", "2905", "1039", "16079", "41408", "22020", "63936", "7623", "56748", "40218", "39680", "67423", "33383", "31869", "18722", "32756", "67574", "10041", "31838", "5873", "64893", "125", "60573", "24545", "41761", "49060", "34051", "55052", "66940", "60228", "43563", "20758", "42753", "15649", "41241", "34494", "56485", "37552", "69228", "36665", "64828", "56032", "30198", "38738", "4075", "17672", "71501", "46542", "839", "57044", "20057", "63045", "58779", "64863", "52809", "70917", "7967", "38780", "58534", "49809", "9456", "51012", "6406", "20242", "66324", "46297", "21018", "33201", "22561", "43274", "41559", "5117", "65065", "21031", "39674", "2591", "43815", "18823", "60084", "28896", "47595", "8841", "3478", "6577", "27276", "23978", "71173", "57374", "32051", "21043", "56200", "25141", "3147", "66452", "3179", "17973", "54887", "35752", "28841", "46508", "50601", "30388", "35832", "72766", "30703", "34304", "42720", "28377", "33089", "36820", "73183", "12571", "28580", "3063", "30718", "31332", "70458", "35258", "48889", "25455", "3940", "17686", "67088", "67331", "2346", "59107", "43518", "3362", "26761", "73223", "40733", "41712", "72376", "6480", "45700", "51046", "55479", "30800", "66977", "68492", "51522", "3677", "52041", "70077", "40159", "36303", "56261", "51456", "56765", "61972", "50216", "68427", "39821", "19671", "31056", "16919", "61841", "52461", "56050", "26050", "71119", "19346", "51739", "5899", "59446", "18167", "63887", "11829", "42313", "11691", "31975", "24120", "46166", "19093", "19063", "65088", "69127", "56330", "46496", "3956", "6902", "21829", "34827", "42484", "23998", "30341", "19726", "42416", "50209", "23043", "65969", "66139", "6593", "72797", "11891", "43594", "31268", "65564", "18622", "694", "2962", "65593", "64412", "27010", "67419", "41552", "12540", "52598", "49821", "40791", "24309", "64264", "31860", "49278", "56907", "16435", "50074", "32534", "7986", "19326", "5385", "27815", "31068", "39416", "56939", "8214", "19907", "69955", "43244", "39306", "6645", "30481", "61022", "3740", "13623", "69389", "3982", "48345", "23814", "24761", "64594", "55876", "38484", "42887", "21193", "48313", "11860", "51751", "57967", "336", "51409", "25857", "28026", "46753", "68211", "9045", "32799", "70373", "54245", "6237", "12487", "5984", "60064", "38552", "20257", "28983", "29541", "65271", "10865", "32188", "62762", "23575", "26651", "66629", "24002", "50575", "71801", "21617", "55814", "5376", "55022", "4648", "8136", "65275", "34351", "47524", "1140", "5545", "18759", "50184", "45772", "67524", "40771", "50840", "56220", "2746", "56191", "45855", "40206", "29281", "28738", "3112", "43804", "59747", "34315", "4374", "72422", "44825", "40262", "1713", "60185", "30934", "39901", "59270", "53316", "54962", "49731", "42659", "8450", "65461", "22011", "1325", "34366", "33510", "27960", "13390", "33182", "20085", "57315", "66552", "15517", "5905", "29143", "3175", "13767", "59422", "60827", "3752", "69681", "40570", "12177", "46155", "54301", "14567", "62158", "61615", "12918", "29320", "1347", "32009", "70229", "64996", "60680", "23627", "24013", "56992", "48193", "6282", "20522", "50448", "27102", "35848", "19316", "11256", "47097", "67520", "11507", "65906", "44380", "34061", "39810", "68819", "43116", "70091", "31116", "22273", "2888", "36555", "67244", "12260", "68306", "54336", "1024", "56326", "39068", "22810", "53929", "47135", "42455", "7432", "70711", "59167", "36131", "48976", "41695", "20847", "30262", "21158", "53034", "51258", "67915", "30466", "11482", "22210", "28808", "64789", "61591", "58749", "39797", "4284", "16626", "5509", "39320", "40344", "6071", "52544", "30239", "45444", "39103", "32517", "68568", "27859", "16161", "29918", "62677", "5366", "5503", "34950", "19354", "66522", "47716", "37497", "22436", "11873", "27156", "62842", "48546", "26206", "33695", "8224", "65789", "24723", "43356", "30292", "53413", "7530", "7543", "53017", "17493", "20888", "41815", "25558", "64588", "23358", "59223", "45272", "22756", "32934", "57809", "15829", "45130", "27477", "37256", "22418", "1581", "27500", "35729", "25185", "17553", "30691", "16994", "20381", "16947", "33514", "22747", "6778", "36260", "72901", "63641", "5695", "72909", "33876", "15056", "25079", "3560", "67571", "5956", "40929", "68236", "37732", "40927", "57229", "27640", "43252", "6075", "32412", "7027", "63129", "12536", "12448", "33111", "44401", "67712", "47436", "16410", "70707", "12875", "72533", "10483", "64237", "18000", "56953", "2830", "13931", "51933", "43986", "41242", "23039", "46759", "58449", "5301", "30634", "36651", "31523", "2221", "46894", "15375", "38716", "61554", "36417", "30947", "31295", "399", "27951", "65927", "64824", "62079", "10482", "6148", "23764", "33882", "32271", "50544", "3091", "9066", "25720", "51824", "37982", "3974", "37604", "59701", "5271", "72735", "21212", "52259", "42918", "15809", "20851", "68327", "27734", "11427", "66839", "8722", "9000", "54627", "24953", "24960", "41876", "25389", "66937", "67458", "28845", "64543", "27066", "62123", "40209", "178", "2496", "61848", "65053", "24691", "43969", "31872", "519", "612", "55970", "40992", "29092", "46510", "35172", "46778", "39019", "52509", "17379", "20891", "34480", "41367", "52345", "3635", "2959", "48046", "1238", "6846", "9559", "53553", "41968", "21022", "22650", "45970", "69627", "39063", "7095", "69869", "24543", "3950", "59688", "2699", "31047", "71046", "28031", "1484", "37030", "13656", "28998", "45694", "66646", "50311", "44710", "52272", "28309", "12994", "32232", "23664", "24005", "62177", "2919", "8081", "44470", "7762", "22693", "11211", "33862", "47742", "42211", "28378", "54141", "64991", "60800", "28108", "60464", "30390", "2451", "43828", "1212", "318", "361", "37351", "34764", "60062", "21824", "31006", "33101", "14682", "63635", "32611", "9797", "13438", "2182", "55697", "12565", "30780", "27663", "6110", "46560", "48380", "54500", "50940", "30534", "50508", "62608", "486", "58841", "51746", "68822", "10731", "43393", "11729", "13597", "40362", "1504", "22564", "47780", "20033", "62271", "12930", "72082", "1116", "33748", "43359", "25214", "18547", "60548", "29810", "34702", "70140", "24924", "48084", "71973", "3688", "10615", "21201", "27090", "72429", "1344", "4341", "20637", "71463", "3066", "52712", "42058", "40520", "49313", "65850", "20528", "24298", "66988", "12788", "70627", "36502", "19824", "38395", "49416", "13111", "56352", "29614", "54542", "29966", "22187", "505", "10864", "23932", "19927", "59068", "73132", "30279", "21961", "10477", "48233", "56594", "51367", "65107", "66708", "66561", "45329", "55543", "49611", "19912", "48870", "30920", "11892", "19737", "53946", "55112", "7165", "50336", "45729", "19376", "34769", "37592", "60720", "35265", "44060", "11931", "43399", "70071", "68938", "23200", "66554", "63579", "48338", "46394", "24468", "14357", "35426", "28016", "23727", "16074", "219", "33614", "51634", "50678", "36282", "20778", "18603", "45162", "28232", "27106", "58657", "2609", "29909", "15298", "27455", "60959", "43371", "1636", "5176", "2978", "60666", "64508", "26520", "50431", "5543", "22784", "52863", "14772", "35913", "37052", "24321", "21367", "24059", "19522", "71050", "47706", "7602", "27304", "58322", "5465", "9170", "64924", "12836", "63429", "17477", "20715", "72231", "64121", "23495", "62452", "4449", "60088", "69466", "63578", "48062", "23086", "42919", "44368", "66225", "17617", "43803", "59437", "8694", "545", "50538", "65381", "50542", "8395", "60076", "8917", "59918", "10426", "47579", "71980", "71894", "14255", "22538", "31681", "36206", "66285", "11549", "59389", "21006", "57517", "65214", "35584", "40586", "16683", "60498", "10302", "1249", "60479", "28306", "19998", "64283", "33833", "23314", "67805", "7188", "48511", "46234", "21375", "10017", "7774", "39716", "22958", "43929", "27242", "34211", "65290", "57550", "659", "10099", "65814", "3197", "54142", "64408", "22515", "15382", "48237", "30306", "71523", "26308", "52708", "32573", "63837", "28937", "27075", "1889", "17900", "4878", "66417", "43327", "31359", "15419", "56782", "44432", "58905", "14235", "34433", "11740", "18459", "54419", "779", "46086", "1747", "56137", "7397", "24563", "26207", "66905", "51232", "38307", "56287", "8313", "63546", "15794", "66864", "39887", "28499", "67892", "8535", "15900", "45750", "13746", "41491", "10627", "65087", "55069", "20621", "6995", "28537", "879", "43357", "61769", "17059", "16732", "6839", "72699", "8340", "4251", "34273", "3492", "37193", "31992", "44264", "45117", "46708", "58713", "43511", "53365", "44190", "58111", "19521", "43085", "23290", "42204", "71377", "61464", "10316", "38259", "32075", "20513", "29737", "3686", "65028", "44613", "5258", "54206", "28225", "46147", "42332", "46339", "50922", "70036", "61793", "28903", "19012", "50739", "63945", "56325", "46677", "55259", "71255", "30967", "5796", "32888", "32006", "39508", "44339", "46857", "23037", "57201", "14820", "27656", "7239", "21263", "28707", "40223", "23074", "24271", "63493", "60609", "10074", "41133", "13966", "7793", "16974", "45704", "57206", "52405", "52162", "71022", "66882", "46713", "62100", "7718", "45106", "40973", "26843", "73059", "29351", "70611", "55115", "12889", "18864", "27789", "44599", "5687", "66729", "37425", "38152", "58583", "4198", "6611", "64163", "72204", "44905", "10728", "44402", "48681", "61057", "55651", "15201", "15090", "25413", "47159", "50239", "65078", "58523", "17975", "49431", "47785", "43738", "41231", "9808", "59862", "47992", "45754", "46235", "60331", "48562", "60159", "1818", "51793", "70987", "59942", "51073", "54343", "10804", "18218", "70327", "25969", "63413", "72030", "12479", "3938", "66647", "50199", "42629", "11775", "23413", "45422", "8324", "72354", "57865", "19049", "56633", "16902", "68625", "5266", "49596", "63935", "65870", "52750", "5587", "31568", "18444", "26039", "31252", "19111", "54998", "9047", "8935", "685", "42303", "55199", "20361", "43312", "6065", "63047", "29506", "68286", "66676", "33486", "1240", "70587", "54160", "1440", "42050", "32751", "13779", "24087", "68533", "63713", "7264", "53164", "12051", "53782", "18060", "33492", "44912", "24", "59478", "48128", "55642", "42055", "60313", "49116", "20142", "51560", "31764", "41657", "22490", "2666", "26826", "52375", "37805", "61728", "19519", "15039", "61479", "14342", "50496", "54166", "1085", "53317", "47235", "60418", "25347", "8253", "46701", "57837", "7688", "66178", "60010", "57698", "21797", "17622", "41654", "5355", "67286", "11089", "30073", "23189", "67666", "47339", "29117", "54045", "23526", "69630", "27178", "19989", "7008", "37704", "56234", "60256", "65092", "18992", "9441", "26730", "21761", "38844", "16419", "39102", "65305", "63648", "17431", "44447", "13088", "27360", "26949", "35338", "6395", "50796", "64020", "27583", "48590", "55120", "61674", "63687", "41627", "46479", "27077", "37808", "8411", "14314", "56413", "14250", "68864", "6379", "66657", "8351", "59641", "34830", "47170", "61668", "57426", "61897", "72388", "9828", "26693", "21654", "21579", "363", "7439", "45541", "9465", "52945", "63637", "6909", "35314", "63118", "46499", "2908", "64253", "32128", "36757", "41570", "37700", "21119", "68479", "72280", "22150", "6089", "22251", "37375", "33863", "51470", "22041", "52967", "41602", "36538", "31397", "60538", "36875", "61810", "22868", "21626", "57796", "68382", "17635", "57844", "37127", "5842", "7048", "31382", "54899", "11534", "36701", "20017", "11202", "42388", "9636", "13120", "289", "68770", "65142", "23005", "12250", "64906", "71186", "39064", "1311", "42452", "50613", "16355", "54816", "47136", "16836", "45169", "59589", "16500", "42284", "57666", "61364", "58686", "62767", "67938", "28531", "44363", "35290", "18419", "45887", "38021", "5334", "21362", "27133", "45013", "26030", "35150", "52660", "11331", "58669", "24453", "52183", "47184", "17595", "35925", "5294", "1857", "65760", "44112", "62017", "33038", "60530", "15200", "20201", "44629", "3490", "11782", "65618", "54753", "38510", "38358", "58123", "31164", "61055", "31995", "59022", "38397", "10183", "20679", "26156", "4792", "4430", "54290", "2839", "47556", "51957", "39828", "15263", "10372", "9746", "62892", "49013", "28882", "32098", "47874", "32257", "62817", "5306", "62822", "36195", "12649", "1410", "26751", "29907", "33856", "20423", "63189", "60575", "19770", "27582", "70187", "9579", "57604", "12502", "21071", "37385", "44283", "5657", "66654", "14121", "3698", "3924", "1418", "46703", "23675", "33515", "61442", "72167", "24372", "43589", "20393", "18862", "4658", "49480", "50911", "29798", "61258", "43845", "34423", "2543", "32005", "67284", "5354", "48365", "23373", "25308", "39185", "49862", "69242", "72575", "55470", "53370", "1838", "41768", "57390", "37647", "26075", "22635", "58787", "47497", "19310", "40826", "27861", "33887", "20853", "41246", "43987", "24977", "62833", "51439", "34815", "72120", "15136", "54677", "37817", "38295", "71608", "25088", "44139", "3914", "24039", "21102", "47371", "48833", "17388", "66987", "6753", "60586", "52377", "48854", "14029", "34613", "14635", "38137", "68806", "44369", "59475", "24629", "49676", "37936", "15168", "28420", "59005", "45966", "29492", "69960", "24979", "36221", "22976", "13499", "72545", "71361", "65136", "58603", "69534", "457", "67780", "11552", "30461", "62010", "34967", "33011", "29721", "57161", "27435", "14531", "23702", "57162", "40555", "38236", "23040", "58914", "3153", "31720", "32192", "53817", "23968", "40767", "23506", "11528", "45309", "50488", "59424", "67914", "31110", "11445", "36491", "56059", "59136", "36954", "60032", "40217", "48272", "10947", "44327", "37120", "28381", "10198", "47713", "15750", "53495", "38475", "46918", "17642", "7396", "37485", "47580", "45509", "59102", "44294", "57186", "16134", "15222", "23958", "57005", "12357", "43612", "47448", "37348", "33640", "54685", "2324", "30577", "23365", "12591", "110", "33903", "16455", "61425", "36789", "5605", "64612", "56405", "12488", "61872", "29475", "61152", "49837", "36331", "56301", "66334", "11692", "28280", "5624", "13717", "64421", "55468", "12457", "44969", "51632", "38489", "70078", "18954", "59408", "53613", "72620", "13282", "44121", "2062", "56996", "29812", "20122", "3436", "15364", "15855", "4511", "62773", "14846", "61415", "31064", "29491", "46426", "28159", "58433", "48647", "38149", "65166", "47303", "16356", "12010", "57013", "57855", "66885", "7589", "58563", "18390", "45198", "8609", "28114", "47657", "26469", "23649", "34196", "18048", "14184", "59209", "48600", "62022", "29478", "34241", "17069", "25143", "67486", "63765", "42528", "25796", "15233", "48685", "26394", "29411", "26602", "26112", "72762", "44373", "50007", "53110", "43893", "10257", "61393", "65354", "46143", "2248", "20137", "31287", "5154", "50657", "68386", "25060", "33053", "6080", "56272", "26769", "21139", "3768", "71177", "52504", "12958", "11130", "72924", "60231", "11569", "55378", "55826", "38036", "52571", "14583", "14199", "51847", "45900", "50709", "53834", "17116", "40336", "16368", "3384", "66044", "31608", "21730", "55989", "44994", "10886", "9426", "45042", "56126", "18379", "60532", "16300", "30038", "21213", "41454", "6236", "71704", "59463", "13066", "41343", "28076", "64842", "51414", "64446", "39118", "67942", "60946", "29324", "13467", "15905", "9995", "71072", "56660", "36800", "59773", "40211", "12742", "28540", "69458", "47971", "10643", "8726", "6537", "16992", "28647", "70533", "31675", "49068", "69035", "17270", "49956", "64995", "68333", "70027", "8706", "63127", "59354", "65971", "1869", "13923", "36609", "26076", "6238", "58999", "38059", "51022", "49923", "10683", "56708", "33392", "4385", "64343", "22533", "67018", "14421", "38101", "68699", "36425", "29886", "18635", "43854", "73133", "32664", "9732", "57662", "5285", "38895", "5532", "12643", "22691", "48949", "66342", "50598", "21735", "70144", "45037", "56903", "31133", "65725", "51939", "17725", "68083", "30775", "43923", "53983", "47978", "58048", "15363", "1422", "25109", "53849", "66451", "46414", "29290", "62627", "36025", "462", "26522", "14157", "14082", "44677", "30771", "17147", "44192", "11366", "64454", "5688", "57140", "69882", "28604", "16702", "13887", "4853", "55741", "45758", "57880", "3618", "1885", "60294", "10846", "34986", "23781", "44295", "63131", "25915", "12450", "19361", "60639", "13406", "60324", "55862", "10499", "48136", "59844", "42039", "63157", "47649", "19148", "62462", "44061", "4358", "68763", "69142", "46012", "26095", "7669", "43974", "31239", "24904", "39620", "35583", "20773", "23848", "15004", "36796", "32910", "37759", "66175", "11114", "38010", "35871", "5721", "69471", "3611", "51514", "29683", "32621", "38917", "63437", "29675", "31916", "41490", "7108", "810", "14755", "45644", "43231", "72756", "2766", "15780", "55656", "29952", "37937", "40646", "54220", "53972", "13198", "51630", "14624", "36060", "29360", "7941", "70632", "43977", "33", "51400", "31758", "60588", "65300", "65374", "32840", "5844", "67330", "34414", "25094", "7596", "33586", "12947", "35046", "41852", "17293", "31401", "50805", "5947", "71643", "7370", "47813", "44688", "70644", "2411", "68605", "35984", "11956", "18226", "15494", "8425", "30486", "43075", "18170", "12603", "67991", "54971", "46769", "14616", "37211", "22782", "52996", "4058", "40413", "55798", "36017", "44260", "28513", "12341", "64269", "7532", "28012", "13562", "52593", "39569", "21869", "28539", "63377", "5183", "6450", "22755", "33046", "34393", "44836", "70606", "54949", "6651", "8277", "36362", "34286", "33237", "52361", "51816", "31489", "71616", "37702", "7082", "13615", "64436", "39865", "43984", "60432", "35723", "70803", "4935", "67065", "6422", "20428", "43584", "9674", "21826", "66726", "58698", "3573", "18499", "28652", "13363", "1524", "39942", "57059", "24552", "14351", "63326", "37068", "32625", "37917", "487", "71778", "13633", "17898", "6309", "69703", "69062", "12258", "50128", "2476", "55791", "12581", "40184", "3093", "62938", "63237", "50568", "68497", "20447", "61743", "56465", "56177", "38447", "20583", "39951", "24941", "21849", "52769", "50030", "64060", "72441", "14533", "7437", "23016", "14882", "8526", "68737", "7649", "54435", "53037", "22974", "29617", "26540", "34732", "67414", "72831", "5776", "30960", "31993", "14070", "45067", "37983", "3602", "12618", "36522", "66308", "37329", "49576", "7787", "54113", "27318", "37646", "15856", "33837", "26595", "5959", "33917", "20101", "32040", "44223", "18430", "66187", "7485", "22919", "51524", "17412", "27454", "35889", "70490", "44475", "68997", "28473", "70520", "50831", "7471", "24688", "42072", "20152", "60292", "42168", "58182", "52845", "27198", "28640", "33770", "9770", "26740", "59542", "41583", "25052", "3530", "22929", "46688", "5685", "26068", "57299", "24283", "501", "67643", "8245", "40882", "48356", "38274", "71208", "32503", "56255", "15582", "28974", "29243", "59971", "37509", "23493", "69662", "41573", "9252", "32854", "15595", "15966", "51036", "24876", "55508", "47877", "3283", "44484", "25992", "37018", "56649", "67350", "4727", "48437", "2498", "22423", "48883", "22453", "7939", "9420", "12000", "10339", "37742", "69425", "8647", "21557", "31111", "56762", "4176", "6445", "1576", "60388", "47080", "26510", "54801", "37050", "60962", "63316", "43177", "3984", "45291", "61186", "5363", "17319", "4505", "68639", "11293", "34502", "71409", "50744", "70108", "58044", "64147", "16692", "63961", "42899", "43672", "50169", "61110", "141", "47534", "70263", "66776", "43224", "21443", "42977", "25718", "6616", "18911", "23337", "16491", "20303", "60068", "6039", "15100", "45537", "38772", "19019", "5704", "5888", "4440", "47224", "12845", "28975", "67477", "14985", "57218", "50037", "8536", "62454", "947", "31669", "37530", "55315", "39571", "22081", "23548", "61277", "21147", "25082", "54664", "155", "19860", "20921", "60361", "47278", "35640", "8211", "10718", "40458", "42319", "44731", "2473", "19840", "24424", "41287", "20121", "9502", "53852", "7302", "13745", "51908", "58532", "63089", "11881", "39595", "20894", "5997", "58894", "65198", "66816", "57252", "30769", "47844", "41617", "32463", "50009", "71321", "47099", "68189", "2394", "43669", "4841", "16470", "51966", "36939", "18188", "36510", "18723", "19105", "19617", "967", "22762", "29141", "66747", "54713", "29179", "67663", "54745", "15884", "66512", "63128", "23136", "7523", "19776", "4646", "4718", "4060", "36481", "45258", "4597", "39166", "55465", "26351", "24310", "40081", "23834", "56544", "42839", "26562", "1321", "29518", "52490", "47669", "58975", "40377", "18568", "55319", "70639", "21208", "61617", "13819", "7223", "39098", "40122", "38719", "50411", "4616", "65033", "8645", "25971", "1265", "43186", "52570", "58518", "61039", "19020", "28693", "17019", "43344", "57151", "5954", "44572", "50476", "16149", "59823", "48104", "7768", "35159", "4524", "62816", "27727", "45960", "51297", "46906", "31498", "18449", "49993", "9367", "73148", "70620", "6199", "4264", "57921", "58879", "55582", "16061", "24756", "60289", "48993", "37781", "31296", "56736", "28524", "33229", "4224", "41748", "37347", "29420", "27562", "37739", "52860", "14654", "13947", "57104", "69431", "920", "33561", "9028", "16614", "16740", "55661", "48830", "42277", "3296", "58850", "52358", "1568", "45827", "63760", "10626", "18088", "37446", "59442", "57354", "19858", "26950", "48470", "72739", "8567", "57874", "13857", "71128", "17865", "64201", "22489", "25128", "3110", "15703", "807", "19836", "22291", "12831", "51115", "27153", "24859", "41785", "49709", "68273", "39011", "56796", "71564", "37965", "69180", "54653", "23597", "42225", "18942", "65720", "59694", "3990", "61851", "60566", "48610", "15095", "22569", "68924", "14971", "44251", "53672", "72723", "48025", "48186", "36097", "41817", "57251", "13974", "41533", "37877", "38562", "4468", "49846", "59281", "73031", "65015", "19919", "33247", "27950", "19328", "63830", "51443", "53156", "55159", "47068", "55105", "65844", "64046", "57519", "50291", "63046", "21024", "50127", "52993", "15211", "21872", "29267", "39450", "53461", "53430", "53771", "9460", "30675", "57159", "15941", "17932", "6062", "33275", "47855", "28153", "8151", "69371", "33305", "10723", "44765", "40111", "43086", "44415", "52917", "11322", "69481", "40440", "9933", "46018", "62737", "44684", "66722", "12041", "44062", "55551", "17934", "52985", "16563", "70857", "51775", "71348", "2359", "44695", "9852", "34724", "34980", "16132", "26989", "44828", "1975", "57138", "4973", "65420", "41507", "45054", "3729", "57503", "37571", "16351", "67720", "39936", "1036", "68975", "43494", "19760", "41669", "68084", "16987", "59774", "33444", "58796", "71644", "46614", "33587", "57397", "26734", "51120", "276", "61443", "38074", "45792", "712", "5717", "58562", "37719", "60448", "60336", "56146", "44212", "3977", "18227", "5474", "54164", "24777", "46781", "16024", "26553", "27895", "6866", "13117", "55448", "15965", "35210", "21035", "42060", "9388", "64649", "4639", "25108", "29203", "37257", "22497", "22088", "28521", "13712", "45385", "62389", "45663", "875", "15082", "9055", "27512", "53015", "48900", "60799", "22629", "69220", "4895", "17952", "63598", "12473", "70797", "40914", "27472", "60624", "69361", "66538", "53585", "69649", "45924", "57762", "57262", "39281", "8573", "38041", "51362", "13032", "7960", "29198", "38163", "39568", "24841", "17036", "1342", "3963", "64160", "47215", "2572", "59472", "18978", "41538", "45340", "25442", "70084", "40529", "46819", "36473", "58902", "11882", "66608", "9856", "45209", "31390", "4680", "35475", "63848", "10419", "62941", "43492", "50788", "10862", "34329", "49937", "72497", "55628", "27200", "52536", "72780", "16773", "39443", "13097", "23041", "1738", "71013", "7253", "9451", "26705", "37437", "9779", "65851", "65246", "61366", "50594", "69916", "23468", "17888", "43028", "35177", "64812", "61997", "10015", "4923", "61085", "50458", "29922", "58417", "33606", "19471", "37614", "17723", "6969", "45722", "45399", "58942", "27433", "31533", "47139", "20346", "38347", "27484", "18760", "28846", "3461", "6854", "1037", "53631", "22223", "72179", "33541", "23075", "21400", "64724", "24875", "58759", "53099", "35541", "22468", "28713", "28121", "29923", "11979", "11087", "35100", "45331", "55", "29859", "2121", "38767", "44844", "29846", "23032", "19820", "63379", "20804", "41598", "32461", "23985", "43714", "19672", "35610", "19081", "11736", "47678", "34539", "33753", "68563", "32745", "21254", "25797", "1137", "32661", "40913", "60731", "37147", "45959", "36218", "15189", "30585", "29228", "26035", "40946", "37846", "69830", "71268", "57927", "39032", "18084", "40266", "49523", "61608", "66748", "12800", "9203", "12715", "70494", "34686", "58574", "30833", "55630", "3324", "40011", "58319", "71029", "52768", "64901", "28786", "21131", "10226", "10368", "68204", "63067", "66242", "70096", "66280", "5895", "36050", "31795", "55576", "41816", "1125", "14174", "50512", "57766", "22262", "64375", "41344", "15962", "11880", "33907", "3655", "4889", "20521", "23555", "2675", "60274", "48892", "580", "63628", "55254", "61976", "52528", "48225", "39400", "72833", "15232", "1933", "48307", "38032", "19454", "2917", "55362", "371", "58521", "28735", "6469", "28596", "65628", "60219", "37575", "58524", "29889", "12677", "68889", "37850", "34716", "64398", "68636", "21120", "60525", "44959", "68888", "45118", "33183", "58899", "13568", "22514", "54378", "60643", "47606", "37873", "68859", "38783", "65638", "18994", "39534", "52455", "39638", "33496", "39217", "35683", "41091", "25577", "46980", "42618", "42778", "21656", "15272", "48421", "57407", "28712", "49215", "45479", "44011", "2242", "26791", "49621", "40795", "28136", "70779", "48406", "29915", "13280", "4054", "404", "44418", "17807", "64441", "28599", "30281", "27916", "60725", "12880", "25459", "21941", "42401", "62881", "11632", "32755", "59829", "25068", "23027", "40735", "55695", "16517", "51532", "15577", "5982", "10621", "67728", "12801", "56199", "41202", "11138", "1161", "19547", "36032", "40013", "66199", "6768", "72798", "53127", "50668", "47086", "71481", "45892", "69423", "6326", "21707", "45621", "66058", "64912", "27684", "34148", "37501", "33529", "66304", "37815", "25638", "7361", "54057", "20186", "55012", "16824", "35880", "58530", "58661", "70047", "12141", "11452", "4389", "15771", "47130", "65743", "57700", "24970", "25641", "42698", "43513", "27470", "43352", "14449", "70976", "6601", "33569", "46066", "58423", "33512", "69040", "62131", "32675", "65483", "66892", "55849", "21978", "72758", "51354", "30329", "7007", "55778", "59200", "36203", "29483", "25810", "41698", "38877", "37652", "2430", "6746", "44771", "31160", "24506", "41578", "67438", "67137", "62638", "35725", "10083", "53561", "65322", "5054", "4621", "6515", "10117", "69442", "47640", "29086", "12079", "48827", "48981", "57710", "36842", "31015", "40784", "35356", "63834", "19869", "22587", "41797", "58093", "23772", "48811", "41656", "65889", "57788", "2165", "55751", "37479", "23406", "12699", "11797", "8650", "23228", "44480", "7330", "1574", "25153", "70063", "18336", "7059", "20790", "35859", "24706", "37232", "17710", "25830", "56258", "33052", "68868", "1982", "22857", "11613", "59761", "34451", "34217", "46335", "24831", "41225", "40542", "2345", "60080", "60208", "37070", "11077", "64717", "48882", "24442", "72777", "56861", "32938", "63027", "57858", "73045", "43294", "2972", "12827", "14807", "15724", "26541", "66901", "44858", "51228", "8156", "58948", "59489", "22061", "30619", "50611", "10397", "16007", "13276", "58635", "4835", "307", "33885", "2049", "48476", "12231", "44873", "13181", "65285", "50579", "30926", "36245", "26263", "49038", "50218", "17087", "46658", "55948", "50010", "12427", "25780", "22457", "16204", "3953", "48657", "23022", "7381", "22789", "17845", "32444", "23057", "23135", "46663", "47906", "58462", "33959", "39364", "25032", "55367", "32832", "70454", "7449", "27550", "37889", "398", "70870", "7632", "64530", "38374", "19740", "615", "69735", "30284", "11483", "47050", "11271", "50743", "71263", "29949", "6664", "43254", "60286", "38801", "4429", "30612", "11498", "70272", "62442", "8269", "36093", "42873", "40233", "22488", "5031", "55527", "14580", "51602", "43012", "11234", "43007", "25147", "69798", "55843", "16777", "51315", "28292", "49080", "44211", "15929", "7786", "72121", "32151", "71876", "10555", "1051", "33937", "57481", "12342", "48096", "5788", "66375", "45909", "67610", "37611", "71691", "32447", "35657", "53237", "23845", "32743", "53106", "20584", "9710", "17333", "45495", "27128", "33440", "49159", "60750", "49660", "24631", "49582", "47047", "33295", "51627", "36773", "58834", "16409", "52111", "9299", "45576", "13846", "69033", "42327", "1684", "48969", "60143", "57249", "9927", "16674", "7804", "60808", "25872", "9076", "41810", "45301", "48113", "66934", "52305", "50033", "531", "67947", "30500", "48247", "48982", "11187", "66332", "72123", "62407", "2499", "12388", "72707", "34598", "56962", "28428", "33605", "44197", "23294", "44028", "2722", "58813", "5167", "56643", "29181", "71062", "40174", "71657", "24040", "55854", "33397", "13794", "48637", "58328", "31870", "47908", "69488", "39284", "57422", "53542", "12303", "29240", "20349", "30507", "64897", "64984", "49872", "48304", "15140", "23812", "13988", "10735", "12558", "54296", "30356", "23639", "14469", "55900", "46095", "50302", "21142", "18525", "32612", "19456", "23611", "54018", "48527", "32453", "13959", "46573", "6325", "68237", "32433", "44046", "39781", "11670", "50979", "14363", "20960", "68676", "46126", "4615", "66507", "14111", "61955", "23093", "14476", "27760", "50382", "50834", "45189", "24072", "66013", "33415", "49405", "25119", "12321", "66127", "15745", "40060", "30251", "67152", "58068", "62375", "58057", "2565", "60195", "31144", "6128", "2705", "1334", "31752", "56225", "25869", "45363", "65095", "48033", "62954", "24739", "12632", "6381", "6609", "36835", "27245", "30948", "23790", "26361", "48607", "8489", "52285", "47879", "24500", "71918", "52271", "15231", "40894", "65701", "13828", "16998", "42991", "57356", "51341", "10968", "1547", "53604", "57499", "45573", "4681", "63330", "39038", "18075", "56770", "62808", "68773", "3652", "33093", "41034", "20072", "55469", "16472", "62681", "13677", "57873", "16651", "10918", "50163", "46449", "6444", "34115", "36379", "50543", "38707", "42865", "64224", "42464", "23316", "10353", "58625", "22931", "16895", "2379", "21853", "36626", "57506", "15555", "33732", "43014", "62148", "5980", "13669", "46696", "64922", "12156", "22729", "39242", "38005", "49122", "63915", "71667", "1087", "8030", "2075", "45582", "18486", "43948", "32950", "36018", "30821", "61486", "6703", "33712", "33007", "50248", "23346", "14386", "52337", "11701", "57671", "71780", "70964", "27116", "45999", "2950", "34085", "72850", "62347", "65668", "27541", "4071", "43500", "33083", "27310", "48989", "42", "42979", "58567", "35804", "39564", "13074", "20874", "22925", "28624", "50726", "72310", "57645", "14705", "68284", "15334", "17488", "33227", "70826", "51032", "53760", "9214", "6449", "22914", "67994", "10053", "53122", "33785", "5094", "17247", "238", "70450", "43922", "7681", "53272", "38882", "58274", "324", "67830", "24832", "64182", "69435", "34344", "53719", "58112", "48205", "38213", "64029", "21760", "9455", "16532", "31413", "20035", "47387", "13190", "42281", "52662", "49127", "7017", "48387", "45627", "63058", "10631", "59564", "28394", "55160", "2570", "58234", "29327", "51185", "46405", "8264", "49765", "53692", "70029", "13062", "6615", "13621", "56747", "17088", "2099", "25175", "32092", "46353", "8671", "7920", "50731", "68454", "3696", "24897", "47867", "57955", "1250", "63792", "5637", "22077", "19831", "10861", "38842", "37888", "103", "65236", "8610", "46952", "24499", "53562", "10548", "71839", "9931", "60069", "32791", "30912", "868", "2773", "25801", "49472", "11295", "8765", "41746", "62685", "40681", "18205", "26713", "30280", "1836", "8529", "53299", "26975", "69261", "49467", "24823", "13339", "53434", "7116", "34990", "64531", "60608", "23023", "10031", "3257", "8640", "58265", "26909", "15527", "14109", "73005", "21063", "67323", "31797", "27458", "55423", "35793", "52605", "8737", "42486", "35388", "66755", "59791", "17989", "22605", "13781", "35527", "47149", "68402", "67211", "1981", "66376", "61537", "53007", "21442", "10607", "16029", "30295", "29084", "45468", "20494", "44883", "11689", "18448", "16872", "8477", "20328", "70915", "36643", "16888", "12513", "54327", "64624", "27317", "53498", "26114", "37137", "6172", "61663", "12243", "27628", "8680", "33226", "34943", "46241", "6260", "33241", "7215", "23280", "1198", "25696", "52780", "613", "22990", "20377", "38579", "51684", "4771", "16791", "57253", "68986", "23492", "26280", "56041", "22851", "3425", "34856", "62634", "8643", "8368", "30594", "41922", "19691", "47947", "64480", "26320", "6229", "22124", "39214", "9173", "18190", "4927", "3616", "21754", "28661", "1698", "10894", "19754", "61314", "57818", "7790", "67875", "13825", "61029", "57073", "72074", "64501", "54586", "58656", "26205", "62235", "19436", "48558", "756", "49241", "24158", "46299", "16229", "60705", "6942", "17029", "31510", "35810", "24415", "38678", "49117", "29187", "60872", "60853", "49230", "45122", "21608", "40719", "7405", "42559", "30018", "60633", "56737", "6518", "50359", "26900", "38498", "35137", "32072", "45834", "58063", "10569", "43979", "53715", "22528", "56850", "70284", "60392", "3543", "62595", "9271", "24998", "62537", "56618", "15297", "58637", "5432", "20628", "71161", "9398", "7496", "71896", "65867", "8516", "52665", "19546", "43166", "51825", "11841", "71238", "16515", "13944", "28993", "31046", "24250", "59160", "4491", "19291", "6008", "59055", "55030", "9758", "21768", "67027", "48140", "1945", "54270", "69080", "72820", "16022", "7709", "62859", "52115", "49822", "857", "70653", "31503", "60377", "35415", "32758", "62429", "58706", "13644", "67818", "52482", "53233", "52156", "58750", "39374", "15", "42415", "3406", "62267", "62730", "7935", "42588", "41517", "15358", "38874", "10715", "49498", "62013", "56091", "56441", "16353", "1318", "36847", "69608", "42700", "54154", "43382", "67580", "22130", "34795", "59717", "4666", "11855", "23138", "54132", "62355", "59545", "51940", "12485", "54240", "52200", "48433", "16480", "7036", "43148", "44460", "39870", "65599", "36138", "35547", "7018", "26169", "1825", "36514", "1300", "6202", "28175", "16844", "24948", "50360", "62688", "17005", "35359", "71516", "64515", "69981", "62667", "42390", "31168", "26147", "42826", "42614", "1722", "36891", "59482", "27446", "50562", "20611", "37928", "58461", "63235", "70874", "42012", "71218", "55106", "56950", "27580", "59235", "17578", "42367", "51791", "36359", "7914", "38078", "37925", "36922", "31538", "27312", "64951", "9563", "32203", "70523", "46602", "15202", "26192", "59762", "50245", "51668", "5614", "50273", "5824", "64156", "57208", "64366", "2329", "70112", "19995", "22156", "32663", "65773", "54598", "18820", "22827", "27523", "68953", "61489", "13721", "51576", "64272", "26034", "21803", "45852", "55229", "10552", "28172", "56180", "46963", "38288", "18697", "44632", "60935", "12312", "60264", "66057", "17787", "50765", "69599", "54289", "43176", "51027", "34910", "13553", "69144", "6076", "24193", "45224", "29140", "42567", "40808", "24805", "66292", "13803", "18612", "24384", "14303", "63859", "69692", "30950", "2215", "16457", "8296", "14947", "19525", "60371", "38392", "3960", "26747", "14072", "63526", "61451", "59061", "17143", "53612", "71911", "2986", "2633", "64941", "63494", "51705", "51859", "67781", "4152", "29061", "8929", "68689", "35518", "31977", "16759", "22231", "25136", "42898", "65359", "36201", "71398", "67684", "35318", "18515", "11193", "60599", "12697", "36401", "41484", "41456", "60628", "9095", "70581", "31045", "5604", "26026", "61288", "34090", "6575", "16087", "28334", "51679", "14212", "24211", "65277", "68631", "45236", "34705", "26203", "9816", "26957", "65286", "12859", "32739", "31647", "53490", "56738", "3949", "29174", "24654", "12694", "56774", "14285", "50301", "68278", "9923", "19494", "17295", "7823", "9091", "15550", "10817", "49773", "63173", "16126", "39755", "60994", "63312", "57674", "34655", "55901", "28758", "2509", "64872", "67637", "16680", "8289", "25283", "58092", "5837", "58968", "60576", "48627", "66371", "40123", "18321", "34909", "26083", "4673", "71343", "37886", "16451", "56670", "58247", "9287", "2161", "44351", "69414", "13425", "21602", "50790", "11240", "37267", "5600", "56211", "8199", "26353", "22894", "65347", "31248", "8319", "13436", "8165", "26557", "12383", "14664", "51873", "17482", "62839", "49381", "71508", "1272", "2615", "37136", "43793", "22162", "42192", "19500", "52510", "4062", "7053", "72226", "17261", "10157", "54851", "20605", "21615", "28863", "36659", "37531", "50265", "19045", "37918", "32571", "23680", "6861", "70234", "26315", "15172", "41326", "36547", "51288", "2541", "8378", "9018", "16980", "31714", "64560", "54973", "18415", "36349", "55287", "72936", "71548", "45452", "2747", "9447", "26402", "38940", "68260", "1643", "39432", "9512", "51777", "72675", "6283", "49642", "15401", "34308", "34060", "45400", "19319", "57556", "54283", "5965", "18349", "65259", "55070", "12647", "8241", "18362", "45312", "436", "3878", "25893", "11150", "73200", "174", "9023", "29964", "68442", "46488", "60370", "6542", "36112", "20764", "57323", "41849", "24479", "4184", "8990", "24791", "59873", "29028", "17290", "34483", "55492", "54031", "42339", "44812", "58519", "57420", "23367", "41934", "33943", "54249", "68535", "14791", "17998", "24407", "3186", "63867", "58077", "8593", "12980", "37225", "65556", "51045", "25499", "26307", "40618", "60694", "57540", "71113", "9629", "48465", "57198", "68239", "53531", "10583", "54684", "70775", "57072", "3338", "72067", "54979", "25431", "46078", "20076", "8312", "63441", "3375", "34956", "68900", "31058", "22869", "44154", "67221", "20396", "24587", "7270", "32762", "6396", "53702", "9859", "6620", "60509", "63780", "30914", "31905", "37586", "45335", "56987", "70947", "67172", "31096", "1572", "49069", "35298", "1708", "64494", "56371", "47028", "29432", "48026", "31161", "53582", "46440", "21578", "52748", "32873", "36515", "35259", "6365", "2805", "36000", "10703", "48498", "56587", "62639", "101", "28626", "49947", "61585", "65370", "4413", "2348", "32628", "56231", "66", "67759", "24547", "37823", "59593", "5836", "53217", "4504", "19715", "37968", "59324", "65863", "60016", "31187", "6411", "47071", "53947", "3054", "18289", "59818", "428", "44797", "57387", "52204", "51792", "35682", "13418", "65194", "58827", "46923", "22685", "24623", "69212", "48058", "27497", "45820", "6732", "56035", "46933", "4348", "61105", "59916", "43493", "27474", "13858", "24982", "29727", "3437", "40602", "34223", "56975", "31054", "16933", "71596", "33045", "16257", "71604", "34574", "22852", "22977", "46649", "33875", "56943", "18157", "60878", "1251", "33140", "63150", "22641", "53647", "29383", "15106", "18372", "11244", "69396", "27913", "22347", "59239", "7210", "17935", "44371", "59921", "20067", "62096", "36881", "61133", "28709", "60325", "13127", "24760", "26465", "25213", "17171", "50093", "15954", "49263", "63619", "9892", "46451", "1260", "19667", "47466", "36212", "7950", "19790", "46173", "63367", "70997", "4360", "1767", "66833", "26758", "15230", "9656", "45068", "49047", "49752", "63921", "60779", "15057", "49344", "48669", "63182", "12750", "50842", "26061", "2759", "35456", "4593", "60375", "18796", "50577", "24525", "19762", "1753", "47635", "524", "48675", "51350", "4596", "38290", "64268", "24183", "61725", "30837", "16403", "51255", "43693", "53641", "21905", "9572", "42190", "33498", "42198", "12207", "18072", "62805", "34224", "11626", "21155", "40002", "72253", "4902", "20043", "61132", "64534", "68158", "18078", "23364", "59886", "52087", "65999", "15273", "6362", "67615", "17156", "32765", "59293", "2039", "42437", "6619", "72768", "15316", "31849", "36632", "38957", "16618", "18135", "58306", "53367", "20269", "49166", "43972", "6610", "35769", "65943", "15347", "3844", "21553", "71359", "71908", "29067", "44400", "52912", "71091", "37392", "56811", "56171", "26168", "12768", "8705", "48529", "46605", "65474", "7110", "46290", "51231", "30581", "62433", "23273", "29284", "63009", "53063", "64768", "16901", "51485", "65391", "71788", "38691", "69638", "68912", "44938", "53019", "22794", "53707", "65594", "56399", "35572", "24448", "40563", "36611", "33113", "33164", "53184", "13057", "63550", "59103", "7707", "8484", "40374", "67165", "40658", "11155", "44321", "54137", "51286", "64676", "64631", "1552", "19325", "30187", "30366", "24781", "48246", "17550", "20769", "48970", "62926", "36120", "41874", "9582", "59112", "54769", "66559", "65615", "5312", "61325", "32494", "55399", "71925", "68028", "69676", "60070", "29185", "48038", "9333", "54338", "33288", "35982", "65589", "16977", "59746", "21166", "53030", "51178", "3542", "39031", "45146", "39685", "54454", "58549", "13141", "23607", "34411", "41616", "1615", "48397", "19844", "23763", "32314", "72141", "55844", "68983", "30348", "26941", "20001", "54617", "31335", "58222", "52832", "28783", "2475", "16754", "13951", "42880", "10923", "48757", "71212", "48213", "3413", "16438", "19798", "44739", "60638", "3773", "62018", "72433", "26523", "3506", "72800", "67091", "36984", "24051", "20671", "61771", "9044", "40051", "5577", "48097", "47950", "32144", "25984", "64504", "39186", "56304", "57722", "7516", "55702", "72421", "7221", "28798", "9029", "27107", "46087", "37639", "4754", "4281", "39929", "30497", "29937", "12188", "58385", "57096", "39551", "68192", "72032", "45014", "70133", "60737", "7187", "49819", "55961", "29285", "62305", "18184", "24327", "32609", "70354", "67355", "34973", "9962", "29741", "28097", "26445", "65523", "24335", "3534", "52766", "33317", "48760", "48650", "44837", "63297", "56439", "21659", "41195", "58922", "23198", "12332", "28585", "16243", "18228", "6743", "53482", "60988", "62489", "575", "5736", "72084", "48943", "29208", "44110", "632", "49971", "35882", "15462", "23186", "6913", "11986", "4876", "61577", "8297", "29809", "64844", "32508", "984", "13942", "43498", "48203", "36994", "40454", "18191", "37382", "52498", "41134", "54552", "63501", "59487", "31637", "45531", "64963", "16115", "71944", "29440", "34309", "49608", "38978", "6016", "34544", "55980", "3401", "72761", "46473", "14582", "33233", "38047", "68984", "58190", "30551", "40236", "20127", "28338", "21216", "5852", "17810", "10993", "35778", "19949", "71166", "46845", "45020", "58604", "71914", "5726", "69847", "48178", "8953", "14903", "54373", "43882", "44593", "68858", "67899", "28517", "47753", "33064", "67376", "5553", "66288", "13843", "66047", "22486", "61071", "40970", "66170", "61092", "47554", "68523", "14338", "18234", "50732", "67475", "618", "17529", "69859", "48122", "6769", "6554", "25580", "53569", "6383", "3393", "8107", "5851", "37943", "67397", "63608", "13983", "56766", "19352", "12480", "67852", "31508", "60826", "4812", "1371", "30259", "41497", "20214", "70591", "33650", "3904", "59349", "52738", "54501", "14446", "8144", "25447", "40738", "3031", "48582", "19714", "67867", "23174", "32205", "9308", "54743", "28245", "16357", "57337", "47381", "18056", "9978", "65301", "46950", "65068", "20116", "69675", "8731", "10687", "51612", "57245", "11395", "69776", "61659", "51421", "63345", "40893", "27891", "55031", "62417", "58336", "30135", "29857", "20624", "69922", "68134", "50092", "15623", "33343", "6965", "40611", "21973", "61070", "40684", "55373", "63605", "9900", "26859", "25855", "45555", "373", "36845", "38793", "37664", "29480", "43322", "44564", "15506", "48411", "13069", "31433", "72097", "72113", "16942", "24458", "13191", "2000", "14326", "72837", "40825", "49687", "46076", "27268", "4186", "4929", "61271", "25511", "58663", "27710", "50965", "66558", "66040", "15212", "59477", "67629", "58634", "52103", "19349", "28387", "66054", "54423", "38531", "54730", "14603", "45375", "43956", "57620", "46588", "60797", "21304", "42087", "21592", "27138", "26226", "66951", "61297", "14817", "54644", "23654", "37814", "23134", "18007", "52217", "18053", "28644", "69377", "8751", "27802", "29566", "8379", "36689", "35789", "21249", "23155", "59707", "72115", "56412", "40161", "4521", "42961", "1683", "48663", "8314", "66801", "41805", "54134", "51051", "15726", "45097", "26437", "27855", "56902", "38592", "55312", "47293", "52274", "62121", "25612", "47350", "34629", "45210", "27171", "71512", "42182", "7394", "26574", "71748", "72335", "56988", "21766", "53058", "44552", "57309", "17145", "54614", "68349", "18693", "2223", "42928", "11778", "21353", "13935", "38146", "32883", "28612", "3959", "67784", "59533", "43545", "23904", "71931", "25422", "21518", "41610", "17979", "48668", "46885", "23944", "62985", "18707", "43083", "9741", "56384", "33122", "9937", "20074", "27327", "65485", "4439", "45602", "4361", "53491", "59056", "1202", "65181", "63662", "16440", "47541", "40033", "41296", "14188", "60332", "62847", "3901", "69918", "57450", "72588", "35114", "40331", "9240", "304", "52124", "52125", "49351", "34013", "8532", "35425", "30606", "33313", "14023", "69978", "16580", "13175", "15683", "6655", "49356", "67283", "40170", "36145", "19092", "69237", "69549", "21110", "32176", "9269", "11948", "40080", "47126", "67488", "30409", "43833", "71883", "70687", "7231", "11672", "31790", "29780", "49327", "45231", "37681", "2535", "54319", "18358", "22096", "73056", "19855", "16959", "22817", "52756", "49542", "35825", "47353", "64592", "33304", "19397", "59312", "15708", "67256", "12609", "55639", "2172", "40931", "5830", "18062", "46307", "29369", "22225", "25132", "62340", "70656", "70955", "37013", "60451", "32797", "8678", "65205", "50728", "20059", "23097", "26711", "34467", "44222", "29686", "66434", "62841", "23182", "36364", "23180", "56857", "56685", "36036", "20733", "15040", "9143", "4433", "19203", "48829", "21912", "69670", "51702", "34609", "8355", "27596", "13374", "52601", "63099", "29529", "28799", "3889", "30523", "35686", "56430", "41152", "23070", "69204", "49354", "5372", "61413", "24018", "42527", "19001", "5467", "37744", "59619", "45521", "24498", "67593", "16770", "69902", "32583", "31025", "49818", "46557", "48924", "25723", "48490", "27923", "20840", "16534", "54755", "44026", "53597", "15399", "53586", "43262", "43479", "28054", "57896", "48029", "30630", "26959", "55326", "16430", "16225", "63585", "35213", "50880", "34281", "56669", "41425", "15293", "15603", "38283", "12990", "25386", "25296", "14875", "48166", "23334", "48410", "3802", "28207", "11014", "8866", "72058", "37044", "13046", "55570", "51207", "37612", "52659", "38682", "30112", "9768", "60243", "16948", "51610", "57549", "14398", "40583", "70488", "31378", "43236", "22202", "53368", "41800", "45366", "18047", "14322", "51123", "57961", "59443", "47472", "66948", "21496", "8010", "32383", "63884", "35206", "62172", "21038", "71859", "25489", "57637", "47538", "48694", "18904", "36766", "26343", "20475", "28385", "183", "7877", "23730", "53654", "24516", "17498", "67421", "43441", "15432", "806", "49393", "31398", "43913", "1472", "13939", "15629", "50420", "12035", "21402", "34094", "66420", "25946", "9178", "7690", "71947", "65894", "56769", "45807", "26209", "56190", "70648", "27372", "8223", "11011", "62619", "10282", "45411", "71582", "50973", "25650", "31074", "53253", "6264", "56001", "55177", "20348", "65184", "7004", "11700", "40431", "54923", "65064", "2688", "41097", "7216", "8998", "6764", "34125", "40499", "29927", "57985", "51648", "36669", "17020", "44409", "21980", "13515", "22335", "21724", "15159", "26837", "58692", "45161", "43222", "4736", "17309", "56514", "64117", "46758", "56322", "47737", "54371", "65973", "70984", "54732", "1106", "17378", "13516", "64559", "2658", "48296", "47599", "67890", "52638", "61807", "35400", "2263", "58167", "53931", "6835", "27308", "14793", "49077", "40281", "2790", "19449", "64456", "26727", "39958", "26257", "28419", "51600", "63310", "10512", "50773", "20547", "56879", "73141", "62319", "50737", "18467", "56318", "52364", "64743", "64860", "37692", "23275", "15193", "5807", "27658", "58792", "27158", "37629", "18356", "16176", "38494", "8398", "60516", "64036", "56324", "9837", "5201", "3787", "28919", "9049", "11320", "52921", "53767", "20228", "26829", "69791", "39532", "10491", "25496", "11285", "2528", "63807", "38539", "2819", "59425", "12062", "33544", "2490", "64801", "64581", "25802", "8950", "70734", "38974", "35463", "14345", "65446", "19660", "3082", "33035", "65444", "21566", "56854", "65957", "20046", "7498", "64573", "58253", "47970", "69399", "11920", "45653", "3549", "15327", "22005", "4708", "60703", "24034", "70256", "67479", "56040", "55480", "502", "47322", "29911", "5883", "34647", "46700", "71397", "41259", "8047", "70977", "15516", "7745", "54568", "24519", "7720", "17763", "73063", "5254", "32201", "14534", "22628", "30255", "48383", "39474", "44624", "60425", "72016", "5149", "25585", "24030", "37595", "11266", "70751", "53042", "44687", "51348", "51173", "43996", "28584", "45270", "21708", "9247", "24188", "47585", "21269", "17745", "64127", "67392", "19772", "18125", "47394", "22659", "54203", "17275", "50673", "49953", "32342", "25678", "2886", "29354", "3717", "19923", "69941", "23230", "66069", "16744", "64451", "39396", "14667", "58613", "25105", "13743", "48728", "18098", "62110", "48481", "24727", "65963", "53822", "59759", "49481", "14543", "38331", "7146", "39121", "62798", "55387", "7469", "69774", "60670", "33301", "6589", "6217", "33042", "70818", "32114", "50679", "63805", "31458", "1339", "61485", "64841", "9706", "51335", "61893", "49607", "71210", "16138", "33624", "33675", "65458", "72586", "47016", "64414", "25096", "27739", "10638", "30336", "16371", "72006", "68494", "46304", "5295", "45197", "53385", "9755", "23582", "7284", "13108", "42773", "61389", "24776", "61011", "57261", "55957", "38661", "44979", "62539", "18178", "11716", "58801", "34206", "39630", "57763", "65921", "37610", "14455", "9715", "18404", "23817", "19935", "24628", "61903", "58773", "11951", "2051", "31699", "68723", "36713", "47221", "29417", "57606", "41101", "31034", "826", "535", "22679", "33941", "7990", "18866", "46764", "58094", "13572", "71815", "26767", "60091", "33074", "19183", "1075", "53284", "34959", "36177", "12365", "10357", "47695", "52566", "51004", "36965", "2668", "24053", "21982", "37046", "16931", "68515", "30012", "60698", "61399", "15179", "33483", "67691", "14301", "66213", "2566", "46309", "68801", "10648", "70767", "16755", "47404", "45690", "6107", "6400", "73210", "15638", "35474", "14124", "70037", "11257", "64700", "33751", "51536", "36809", "63825", "29400", "53294", "61319", "49700", "22739", "1113", "56404", "8833", "31803", "67176", "62027", "41208", "69760", "26245", "50058", "11998", "32096", "41502", "37154", "2328", "17987", "50936", "16211", "12086", "26378", "976", "10772", "51289", "62876", "30588", "67674", "51381", "4602", "15148", "66412", "6525", "61434", "15976", "50026", "10020", "59887", "32468", "29610", "29305", "47958", "38009", "71487", "15970", "27117", "35283", "52168", "39495", "26417", "50849", "60926", "31467", "10318", "54761", "27147", "12503", "63228", "4023", "22948", "59704", "42699", "51055", "71529", "55558", "61557", "29145", "37146", "48680", "21223", "71514", "38037", "71998", "30670", "52161", "3308", "53358", "51835", "21444", "1720", "19560", "38934", "11401", "17730", "35782", "46698", "28361", "20505", "31460", "10463", "48878", "57638", "71425", "44842", "38034", "33621", "25", "28827", "14343", "31334", "40418", "58475", "27115", "45347", "57988", "66036", "4242", "26116", "59205", "17704", "25340", "59407", "72912", "58082", "10121", "24410", "66099", "15000", "14955", "154", "40272", "12641", "58855", "71721", "52095", "44393", "68007", "16656", "49374", "62565", "64554", "26476", "34555", "30778", "16234", "72655", "28414", "14854", "60493", "69712", "66269", "5008", "50151", "43235", "41000", "52352", "29913", "5310", "22143", "67905", "20079", "68581", "53081", "53314", "24680", "33827", "62916", "3058", "50844", "68980", "67647", "17332", "62810", "65714", "46016", "38224", "43690", "27708", "54055", "33471", "53514", "4366", "1665", "19305", "24886", "20421", "864", "36738", "15825", "14093", "64081", "21877", "8015", "51681", "52507", "39029", "24928", "37461", "40598", "21544", "29531", "61947", "21445", "10920", "54734", "16536", "35124", "12309", "32624", "27648", "7940", "32940", "53593", "39202", "37941", "43890", "7506", "71418", "6415", "14784", "56391", "21738", "28805", "53922", "72516", "69839", "2160", "48749", "35788", "52427", "35466", "20780", "65568", "6574", "15061", "21727", "39796", "64347", "49960", "45300", "67604", "45288", "19746", "4448", "18028", "14835", "32818", "63342", "19415", "65869", "60723", "60301", "47701", "46063", "14720", "57388", "69727", "34225", "54440", "54079", "41795", "64877", "58254", "30546", "30014", "44116", "48978", "12645", "69310", "36563", "31808", "40254", "55282", "62583", "40455", "13850", "9905", "27478", "2", "49477", "56617", "69380", "28899", "31209", "35811", "10736", "43342", "67589", "39699", "48581", "28702", "57804", "42755", "31155", "14761", "68961", "31100", "49535", "59347", "28461", "57043", "434", "60386", "67644", "34644", "57845", "63044", "7178", "6674", "31678", "36840", "66160", "71520", "35444", "55868", "1184", "25964", "36816", "47622", "16433", "4691", "72332", "26421", "1503", "10567", "32471", "17982", "36784", "59289", "37577", "49566", "53646", "62515", "7058", "3029", "60021", "16876", "45614", "36161", "2664", "11406", "3624", "19639", "6204", "47000", "45964", "21011", "1048", "41932", "21863", "42483", "14892", "17511", "13603", "42647", "37084", "27623", "9822", "64677", "67399", "1025", "59590", "64771", "57327", "43226", "56038", "31592", "4969", "53421", "36398", "54484", "53875", "4328", "41189", "16100", "37767", "41993", "56676", "28140", "70318", "43634", "69255", "52497", "21958", "58221", "18192", "58983", "38436", "53888", "10060", "63301", "63364", "58353", "42216", "48409", "9556", "14352", "33672", "44575", "17983", "13433", "72605", "69913", "16093", "525", "40149", "62408", "51614", "46457", "37311", "63133", "62438", "53784", "32626", "4278", "12978", "14309", "72715", "64738", "30451", "23411", "25008", "0", "42999", "43543", "70543", "16920", "2072", "38972", "59525", "21925", "66886", "16841", "4122", "2267", "62936", "3320", "1481", "56079", "59956", "13239", "6801", "37801", "31263", "38771", "58299", "48032", "47285", "29836", "66318", "37556", "6825", "41003", "64777", "41905", "40109", "54402", "36251", "21326", "53948", "882", "68905", "62794", "42652", "35671", "50747", "44207", "58745", "37859", "26176", "949", "71164", "34039", "57331", "43585", "8209", "51411", "24726", "66902", "22618", "1760", "32501", "59755", "70104", "26145", "7553", "34282", "36109", "35202", "23780", "3505", "64107", "36264", "49925", "14381", "43812", "69259", "26960", "32050", "32301", "35997", "37496", "5077", "21775", "67677", "46634", "22010", "62924", "20082", "56569", "8848", "33360", "24789", "55895", "28739", "11335", "46714", "11530", "69887", "1856", "64799", "63334", "60044", "20048", "14369", "26803", "54870", "52044", "20941", "51556", "63951", "20322", "13439", "65511", "52019", "26367", "25858", "56337", "6409", "62293", "39027", "38651", "20", "41143", "5093", "39160", "47504", "39996", "71736", "30040", "13868", "54936", "41291", "42485", "27921", "29418", "19481", "68394", "71095", "7971", "63376", "8371", "6935", "37560", "8620", "46814", "14297", "43626", "14554", "759", "25832", "35770", "22993", "19565", "16661", "28615", "71581", "59867", "16950", "18957", "59341", "69432", "23820", "64925", "503", "41464", "56084", "16979", "36640", "5580", "73194", "61916", "59076", "9509", "29806", "40855", "17718", "41690", "24772", "61310", "42136", "40461", "51364", "24971", "52094", "25142", "56008", "46222", "18186", "17236", "38325", "51124", "38744", "71137", "71892", "26667", "14587", "5702", "49873", "50105", "38091", "65439", "42448", "62620", "8227", "60963", "22235", "854", "71766", "36821", "5538", "22876", "54747", "58943", "24750", "26575", "24974", "31556", "27002", "11667", "51276", "25246", "27475", "14911", "25746", "1089", "34053", "68626", "56294", "50602", "30788", "25487", "27274", "37246", "26455", "44012", "39981", "2882", "41741", "24985", "23966", "45981", "72993", "19250", "56000", "49768", "8860", "22424", "52459", "54123", "11157", "2282", "62252", "63979", "50663", "27503", "26724", "19054", "30167", "15144", "68094", "37245", "5762", "67742", "27539", "65496", "7575", "46868", "28467", "5686", "32434", "9964", "59264", "59870", "24332", "56709", "20453", "12652", "17130", "23837", "46081", "2952", "53564", "67533", "34065", "30359", "20165", "45305", "20458", "55520", "67716", "26720", "68282", "27555", "69535", "56369", "34193", "62827", "66100", "9501", "4695", "55404", "43849", "9033", "70475", "10602", "34244", "40276", "24784", "53278", "1217", "44645", "39648", "17633", "4466", "40140", "21452", "5636", "1030", "42015", "52901", "40067", "30847", "25886", "15315", "5790", "70117", "9720", "70315", "20422", "51269", "19879", "36884", "68416", "50746", "53266", "65472", "38116", "23913", "7145", "31589", "2611", "42478", "69329", "68450", "12392", "30214", "944", "59600", "71625", "25261", "20290", "71489", "56388", "39152", "43873", "58276", "25667", "5690", "46139", "7675", "11906", "71334", "10202", "3414", "26117", "19351", "66713", "31769", "26148", "40232", "41728", "46318", "24864", "2338", "72426", "35041", "51633", "10265", "3052", "18909", "5909", "20325", "60416", "57256", "42300", "57760", "2409", "52026", "56718", "62894", "29746", "7283", "24046", "36193", "32690", "40748", "56726", "51469", "31888", "57419", "52286", "58984", "28491", "73208", "27108", "62753", "23459", "54577", "37906", "69808", "17460", "71709", "6916", "21924", "69490", "12635", "28313", "22670", "52388", "31289", "55528", "56393", "9638", "29611", "56048", "51842", "23622", "52719", "7805", "24420", "8708", "28095", "69102", "52973", "43022", "72469", "71836", "7269", "1803", "68469", "42697", "44205", "48415", "72891", "47553", "46287", "27163", "5273", "66623", "59255", "68314", "41032", "1729", "11131", "44481", "29211", "31954", "72110", "71791", "55974", "18296", "41581", "56", "11738", "70231", "64183", "61873", "27917", "37155", "60090", "13303", "29900", "14031", "7157", "31212", "13875", "24306", "5987", "8548", "70240", "38886", "49559", "25170", "31473", "59691", "29494", "60223", "45016", "35420", "27152", "7737", "26397", "57119", "15776", "4507", "34742", "31989", "22898", "20776", "25750", "894", "20692", "41285", "58705", "43009", "6738", "46032", "31796", "24741", "13455", "68310", "20334", "6055", "56658", "8497", "50823", "29022", "14505", "28303", "2220", "25483", "14648", "62902", "8649", "38336", "18863", "60672", "27919", "37416", "4928", "65121", "67270", "45615", "3261", "28914", "4444", "72941", "51095", "1776", "1388", "32290", "27887", "28730", "44464", "2516", "16617", "70492", "28547", "10248", "28976", "66969", "3397", "65674", "42904", "16431", "46118", "57742", "6332", "21456", "64904", "28733", "32524", "65089", "31491", "12171", "6494", "6914", "68339", "56398", "24638", "9454", "17101", "21539", "4410", "6349", "55432", "32671", "61324", "69853", "16162", "8123", "69550", "39608", "15408", "54106", "29783", "8854", "2647", "45038", "55467", "26855", "8448", "7876", "25047", "10030", "45287", "45371", "14022", "60917", "71406", "57543", "16092", "43332", "33603", "41652", "47458", "51222", "67224", "32019", "38631", "59090", "41509", "35898", "48167", "32437", "67234", "25405", "7034", "6856", "18491", "72710", "16160", "41771", "22771", "5834", "7619", "50912", "71849", "17673", "6624", "63601", "13065", "19458", "12966", "27027", "33266", "52888", "63952", "49010", "48780", "68805", "2489", "57494", "4117", "29287", "59667", "43102", "29917", "46945", "42034", "14751", "70739", "68104", "45136", "14105", "42030", "10420", "28569", "28801", "21189", "32212", "20416", "61282", "30033", "22480", "22825", "33327", "45104", "27864", "56844", "30659", "19341", "33339", "23641", "23872", "334", "5026", "66846", "38684", "36697", "13696", "28386", "23527", "39535", "19707", "63549", "47929", "58272", "57569", "3201", "19585", "72339", "30246", "34527", "52142", "48318", "38909", "47567", "53043", "10847", "69767", "45759", "15508", "49508", "55409", "40829", "68690", "3853", "23765", "16611", "24281", "20826", "14", "15729", "49747", "24676", "20054", "43955", "41227", "58401", "32076", "43805", "53952", "177", "71058", "58918", "20203", "55356", "22846", "54180", "12354", "19429", "27067", "3151", "34892", "71507", "7114", "72161", "10021", "48192", "59418", "66659", "27485", "55913", "21185", "15804", "12083", "13366", "24646", "50573", "22006", "15957", "7352", "63234", "3891", "66655", "18276", "50416", "47491", "10313", "32119", "25253", "42820", "2547", "999", "42514", "8997", "57412", "32816", "11172", "46523", "6296", "61883", "37032", "39530", "71038", "25041", "2226", "69020", "18470", "5200", "61388", "18110", "32438", "13570", "23532", "43792", "15679", "53693", "59284", "18493", "33156", "71276", "11039", "26190", "54767", "59710", "36041", "70557", "12193", "7401", "41127", "29013", "7385", "30333", "1923", "21199", "24491", "36633", "32719", "19654", "41754", "36747", "53395", "49879", "19966", "4644", "31593", "43055", "13867", "27637", "35966", "51015", "27771", "60979", "49229", "2180", "8228", "62330", "54247", "42983", "40135", "76", "13863", "17940", "37986", "29087", "45436", "29532", "21358", "574", "3170", "46365", "11447", "56847", "70616", "32279", "31369", "53918", "36568", "44070", "15667", "35231", "10525", "11003", "59420", "41301", "19028", "32676", "53534", "27705", "57101", "48847", "50388", "29858", "28992", "14412", "15863", "13320", "55596", "54266", "17480", "41272", "26977", "6820", "29829", "7304", "31170", "57568", "47537", "44131", "68362", "2434", "3456", "25195", "40023", "11469", "69374", "46295", "37862", "46195", "2127", "48469", "27208", "44944", "70206", "48257", "10734", "25989", "3556", "28567", "44775", "63998", "72466", "9769", "7091", "49593", "18031", "68107", "40495", "6626", "26972", "50461", "70960", "44495", "25708", "49005", "59724", "66602", "56641", "3469", "27976", "26236", "49281", "38989", "57578", "22495", "14570", "69463", "44049", "11325", "6154", "31738", "6809", "19212", "3778", "17564", "55513", "2243", "27579", "62754", "9786", "6285", "50012", "40049", "50470", "10249", "44515", "27639", "21779", "59177", "3262", "63596", "69443", "8215", "68829", "66978", "41215", "41166", "39923", "11054", "52495", "5308", "48638", "18858", "14293", "26439", "3689", "67084", "58148", "23403", "25942", "28038", "48316", "55412", "30683", "33737", "58546", "44423", "65120", "44892", "63784", "13026", "52710", "24035", "62374", "51537", "62463", "3366", "37391", "25419", "8251", "46410", "22510", "32090", "59077", "44772", "59638", "68959", "73198", "10543", "20204", "21150", "73096", "55228", "70583", "14457", "36399", "24600", "7344", "30516", "49568", "66120", "36471", "38085", "33210", "343", "9676", "37934", "15476", "71317", "16531", "10744", "59917", "36841", "59091", "46175", "72209", "13489", "52158", "38334", "4201", "59448", "17225", "8942", "55911", "60999", "59810", "49350", "53988", "7366", "71571", "18484", "67671", "12218", "21234", "9895", "1745", "2268", "62324", "42407", "53313", "38962", "39911", "67225", "28228", "33018", "16141", "51539", "26558", "71170", "54858", "48563", "41791", "55539", "47941", "25569", "65098", "50205", "69447", "20765", "73145", "45138", "32598", "25095", "67041", "3579", "57924", "73245", "62739", "12532", "560", "14045", "8083", "2665", "11804", "44365", "51200", "66687", "31920", "23187", "54086", "1133", "3142", "40097", "11677", "51068", "49959", "34347", "13262", "45204", "31743", "2238", "29891", "8232", "68539", "64026", "63745", "65766", "58030", "8243", "50107", "25421", "33387", "64410", "71341", "39392", "27463", "65132", "33620", "4072", "40961", "17706", "5797", "54299", "40017", "9107", "41866", "44625", "460", "68551", "27447", "52930", "6028", "50502", "16297", "64149", "66992", "54819", "54420", "23325", "54050", "26633", "3836", "65364", "28163", "60953", "14888", "33504", "7714", "71550", "45620", "31837", "64220", "17941", "10240", "14825", "19136", "42470", "43266", "19142", "64831", "11550", "52634", "41643", "38252", "16583", "52301", "70321", "69451", "33279", "9353", "33003", "62997", "14477", "23545", "36526", "56784", "68901", "41839", "65484", "42453", "45286", "57366", "10247", "19480", "63941", "22092", "20798", "25266", "59798", "18247", "37856", "26873", "22356", "19948", "52354", "48962", "20514", "59612", "7687", "64053", "59307", "28201", "50417", "22390", "34806", "72565", "38156", "24116", "32776", "10494", "20833", "14281", "14026", "26146", "9310", "52735", "63023", "29988", "70055", "30510", "6567", "12287", "51871", "37797", "53238", "14349", "44943", "3873", "53281", "40519", "42902", "51417", "32482", "18655", "12084", "66294", "61510", "16384", "58561", "36237", "41915", "45927", "69202", "55983", "65755", "62186", "60480", "44977", "34701", "69618", "72219", "68276", "14410", "44000", "20053", "37082", "38954", "21986", "25642", "14005", "8187", "28888", "24381", "1317", "50233", "61685", "42885", "29996", "37270", "33168", "58752", "15333", "1832", "824", "70602", "19088", "67200", "5616", "6672", "29756", "10753", "18861", "44098", "29969", "3871", "34660", "52518", "4181", "6872", "45748", "20339", "28444", "52951", "40837", "9622", "57054", "56274", "23475", "66754", "66985", "26163", "51822", "13478", "59510", "61030", "39303", "58467", "3528", "39325", "11425", "40942", "11412", "44405", "66092", "57067", "72238", "71804", "62642", "64488", "46512", "68424", "46859", "7544", "37371", "46784", "37261", "21783", "56560", "10960", "26893", "27939", "58901", "65754", "59257", "48966", "60506", "54495", "53823", "10277", "10996", "6341", "32939", "36480", "50090", "16367", "8173", "1730", "2098", "6523", "72449", "24751", "25894", "71472", "69152", "7327", "35491", "11339", "63479", "10721", "58468", "43292", "58558", "38193", "14996", "20752", "64426", "59060", "41892", "22373", "56473", "46464", "60212", "4270", "51969", "3783", "36734", "29562", "9301", "12637", "886", "43020", "45002", "3195", "38528", "61828", "18946", "6040", "61455", "28684", "71291", "28259", "7743", "6023", "522", "40867", "68998", "39971", "69417", "72486", "39452", "9687", "62889", "47588", "6145", "69994", "16159", "210", "68440", "49133", "25786", "53059", "41673", "18762", "67004", "57076", "36770", "10704", "45269", "38330", "26637", "60721", "66008", "49131", "70981", "72642", "2711", "49748", "6255", "54995", "14095", "18035", "39114", "26475", "61478", "60885", "20460", "28320", "38893", "18083", "65260", "14287", "40750", "38173", "11142", "57418", "15109", "33685", "17042", "26689", "9695", "10294", "12361", "372", "16415", "47661", "73147", "37189", "62862", "62105", "11088", "55332", "43163", "5350", "14316", "68788", "66462", "38207", "52684", "13281", "46255", "21927", "36903", "9064", "65243", "1177", "7191", "50465", "42875", "56652", "41252", "64918", "72603", "37442", "50897", "14660", "27716", "72217", "45897", "62959", "24851", "18698", "21462", "56846", "36118", "50159", "31356", "48625", "19083", "27047", "43184", "62475", "40758", "3358", "45404", "25717", "19773", "730", "21597", "47545", "19600", "27053", "37493", "70245", "61680", "19000", "45327", "42241", "42038", "14269", "48725", "12981", "70965", "5169", "42746", "25356", "17985", "41683", "40486", "72096", "33563", "10550", "47710", "988", "30093", "55253", "12476", "64374", "7454", "19307", "34811", "44313", "30405", "52092", "46687", "10188", "32171", "49497", "8845", "16486", "24892", "46393", "40406", "59919", "26800", "61772", "7161", "29965", "4163", "22085", "28910", "65616", "50997", "68638", "46978", "55655", "49850", "38015", "69290", "60545", "72592", "19249", "65585", "60948", "64561", "62970", "55931", "64627", "15359", "8164", "41163", "14219", "68932", "19674", "61978", "39037", "66888", "33594", "20650", "8847", "44077", "7727", "60572", "21696", "7869", "52318", "72523", "35279", "58609", "71547", "2522", "1242", "32727", "2780", "23352", "42881", "21197", "33463", "57888", "22169", "11221", "59912", "53248", "32942", "66309", "25456", "29122", "60709", "63483", "35174", "22498", "19112", "69759", "18152", "24225", "53257", "35424", "43160", "38904", "5445", "5564", "20820", "3160", "490", "39727", "59233", "54826", "48674", "17754", "36913", "42780", "48232", "18255", "46928", "11484", "28332", "5251", "66767", "16868", "6263", "53481", "58083", "51473", "72852", "40444", "17486", "9389", "19448", "1492", "59972", "62298", "12766", "18380", "36917", "16808", "46427", "14225", "21087", "7562", "49379", "31925", "70413", "27105", "63678", "68933", "44431", "3976", "38670", "15951", "41310", "4829", "60741", "55785", "4775", "73202", "63816", "39010", "45880", "68948", "2460", "48432", "42000", "1941", "40700", "53772", "21787", "39489", "17613", "18790", "2474", "19587", "59340", "52721", "9694", "47687", "16572", "20146", "72678", "24508", "67275", "20411", "46327", "53083", "69120", "7443", "24405", "53911", "70008", "13206", "11704", "46238", "67332", "72385", "1035", "73041", "637", "23745", "5792", "46856", "41478", "67190", "62288", "14753", "3068", "10367", "68791", "13094", "64013", "18134", "26947", "45015", "3585", "32599", "58156", "28264", "41473", "10507", "58931", "50329", "63053", "4372", "49364", "65845", "57559", "20024", "19097", "5384", "29571", "39229", "66945", "28894", "36413", "59681", "27042", "50843", "38544", "59929", "23121", "55104", "4102", "72651", "30022", "68130", "41256", "30403", "67963", "19132", "2750", "56989", "12865", "70941", "12612", "17267", "30904", "34093", "3681", "18586", "66322", "1357", "52438", "36127", "58035", "40985", "36324", "57649", "37616", "64557", "46065", "29540", "9875", "36466", "18147", "55792", "15820", "34590", "68573", "34992", "12765", "35433", "66098", "22714", "55893", "38416", "4499", "40813", "38444", "17624", "71897", "47129", "36308", "37958", "49211", "5787", "35842", "12929", "59811", "51663", "14426", "20150", "22603", "18637", "51917", "67628", "70279", "66728", "57244", "18089", "39928", "68206", "62318", "1189", "12913", "67375", "44002", "44270", "56021", "21348", "46296", "26449", "69003", "11141", "25011", "61650", "10315", "18667", "20380", "18375", "21214", "36244", "52783", "71295", "1822", "14189", "15033", "11380", "72379", "52203", "52278", "62240", "40698", "16660", "10344", "65310", "48258", "56604", "52960", "68415", "48102", "71329", "45389", "45261", "61693", "43528", "29082", "31602", "61923", "25611", "15735", "13527", "28028", "65813", "3154", "65537", "31514", "45673", "25097", "36135", "34384", "25320", "41306", "2340", "8137", "31680", "57210", "58471", "16877", "26926", "34530", "65769", "12024", "721", "18755", "13451", "6652", "31475", "3097", "72568", "26669", "15543", "20162", "48135", "60908", "43623", "22922", "56167", "3215", "18469", "70221", "21798", "14408", "24463", "37405", "58110", "19959", "10325", "59523", "11353", "17292", "56684", "43329", "20908", "43901", "23090", "33094", "40328", "50566", "49437", "4110", "32780", "53877", "12369", "38944", "56693", "44699", "38580", "69587", "29530", "68536", "64357", "51799", "13647", "20254", "16847", "5968", "62799", "36043", "39271", "58995", "65328", "55779", "34046", "5221", "38468", "30083", "11565", "18219", "45333", "8104", "43315", "68675", "58766", "62373", "33688", "30048", "71605", "15339", "14539", "65822", "65665", "5482", "58214", "10314", "51110", "15499", "57440", "33080", "49474", "20408", "42683", "21336", "8097", "43832", "70422", "60906", "32754", "48818", "8928", "18985", "51184", "71619", "71123", "39043", "68421", "34571", "64785", "55134", "56741", "36102", "29659", "25945", "17055", "13493", "60408", "56590", "53016", "240", "72530", "61240", "66286", "70840", "70189", "44420", "46486", "9646", "56758", "44067", "23011", "44768", "39390", "60791", "64881", "13350", "41148", "9121", "8562", "16117", "52185", "48977", "66979", "34255", "10011", "58599", "58600", "39549", "60580", "32491", "55984", "51580", "55495", "70365", "15353", "13674", "63644", "20437", "23895", "34352", "39853", "35873", "30008", "14876", "40311", "55908", "46165", "31892", "17944", "23967", "39219", "27169", "413", "65633", "21618", "40530", "53722", "37829", "17657", "30082", "25838", "45699", "20730", "70931", "70268", "39195", "49628", "2994", "27331", "35545", "68513", "36656", "10281", "8905", "41289", "39036", "11243", "6605", "45894", "18210", "32329", "15959", "5399", "38988", "49708", "53023", "21207", "27226", "30655", "52412", "60892", "52926", "61535", "39291", "8662", "31610", "51460", "65227", "19616", "48491", "32997", "4386", "19743", "42806", "17168", "13432", "15660", "2386", "25381", "55471", "70095", "51069", "54170", "13744", "21698", "57631", "118", "44840", "66709", "64565", "36123", "26673", "17636", "15851", "39462", "29006", "63016", "8170", "15903", "5914", "41308", "55964", "40541", "71793", "10237", "67429", "35112", "15789", "39056", "23205", "42560", "30365", "54337", "33911", "66077", "55273", "22776", "45447", "61636", "330", "8748", "44056", "43942", "7112", "50467", "36392", "33178", "39969", "27752", "71183", "41105", "27551", "30746", "15492", "50641", "40926", "32043", "53489", "61625", "23983", "73088", "32258", "12432", "66792", "62542", "24533", "35144", "36553", "37175", "23962", "34142", "20964", "50650", "62039", "25564", "7599", "8074", "21163", "60346", "6708", "66002", "49682", "53529", "4272", "62208", "47362", "29633", "57871", "41161", "70086", "21623", "2712", "714", "18699", "58868", "22712", "28308", "44126", "53962", "39476", "42815", "25822", "59560", "15936", "57282", "60533", "72738", "20612", "46751", "33749", "3620", "44269", "35964", "28905", "19015", "34643", "40178", "11522", "10730", "442", "69855", "46668", "46382", "4782", "14634", "24429", "53987", "59736", "39970", "10413", "53138", "9154", "20537", "31611", "39119", "23389", "59827", "62275", "34944", "30370", "50610", "40082", "1432", "70754", "13196", "2070", "13999", "68106", "61096", "14821", "25587", "792", "4759", "62672", "28030", "39150", "44081", "24557", "47004", "3755", "67281", "6492", "64544", "13503", "67945", "24620", "16728", "33333", "44753", "36830", "10234", "60478", "61598", "65803", "48869", "60756", "34071", "21781", "59807", "35017", "53579", "16787", "9439", "5748", "13758", "10363", "8229", "1954", "22622", "33754", "55506", "24157", "5246", "56711", "60362", "36597", "35649", "54629", "64641", "35044", "40424", "40967", "33892", "21740", "72561", "15031", "46216", "19258", "59575", "21313", "31774", "12819", "46461", "32064", "57171", "45935", "71576", "63351", "17205", "70432", "64098", "30961", "15846", "9788", "15535", "35334", "70836", "53389", "37444", "37605", "33929", "42690", "17776", "36233", "11959", "35631", "4953", "59234", "34664", "14617", "5641", "30053", "41132", "23751", "40828", "7087", "28194", "59757", "3614", "5392", "49913", "19488", "47965", "24497", "70239", "58099", "39736", "35483", "64895", "35449", "72779", "12992", "17599", "66408", "55969", "62758", "39927", "14563", "68659", "6493", "39065", "17508", "18832", "33948", "29580", "17815", "43411", "12214", "2017", "43487", "16866", "65891", "64230", "71229", "9069", "31420", "10250", "52840", "52843", "71221", "29787", "30635", "62119", "54892", "26188", "41960", "7522", "52772", "2046", "33828", "15256", "14639", "69980", "31347", "50036", "951", "20062", "57548", "55530", "54265", "63272", "9002", "64340", "63254", "18401", "29316", "2043", "65965", "24796", "68168", "67729", "26951", "10986", "30529", "5559", "57669", "15116", "35393", "71224", "28200", "33910", "27469", "34663", "18049", "45829", "72861", "1510", "72734", "19333", "4329", "3440", "49876", "43998", "49571", "37421", "8294", "12266", "60341", "58946", "65984", "46209", "3762", "40659", "8793", "20693", "18977", "69874", "4030", "61254", "58541", "6917", "51006", "40721", "54332", "41920", "71972", "23221", "68947", "26194", "57761", "38154", "15699", "41547", "35214", "12777", "47863", "6706", "69842", "55197", "55443", "58517", "37381", "56479", "72664", "38480", "45365", "32572", "15065", "28423", "65199", "5298", "38343", "50639", "18198", "18016", "9863", "16262", "12660", "66402", "43910", "3986", "3017", "20180", "56804", "11715", "6988", "4710", "72019", "15772", "40805", "20635", "59172", "47176", "62554", "24308", "21448", "13182", "66214", "45619", "31232", "29769", "61513", "35158", "2247", "57767", "68271", "45616", "68117", "71877", "32937", "72081", "5676", "57616", "18773", "59413", "11712", "62467", "8861", "66906", "64316", "67709", "7778", "9302", "32369", "43180", "47722", "3870", "68761", "37722", "28286", "4104", "47475", "34477", "11016", "374", "40565", "61403", "31205", "20598", "31858", "57020", "19216", "21399", "14771", "1377", "61051", "32346", "25425", "27598", "72492", "59698", "4804", "7685", "2875", "5565", "12810", "62992", "57684", "58061", "5662", "54944", "52949", "59079", "45597", "54522", "16395", "47542", "49669", "48394", "20899", "27035", "47798", "24168", "62930", "23391", "17561", "32121", "24810", "49334", "33416", "64348", "66553", "19477", "17861", "3379", "62949", "42335", "62403", "18330", "31741", "8207", "53232", "66636", "24799", "29874", "44795", "39353", "61214", "49051", "20402", "22209", "24026", "63689", "850", "58747", "17829", "53187", "52639", "66916", "42268", "36769", "35820", "73214", "46853", "72472", "69116", "32734", "26458", "20593", "17705", "55339", "15305", "25219", "12909", "32286", "71928", "56657", "57134", "69505", "65958", "46460", "49272", "35413", "1374", "53635", "67998", "13312", "6629", "66248", "66761", "61375", "23940", "43525", "50043", "18398", "8014", "67988", "59249", "63795", "8420", "7679", "10820", "54680", "46590", "11824", "50815", "28406", "44157", "46547", "54895", "2388", "24682", "32316", "71685", "5561", "6150", "9793", "27239", "62867", "8375", "15656", "22772", "36740", "17458", "17017", "2602", "1626", "54618", "12814", "9612", "15883", "17569", "40587", "10877", "49190", "39368", "39101", "61431", "34174", "61523", "7470", "25610", "19780", "15063", "31355", "41685", "8510", "50405", "20737", "56866", "59800", "2076", "44198", "51487", "23871", "20801", "46619", "33703", "26794", "71251", "53406", "37473", "13787", "72021", "65627", "406", "49744", "2907", "42259", "72489", "22114", "68473", "61317", "50286", "11749", "3580", "51127", "55682", "58788", "1886", "25987", "65518", "46153", "53140", "38662", "40648", "5230", "12626", "47228", "933", "1994", "6018", "62130", "45361", "59299", "55240", "69585", "27055", "23972", "7061", "11037", "4519", "55505", "13476", "72757", "24131", "33433", "61594", "39030", "46850", "72425", "25647", "33637", "47686", "21979", "42848", "32345", "44737", "30342", "60081", "7222", "43771", "19181", "36717", "69247", "25840", "60590", "49572", "4703", "44228", "20508", "37293", "47571", "47956", "63293", "23084", "19009", "24710", "22720", "40978", "64549", "30421", "66877", "12421", "61576", "72929", "16041", "35574", "17954", "46976", "27629", "38126", "10966", "44904", "42606", "10384", "7671", "32605", "37500", "5801", "18304", "59116", "38786", "23982", "61069", "7905", "30138", "26330", "772", "37764", "57242", "50229", "11711", "64606", "11509", "23852", "72282", "28543", "3339", "48739", "57279", "21858", "1147", "60927", "28315", "20539", "17149", "54005", "17033", "70601", "40188", "29139", "54361", "4946", "4086", "6186", "69726", "23716", "5153", "382", "36263", "27473", "26251", "37975", "53894", "39839", "37468", "20836", "51041", "472", "62989", "25668", "23877", "68765", "65312", "19881", "33076", "63953", "18583", "20843", "3483", "17822", "9736", "45364", "67039", "8800", "48917", "42181", "22781", "33974", "71126", "43763", "15720", "24478", "9873", "24874", "61052", "60140", "34029", "27400", "11832", "22758", "2381", "67136", "64419", "16575", "51680", "38209", "37361", "42439", "61801", "34671", "64074", "2648", "12732", "34782", "17563", "54596", "50558", "11575", "16339", "12442", "12902", "8108", "16151", "62174", "30639", "1452", "53538", "24780", "36748", "49279", "5101", "38721", "19609", "70418", "41775", "67774", "62496", "52266", "17021", "52249", "40018", "31479", "2844", "44299", "6369", "26086", "12567", "58672", "2044", "61578", "28229", "63800", "22186", "28451", "25016", "64463", "19478", "49240", "71652", "17581", "67873", "62264", "23382", "26626", "15643", "66374", "72835", "68902", "4356", "24029", "25264", "38308", "45872", "14325", "5336", "10304", "7254", "65209", "21395", "44169", "73225", "630", "22609", "25685", "28413", "58673", "25525", "6032", "30432", "71753", "69641", "35402", "61193", "28349", "17083", "40839", "3967", "5494", "13589", "50357", "17400", "11875", "18950", "16665", "20957", "54120", "70828", "6888", "56619", "23318", "56957", "15096", "37060", "30177", "38864", "58173", "6328", "71178", "8606", "15281", "57810", "33818", "66284", "60701", "60309", "51709", "8525", "37670", "48253", "28105", "26467", "10350", "35999", "53057", "13604", "36080", "23854", "52420", "56052", "22844", "63000", "4063", "50122", "21548", "27344", "15761", "46024", "23400", "59857", "1619", "1734", "21387", "17456", "65831", "7946", "41714", "53837", "22849", "5891", "30226", "32292", "53884", "24336", "16270", "8625", "35554", "515", "39912", "16056", "36464", "20524", "34774", "3006", "70311", "31994", "55583", "63990", "59058", "41213", "58959", "27602", "9672", "46292", "31499", "58422", "37319", "15463", "49683", "23447", "1201", "24595", "27330", "49732", "68710", "53228", "41082", "53568", "27358", "30508", "43557", "66751", "10474", "20973", "25955", "26872", "3217", "68463", "60458", "46057", "19453", "61950", "68616", "13048", "473", "36824", "58490", "27187", "18480", "39710", "43866", "7171", "39858", "41073", "22511", "11223", "14249", "40554", "70324", "44359", "13063", "46162", "21776", "66976", "53736", "34762", "72731", "18224", "60318", "12441", "68702", "42334", "7077", "46441", "36352", "51609", "25280", "26739", "48789", "23752", "68489", "55680", "45217", "1864", "18825", "67694", "37477", "59501", "12773", "65228", "45788", "11673", "267", "50724", "41016", "25148", "46847", "3157", "64825", "42705", "16490", "55334", "27181", "47842", "50056", "35766", "22294", "37790", "65512", "32282", "62436", "9630", "37417", "14742", "23438", "36823", "55668", "69979", "72099", "31683", "56012", "18561", "54173", "3473", "69060", "28802", "3369", "10229", "55153", "55873", "12597", "43385", "11019", "69426", "28191", "2909", "9215", "67154", "56045", "51581", "21624", "31904", "953", "7367", "27970", "62724", "23473", "54119", "27723", "15709", "57641", "23860", "51198", "71926", "59433", "33635", "15389", "49994", "30049", "28246", "32152", "66052", "36348", "37578", "6059", "36596", "59149", "21308", "48182", "18815", "20302", "39395", "70904", "14910", "33340", "30830", "54739", "57978", "9506", "38937", "14492", "23095", "40713", "40409", "57997", "4344", "20034", "1000", "45464", "59488", "65097", "41499", "32014", "14120", "19636", "58835", "18196", "63464", "6090", "20603", "46562", "39162", "63462", "30795", "41957", "9957", "9946", "1154", "38559", "29508", "11117", "15277", "37197", "19888", "41198", "21027", "762", "30465", "51912", "70288", "61746", "46799", "54638", "24276", "55601", "62234", "72811", "18442", "15579", "34034", "70325", "50883", "29502", "14008", "15706", "21763", "19042", "30105", "26755", "34038", "54084", "53515", "17094", "44068", "31236", "71153", "34098", "60696", "20092", "40600", "47856", "7070", "57689", "16114", "11922", "14736", "66059", "23258", "45323", "10524", "50424", "66604", "72543", "51658", "19170", "5439", "13932", "49425", "9945", "54947", "31981", "63260", "70629", "18414", "51982", "56981", "32150", "27183", "53658", "64145", "21178", "40155", "63012", "5712", "42101", "58596", "61611", "36807", "32431", "61229", "55351", "57868", "10729", "11040", "28756", "29954", "62134", "49711", "33374", "14707", "33700", "38619", "20029", "3140", "35306", "6921", "68389", "4767", "58544", "65719", "1597", "46628", "66126", "57799", "16", "49194", "56037", "4901", "2170", "52251", "36300", "49009", "64772", "71730", "30450", "4179", "40512", "15081", "16562", "61981", "38491", "7650", "49632", "65048", "43377", "65130", "35498", "48219", "9341", "40163", "30835", "4265", "2629", "16905", "22649", "2569", "51117", "12399", "26084", "50618", "39078", "31814", "40778", "3511", "57541", "2511", "22456", "67949", "30735", "31240", "50048", "401", "72791", "69780", "34235", "58989", "32995", "38118", "56899", "922", "82", "13585", "19210", "57395", "56462", "73042", "31923", "46721", "32167", "3057", "44304", "38808", "50651", "167", "52370", "7584", "65923", "42807", "14790", "13901", "27461", "41689", "52796", "72618", "13316", "70704", "72453", "47377", "64616", "46849", "43098", "71331", "20299", "27278", "11208", "56810", "822", "51700", "30412", "218", "13996", "41119", "38429", "2083", "61422", "26284", "31924", "6273", "45199", "5418", "14448", "59840", "35084", "26608", "67834", "29631", "39670", "22338", "51911", "5724", "22126", "14750", "60026", "2604", "33495", "6895", "33207", "65038", "1261", "42146", "24712", "8142", "55832", "46599", "55098", "42750", "55433", "50376", "36827", "37655", "52244", "40881", "60985", "8874", "2402", "24944", "22694", "24863", "33300", "48281", "37251", "17733", "17750", "14422", "48267", "65740", "32265", "3374", "58029", "63332", "39263", "14358", "2707", "4363", "8590", "21357", "34610", "37526", "31689", "33881", "21046", "14922", "30681", "32146", "52269", "28051", "16478", "57027", "51952", "18092", "6698", "37016", "16122", "144", "26350", "5880", "69615", "5029", "49501", "53545", "62072", "30770", "38873", "4840", "17384", "40525", "9906", "55762", "44059", "25201", "66908", "6524", "19807", "6079", "20091", "9919", "22144", "67465", "42802", "46564", "32025", "61295", "35517", "3707", "55398", "39009", "48088", "16163", "67640", "4741", "65424", "63931", "30781", "43583", "7806", "16393", "33554", "68364", "62416", "35473", "9186", "34073", "40069", "49421", "26836", "58751", "48228", "31388", "52732", "35220", "49337", "5536", "22227", "61620", "58351", "30745", "12270", "23269", "50002", "59051", "48926", "45670", "49678", "9221", "59894", "50114", "44623", "61302", "1522", "49728", "14604", "19491", "33564", "4089", "72", "15036", "39144", "65910", "49911", "5228", "51818", "17869", "45917", "64926", "22764", "49557", "44835", "49786", "59143", "17234", "59496", "60495", "42316", "45552", "43146", "4056", "72581", "3495", "5346", "17122", "28837", "6367", "9452", "53254", "62829", "8346", "69540", "15687", "13227", "44506", "19356", "11396", "48770", "12300", "61596", "26140", "43723", "49299", "27920", "35799", "6009", "26773", "37555", "61220", "16411", "66786", "57773", "18408", "45806", "65572", "19225", "34443", "69117", "19605", "43780", "38710", "35909", "27442", "14832", "56928", "47029", "11747", "13523", "61322", "50179", "17089", "49737", "14770", "59576", "68794", "53757", "3888", "72301", "26172", "34715", "32174", "44361", "22454", "5518", "42149", "4411", "47084", "5609", "71002", "47446", "53171", "54295", "26345", "52344", "40963", "47707", "31372", "32162", "3619", "57212", "44997", "15861", "72770", "38652", "3148", "69113", "46623", "71597", "15937", "20702", "31254", "24055", "66110", "41913", "66746", "42891", "60162", "18306", "71277", "41199", "18366", "51048", "26131", "48862", "33154", "29246", "47872", "2057", "3856", "72103", "59837", "57346", "36003", "32467", "30227", "71266", "2542", "58768", "49183", "45870", "55102", "1735", "54775", "71258", "68232", "63527", "68529", "70792", "4150", "31424", "10601", "41641", "44748", "4814", "53799", "21394", "55397", "7845", "526", "4924", "62879", "69551", "19682", "62180", "42424", "45985", "37563", "9169", "61958", "60372", "121", "11080", "12489", "53289", "28209", "47454", "9281", "67241", "14762", "67706", "59524", "64243", "47830", "13369", "32556", "56966", "43894", "24106", "69191", "54003", "32371", "42129", "60513", "46062", "33129", "42145", "23808", "6841", "54035", "62362", "17913", "61225", "63604", "44927", "53746", "37525", "21165", "50296", "54827", "65703", "52499", "17690", "55370", "34756", "30030", "45688", "50348", "29688", "68445", "57528", "72657", "14595", "66501", "38049", "8492", "55573", "69809", "56915", "62181", "3003", "29224", "27803", "17753", "72629", "22504", "46312", "10785", "32526", "13852", "46966", "51129", "72051", "9184", "3519", "58039", "26487", "60949", "59735", "63600", "17453", "1043", "1529", "54407", "25735", "57094", "13692", "7570", "46579", "39310", "34293", "58407", "41177", "32917", "70737", "29366", "19024", "10066", "54691", "48687", "39572", "31734", "70699", "68768", "17957", "6497", "43877", "14625", "49081", "70316", "42077", "15652", "5299", "27576", "14403", "15882", "12215", "57490", "21318", "36068", "47847", "54520", "45063", "4353", "30195", "61785", "13162", "12665", "37129", "9678", "7207", "10375", "19733", "48074", "69817", "47014", "68816", "24737", "17597", "45595", "69252", "22211", "46989", "26917", "30107", "1640", "64200", "64297", "1521", "30627", "20913", "25061", "50898", "17799", "689", "4349", "63901", "5869", "51233", "51587", "62649", "12316", "57627", "59367", "39724", "69178", "483", "36022", "6898", "14313", "2812", "29700", "33396", "47548", "45229", "39488", "333", "45393", "68446", "38211", "7991", "40983", "67432", "60640", "10266", "69269", "25515", "10359", "71102", "41004", "70586", "13251", "11216", "72628", "7747", "61572", "59267", "55025", "45250", "55819", "10542", "53551", "18875", "51771", "6152", "7992", "51794", "64113", "6791", "19558", "19071", "259", "35477", "34825", "7212", "40435", "28923", "58085", "29785", "21810", "32869", "24734", "31547", "2657", "28216", "19835", "25467", "4258", "46729", "3691", "70781", "59841", "48634", "32047", "16012", "59336", "61097", "33354", "11326", "64260", "53581", "60909", "42473", "30379", "25230", "39394", "56794", "19474", "44332", "35988", "63767", "25927", "55921", "51206", "7780", "54333", "55329", "27568", "65595", "16294", "39995", "12919", "48813", "61890", "11835", "71786", "37874", "67317", "71650", "5426", "24284", "66807", "57108", "66016", "70604", "14499", "52529", "23853", "36384", "12521", "21215", "43138", "1259", "53963", "21273", "20985", "48890", "10201", "2876", "60846", "24878", "36064", "59470", "7368", "34994", "72396", "12452", "49059", "61206", "161", "565", "43154", "19379", "64021", "740", "63669", "19929", "33800", "38565", "39869", "1897", "35373", "68259", "67291", "35241", "7319", "53661", "8880", "6160", "53271", "887", "4891", "60969", "1699", "45635", "22792", "39584", "33900", "34263", "33199", "70848", "42104", "48783", "1159", "47013", "25958", "63192", "60529", "69043", "69778", "67564", "48865", "344", "13724", "21217", "20147", "37864", "45245", "42238", "16252", "6103", "47719", "34299", "15531", "1412", "4979", "51065", "4145", "25440", "19029", "36826", "58257", "44899", "54557", "1958", "24000", "70923", "15017", "13894", "22861", "8086", "38696", "7482", "54455", "12726", "10389", "46545", "43483", "3107", "51240", "60463", "55483", "57870", "38045", "4419", "52139", "59881", "40553", "21259", "70847", "62153", "2831", "46823", "5725", "13588", "68789", "38876", "50503", "71358", "8549", "50966", "66782", "56754", "1350", "42643", "50736", "64644", "60822", "8", "61954", "42869", "30721", "46937", "16943", "18176", "13625", "67269", "22146", "53361", "61089", "10714", "20480", "51695", "51305", "5448", "2689", "10866", "26480", "45649", "1814", "22308", "10492", "17909", "59904", "9621", "19275", "63387", "48243", "7705", "33615", "72001", "53372", "45884", "16303", "20115", "45512", "21640", "23274", "64745", "22382", "20995", "58172", "10919", "30944", "42789", "43735", "27434", "49372", "11849", "640", "36802", "62508", "4074", "55453", "57360", "56856", "54613", "29545", "55420", "10870", "68817", "15348", "17297", "7130", "53053", "59252", "49688", "60267", "40065", "50421", "32158", "50676", "51113", "10807", "68633", "21729", "2663", "64800", "12304", "10818", "16552", "11198", "8033", "38534", "16344", "65634", "73027", "59794", "1194", "39470", "34316", "67782", "15791", "20141", "15427", "63365", "6360", "49658", "4153", "56087", "48004", "2529", "54822", "15812", "59883", "31535", "2035", "17339", "8339", "37390", "43275", "5377", "4625", "71451", "21033", "2155", "29802", "18111", "44254", "48515", "9467", "18951", "29577", "35478", "63176", "53539", "63109", "14049", "26514", "19364", "37828", "57905", "1712", "9642", "13299", "122", "71792", "60176", "3576", "12595", "13010", "66129", "58142", "21575", "61103", "37286", "5111", "65425", "51872", "31476", "68169", "1651", "22389", "1963", "23952", "49290", "11374", "6465", "47550", "26903", "34859", "24633", "59382", "6105", "16705", "30649", "67050", "3119", "38782", "39074", "29134", "63290", "63188", "29646", "72499", "56483", "14374", "64279", "55024", "20003", "54694", "53107", "46117", "19245", "24549", "52243", "47209", "20350", "33842", "43661", "67611", "37516", "27261", "32298", "23806", "71923", "6133", "64164", "23028", "27230", "40196", "27247", "47699", "49989", "13459", "539", "62210", "28734", "7772", "31312", "70233", "39116", "41430", "45775", "67010", "9104", "59864", "49667", "4730", "4761", "23439", "33134", "10848", "58678", "17331", "16282", "49694", "39039", "25183", "65964", "3108", "63564", "41945", "12712", "40580", "73040", "72927", "15319", "34026", "311", "19638", "6828", "45631", "15105", "33897", "29536", "24480", "25092", "32794", "22239", "47273", "40863", "26537", "6804", "26366", "50533", "56779", "7628", "70869", "53333", "59551", "23514", "11916", "42166", "39709", "7872", "67768", "18087", "57429", "26134", "32253", "28755", "24634", "10563", "14847", "66326", "20125", "26601", "44001", "37345", "1181", "20682", "46671", "42920", "14773", "40610", "63492", "63403", "52876", "1204", "9343", "7275", "36512", "20075", "20239", "51058", "44846", "52367", "49958", "22461", "2463", "24365", "45658", "64052", "55728", "21534", "31798", "10158", "24477", "69880", "12044", "55149", "69764", "52837", "7279", "50306", "25491", "72677", "62482", "46136", "65767", "53018", "16232", "28577", "28017", "34903", "47092", "71823", "46352", "23542", "42413", "24842", "50653", "49726", "42053", "59779", "68926", "53169", "53955", "4424", "11413", "54042", "46436", "13851", "11356", "30001", "60405", "62887", "3867", "5668", "1196", "26782", "29386", "310", "25920", "60905", "119", "63386", "20976", "12681", "21049", "28954", "26493", "4243", "58862", "69583", "67268", "43885", "37195", "64909", "45922", "64815", "27093", "65010", "38223", "51573", "62414", "16215", "73146", "6474", "63752", "14006", "57115", "66142", "37262", "10814", "30580", "21351", "44779", "60711", "52439", "12406", "14311", "23002", "54670", "37959", "43789", "64106", "51900", "8727", "24363", "27001", "49199", "52827", "11962", "35122", "1974", "4559", "32866", "37662", "48702", "32027", "6372", "10941", "54418", "32989", "27074", "20979", "61822", "17189", "34725", "17831", "58465", "26889", "68869", "16456", "29229", "34084", "63145", "28227", "67717", "41496", "51491", "28093", "16640", "58581", "38205", "37125", "60255", "57829", "2913", "34080", "35063", "44627", "41153", "7769", "47465", "70916", "65379", "17555", "11631", "18061", "36356", "2616", "48771", "58352", "54800", "41172", "2806", "62461", "38072", "56967", "40907", "60440", "66391", "8617", "8716", "42515", "11614", "19479", "11298", "25268", "7668", "64339", "31875", "19827", "68161", "15264", "33789", "10388", "10217", "54392", "846", "38400", "35047", "7155", "72336", "19857", "15175", "56157", "53022", "14507", "50955", "1924", "21845", "47173", "58817", "50324", "24632", "37463", "31827", "33048", "34253", "64422", "13179", "4655", "41822", "2577", "60648", "39566", "61189", "46749", "18717", "24290", "7645", "55517", "24128", "44013", "25145", "35993", "69070", "40709", "16929", "62286", "16607", "64948", "8244", "71182", "49017", "64207", "44463", "3916", "32489", "17755", "69635", "51221", "38231", "58438", "31829", "21026", "39666", "13683", "48661", "29593", "1020", "45786", "46598", "65249", "42207", "55856", "38646", "27816", "16550", "53938", "54404", "61239", "45624", "55454", "5218", "27517", "14459", "27491", "61001", "26710", "29752", "62068", "6807", "59447", "4174", "70328", "4042", "25738", "53609", "37869", "11666", "41283", "20814", "39964", "58305", "69394", "46639", "64956", "8044", "65059", "12363", "34136", "71964", "4365", "42693", "29533", "45230", "39334", "61350", "62811", "6004", "57318", "5885", "35615", "53474", "38117", "56545", "27519", "52950", "62427", "24142", "58933", "9874", "56144", "56283", "71146", "36163", "60494", "50170", "32184", "56425", "9296", "34853", "62832", "43269", "65990", "53144", "34659", "5633", "43939", "27370", "4642", "70449", "35502", "16328", "55488", "31656", "2330", "35481", "45996", "72760", "20620", "35270", "5144", "21277", "41873", "42731", "29000", "29534", "69054", "43056", "54043", "53521", "64744", "19166", "44878", "43774", "30869", "4609", "19895", "41769", "35252", "18579", "48413", "72661", "27114", "35263", "41917", "63973", "17215", "35611", "72513", "47439", "47401", "58988", "51574", "70672", "54673", "27324", "28654", "31027", "37452", "30268", "7458", "36899", "27452", "4396", "23718", "1624", "39562", "17659", "31403", "25887", "20950", "3563", "31832", "67874", "26349", "15644", "59486", "14400", "28278", "19787", "38541", "21777", "45581", "23207", "22328", "45513", "64506", "30339", "29894", "15544", "46151", "50534", "26621", "47200", "11879", "71899", "7846", "71575", "25364", "43865", "44919", "6443", "19669", "7629", "51375", "20937", "55894", "29049", "71707", "9730", "13038", "57648", "60629", "31300", "23847", "69674", "10290", "16719", "7088", "7149", "53512", "68564", "14179", "32691", "8117", "59970", "65222", "57768", "12735", "68871", "22699", "35542", "69050", "69145", "37229", "56946", "33990", "63893", "37010", "4987", "59753", "66928", "30183", "48181", "19298", "53615", "65749", "17000", "16990", "23288", "61045", "18123", "62346", "40552", "60039", "27109", "50482", "72159", "17035", "20788", "63167", "48263", "532", "21716", "9342", "35521", "21486", "64232", "10658", "29982", "58411", "45990", "64261", "24196", "63565", "61042", "14515", "38999", "52466", "28454", "20378", "20462", "65093", "38702", "51199", "45264", "73038", "23349", "36890", "33806", "32323", "5471", "67854", "2323", "41204", "30351", "47638", "29090", "34027", "26596", "791", "7867", "56889", "29220", "44220", "63927", "34302", "46265", "25257", "56893", "59457", "22428", "60928", "62849", "52799", "6442", "21630", "280", "2841", "43339", "68142", "16783", "64510", "38764", "4182", "64516", "51943", "54864", "25269", "598", "10253", "41277", "14559", "70726", "34777", "14760", "4367", "16212", "1854", "60742", "13838", "42144", "18629", "20376", "19371", "3416", "52035", "40724", "67692", "69222", "52671", "38865", "45866", "33417", "58978", "18345", "700", "55292", "59012", "6720", "6513", "5866", "38826", "1579", "13243", "6815", "1788", "48807", "18285", "63509", "70625", "69645", "31367", "27402", "58134", "65126", "46720", "21585", "8336", "56397", "33735", "4467", "67656", "63191", "64629", "53045", "17170", "37469", "8409", "2992", "47772", "25338", "9719", "12221", "27968", "54131", "56506", "1928", "15909", "70236", "54196", "39596", "66150", "33297", "10221", "30316", "21916", "46666", "6102", "49033", "26376", "64551", "3120", "16549", "64039", "54814", "25348", "71075", "62735", "59019", "71820", "28595", "12490", "40578", "67919", "20482", "26299", "10457", "41667", "50974", "50031", "30613", "70430", "44057", "62732", "44717", "32099", "33125", "18518", "2573", "63517", "11013", "56771", "15060", "9268", "38080", "72643", "69465", "39511", "56129", "26691", "6170", "54000", "10450", "69325", "39768", "24917", "43751", "48065", "65148", "6013", "56935", "26252", "52866", "65699", "66935", "10269", "43821", "30496", "58865", "9911", "42492", "41542", "30561", "65801", "59415", "5229", "66615", "2118", "37807", "19968", "35619", "68867", "30184", "7444", "59517", "15984", "3809", "22065", "57839", "63622", "30422", "16966", "65856", "10430", "54999", "52517", "12563", "50947", "66799", "19505", "42228", "26249", "61626", "47344", "10296", "35756", "8582", "30147", "35809", "29656", "50537", "5998", "28045", "64225", "19823", "22648", "48940", "23665", "41210", "40550", "68004", "25742", "66596", "68148", "7919", "24679", "8638", "24488", "8613", "37731", "51187", "50478", "69895", "57036", "2859", "68545", "47555", "55711", "40705", "3704", "16678", "19423", "32284", "27662", "32565", "61875", "69929", "69069", "54619", "9644", "4613", "67531", "38320", "53537", "64242", "17942", "63746", "44801", "58059", "44045", "60718", "44751", "42116", "46593", "28773", "30373", "9590", "54409", "24180", "130", "2281", "59570", "1955", "12614", "52239", "2307", "10912", "58163", "22640", "47899", "37905", "60270", "40991", "10970", "67935", "30658", "64334", "8601", "2159", "4255", "68554", "21891", "48517", "43457", "51718", "34096", "120", "16502", "49613", "51106", "27809", "9385", "50068", "45893", "9476", "71467", "60468", "43512", "30400", "34858", "23770", "19172", "39974", "28873", "9851", "37015", "44746", "7536", "68065", "43868", "71151", "22553", "18468", "28578", "39594", "56427", "14700", "67125", "47177", "35736", "46197", "38042", "11957", "61995", "41942", "17830", "36239", "48131", "47408", "66410", "30855", "54321", "44302", "37881", "38690", "17521", "27788", "34295", "40366", "52543", "13311", "28870", "45867", "61698", "32024", "65583", "49652", "8580", "11108", "19267", "68281", "18130", "33224", "64599", "19625", "34513", "31597", "20095", "7633", "56542", "8480", "57471", "2889", "73013", "21517", "35791", "70985", "10332", "42117", "5490", "7651", "27428", "41788", "10710", "3016", "14330", "44651", "67668", "29986", "65480", "61372", "50982", "38664", "29073", "16952", "38082", "30511", "55436", "59117", "40588", "11866", "71009", "2513", "61448", "41405", "22285", "9596", "24047", "19457", "20832", "51447", "63222", "62804", "8216", "72049", "1023", "67322", "13242", "72727", "66569", "5404", "10812", "1326", "25630", "60915", "1058", "46719", "22821", "45702", "13442", "60843", "64396", "33268", "67356", "41019", "53856", "29396", "55925", "32640", "14201", "62702", "46160", "55350", "3649", "59696", "6219", "47573", "28866", "8167", "2638", "15888", "67212", "10635", "36391", "24452", "41330", "33084", "40345", "22178", "10025", "68011", "52294", "30540", "52246", "52664", "33378", "69486", "46103", "53683", "39491", "21920", "72809", "67364", "39947", "71386", "43367", "58279", "27731", "70477", "3429", "44533", "24208", "20279", "55166", "57194", "28800", "4912", "5699", "56697", "18611", "26238", "6311", "36284", "11340", "30629", "29210", "50913", "66593", "23740", "9284", "64338", "13662", "54650", "8591", "19446", "49154", "64835", "15728", "45953", "39365", "72457", "43684", "10073", "13815", "15300", "50915", "45684", "13110", "23453", "25414", "31449", "70607", "37427", "20615", "52214", "51108", "59383", "38401", "70232", "65563", "15969", "65293", "23609", "24082", "16173", "4037", "66717", "55416", "12657", "4885", "6793", "22122", "45322", "34169", "64848", "54345", "27685", "72299", "62381", "766", "61870", "68657", "3999", "3367", "68886", "40483", "19462", "26427", "72168", "69106", "45942", "12364", "34854", "4022", "31919", "6591", "36240", "52471", "69977", "2023", "57358", "18391", "34453", "39338", "9830", "61033", "71851", "52821", "26481", "25804", "15824", "33879", "52366", "3668", "53945", "19800", "41872", "5622", "22282", "50289", "6550", "1336", "56456", "9749", "39253", "55478", "72196", "47611", "16452", "39000", "39619", "47827", "14013", "36256", "51725", "3714", "18271", "69004", "26338", "6560", "62128", "46699", "34516", "52447", "12670", "36894", "27957", "29484", "4716", "22853", "13675", "65560", "62244", "42726", "64466", "36354", "53629", "42980", "13664", "36162", "39274", "70768", "39894", "33769", "117", "39871", "68224", "23736", "13081", "36156", "70994", "66347", "58736", "795", "12344", "65394", "60412", "56459", "192", "51189", "51442", "66855", "52800", "13653", "39987", "9072", "61043", "4354", "41390", "72529", "67990", "51992", "32646", "61367", "11838", "3347", "19467", "14915", "61473", "60417", "73181", "10192", "36442", "40757", "25329", "18620", "35352", "69267", "39921", "26866", "2361", "36058", "21899", "8242", "54972", "4477", "14191", "34498", "17102", "43881", "40177", "44047", "15330", "5738", "57711", "44715", "63104", "47189", "67636", "6228", "49301", "59417", "48093", "831", "40147", "34831", "4142", "8078", "69503", "42218", "69352", "1483", "14749", "71035", "54316", "33559", "11349", "8762", "22905", "59966", "63005", "35003", "31730", "58002", "43541", "4997", "61176", "9969", "9272", "39771", "19043", "21832", "34938", "8960", "4588", "68146", "13643", "5166", "52798", "69354", "41474", "72647", "69831", "55023", "53885", "68963", "33648", "68555", "12376", "36344", "66787", "62447", "1573", "51832", "11201", "43994", "61127", "48904", "37507", "72635", "7379", "19294", "31228", "26934", "22824", "56061", "55549", "43766", "10136", "65183", "41848", "3370", "70730", "60942", "49443", "57971", "1150", "14161", "10203", "22352", "26151", "62602", "18261", "21054", "45945", "29652", "18764", "3491", "44188", "10239", "44935", "36991", "13420", "61670", "36703", "54728", "647", "597", "49771", "28055", "14370", "57191", "71482", "40933", "54925", "66378", "53956", "59212", "27492", "41843", "58808", "15798", "47823", "31014", "10876", "3786", "30640", "64477", "48597", "17996", "24778", "71327", "43430", "15775", "31903", "46802", "40000", "6113", "16873", "22180", "3131", "51698", "51762", "17907", "49001", "61709", "24020", "42895", "15340", "997", "10169", "58748", "12798", "38268", "39333", "52417", "17009", "3869", "2077", "7963", "49014", "2521", "26796", "7334", "23640", "55284", "47969", "45567", "17883", "17492", "65061", "29349", "45554", "15910", "57737", "13365", "43892", "43305", "67119", "57132", "28971", "38458", "38548", "30624", "39307", "10299", "42584", "30525", "18713", "21662", "33660", "57923", "67559", "11179", "50885", "69457", "29598", "48428", "22946", "3106", "16135", "53291", "5059", "47711", "25486", "51267", "19854", "72742", "53541", "6683", "50591", "20506", "59085", "42644", "37141", "41393", "13106", "69393", "72880", "8752", "52696", "61918", "31329", "7987", "285", "65790", "29397", "28810", "45226", "10948", "27853", "28984", "38001", "23392", "23659", "18006", "60269", "19428", "29910", "69041", "46128", "67363", "8574", "35249", "63562", "32195", "54494", "2112", "24237", "55611", "20557", "58508", "24207", "67401", "65477", "9129", "59876", "20495", "23508", "68351", "71464", "16441", "49063", "27085", "55716", "31974", "10842", "43451", "44015", "19246", "59885", "56719", "47949", "19416", "50609", "1508", "67661", "51897", "34658", "6692", "47851", "71440", "70286", "69904", "67897", "72273", "45353", "42254", "29925", "49238", "46717", "22525", "11883", "16219", "52036", "39250", "11748", "13689", "24705", "494", "23866", "37260", "37104", "57841", "30576", "27350", "2687", "38107", "65204", "40472", "43061", "61806", "59096", "63503", "27259", "57267", "49169", "46022", "33921", "35987", "57478", "72091", "12105", "42159", "19417", "34709", "13238", "2291", "36678", "65372", "26932", "5850", "41437", "17404", "21771", "16756", "59549", "39806", "24080", "15821", "45470", "81", "33740", "14027", "44189", "28830", "9438", "16930", "62135", "25614", "11807", "64015", "14215", "37779", "60681", "57703", "56678", "8341", "69797", "47819", "42457", "55683", "44890", "68865", "33774", "22318", "60651", "38162", "16268", "56530", "24366", "28781", "46047", "40059", "32980", "34228", "6336", "52627", "62448", "3188", "16869", "65975", "48831", "24172", "4807", "31926", "37827", "26886", "40337", "55457", "29150", "35288", "45975", "35715", "70436", "66389", "34635", "51450", "26589", "72243", "51601", "14536", "17524", "13948", "69336", "46275", "67182", "3387", "19752", "68561", "26110", "63843", "53900", "45603", "1460", "1786", "67724", "31567", "68546", "14294", "60389", "34305", "51864", "59852", "36320", "34876", "11674", "59926", "71199", "63357", "9782", "44757", "31722", "59740", "19717", "1967", "42713", "39435", "24721", "61117", "64358", "21439", "48571", "36722", "44982", "62592", "49286", "20493", "16034", "61404", "9659", "62994", "10698", "47786", "62719", "7340", "33470", "53089", "21095", "61968", "68071", "49982", "64423", "23593", "68429", "30893", "26776", "16648", "51878", "43944", "23739", "6673", "48608", "64092", "44148", "39220", "58930", "18172", "56838", "51788", "24361", "42797", "19532", "44680", "61005", "31098", "38", "72279", "11390", "45419", "835", "69665", "45655", "5277", "26650", "44247", "32341", "47982", "426", "43719", "50656", "10452", "22183", "37548", "35679", "38804", "27016", "39147", "29893", "26457", "45500", "5703", "72448", "17956", "70483", "26106", "32968", "38131", "16329", "7534", "35546", "54235", "15694", "30072", "67410", "52957", "40250", "12345", "22697", "1006", "64853", "35467", "39197", "67280", "55126", "70306", "40925", "42488", "33337", "14712", "17531", "6890", "49629", "73070", "31376", "30617", "52770", "57836", "71056", "38604", "6844", "8514", "56785", "69625", "37331", "23072", "71349", "18445", "71496", "54149", "47207", "68188", "17875", "67883", "33871", "9809", "24493", "41895", "6439", "15646", "6927", "41130", "67452", "49955", "70728", "30493", "10827", "3512", "30401", "70374", "11084", "26288", "26788", "9437", "51838", "3703", "6418", "89", "17010", "38328", "29804", "2937", "44869", "30286", "68082", "40287", "69910", "12159", "69199", "68022", "68880", "62951", "10162", "30222", "35530", "7724", "477", "24472", "28947", "25851", "67627", "48221", "22936", "57438", "19191", "58026", "64535", "72794", "60290", "57090", "9800", "65242", "44187", "20132", "71665", "28941", "14514", "8050", "34373", "61387", "63309", "17695", "23887", "18183", "14475", "57751", "36798", "66763", "41087", "58936", "24571", "60706", "52426", "35567", "20515", "17804", "35313", "65606", "45391", "73170", "41154", "32650", "6767", "20728", "57780", "65924", "71628", "28558", "50409", "52814", "67034", "60919", "54346", "40405", "65315", "58500", "67792", "30404", "21015", "23767", "11688", "1913", "50942", "54268", "4405", "3322", "49880", "7969", "30231", "1219", "70158", "50308", "15509", "42094", "54280", "68919", "59353", "68092", "13236", "51809", "36543", "21098", "39812", "55998", "499", "20038", "68039", "57033", "36469", "18990", "24861", "14076", "47562", "66780", "65325", "59428", "25232", "65124", "59948", "55789", "18432", "56809", "63399", "41717", "31753", "64086", "29592", "534", "27078", "1771", "6319", "20342", "5870", "68115", "17639", "20922", "60929", "62088", "70227", "51659", "8592", "40920", "10562", "17908", "44728", "4825", "9063", "50021", "10394", "14647", "46615", "57701", "43", "68264", "1447", "56883", "36077", "10490", "34649", "32810", "62594", "15423", "48448", "11935", "69211", "38003", "67072", "42329", "30532", "47516", "2301", "47067", "66674", "13345", "40354", "12395", "17474", "48502", "1328", "8458", "27084", "41329", "46955", "15496", "68322", "44918", "51989", "18102", "27987", "51688", "72849", "37913", "17966", "35647", "13892", "31705", "61989", "54179", "46386", "55375", "39931", "54380", "63199", "72755", "22158", "2169", "33272", "71875", "52842", "52709", "69610", "5548", "62803", "71044", "11820", "44037", "18726", "38242", "36513", "31383", "33283", "65369", "4692", "8719", "30344", "648", "4583", "14035", "63535", "57686", "53450", "39762", "18298", "33545", "63828", "69200", "19563", "52276", "59344", "37979", "18217", "39701", "39180", "61952", "64894", "19614", "69714", "67484", "50964", "33689", "59941", "38210", "49161", "34872", "32824", "47653", "9276", "8321", "30151", "65821", "71133", "67022", "15304", "16382", "16655", "60281", "21398", "41764", "19593", "10206", "22411", "43553", "666", "14943", "51572", "13701", "42386", "69480", "58543", "19439", "40263", "52290", "46662", "66090", "53524", "37213", "62361", "40538", "32876", "53883", "61571", "28907", "6874", "28722", "29594", "18238", "423", "47061", "3455", "9242", "31652", "50246", "11136", "39574", "19318", "5900", "41811", "11348", "41644", "65746", "43408", "44467", "15369", "65617", "64433", "48798", "56970", "64275", "16342", "43258", "43925", "45538", "36844", "42598", "49141", "31670", "51237", "54080", "47470", "2814", "41361", "24418", "2029", "10258", "11532", "63026", "34855", "14112", "53435", "28908", "49995", "36273", "63126", "57228", "19969", "8385", "67000", "5014", "21782", "9059", "31003", "21113", "62646", "3712", "15250", "46467", "48408", "59197", "42196", "10028", "62000", "50767", "29119", "61082", "42156", "35054", "51956", "20255", "38792", "25571", "12791", "65004", "6548", "53354", "11854", "9218", "48576", "17714", "28470", "14086", "4266", "30350", "17991", "55262", "65178", "24586", "18682", "43309", "61849", "39054", "53805", "28917", "35185", "68016", "610", "41451", "14394", "44850", "54974", "62227", "47119", "22778", "16279", "58268", "4114", "28422", "8055", "67398", "42634", "6490", "66570", "69725", "57555", "12415", "40689", "3325", "40260", "54331", "47106", "9631", "65138", "51795", "14376", "60491", "51808", "10089", "45482", "55279", "39421", "65463", "54006", "56862", "1227", "43002", "41170", "67579", "40467", "20015", "59550", "21509", "20163", "18314", "12679", "63489", "30054", "66072", "72108", "47771", "46387", "61079", "23066", "67168", "59028", "9293", "70651", "59210", "58197", "22842", "32575", "48732", "6642", "50970", "42914", "39914", "48719", "49795", "32588", "44517", "54640", "11987", "20464", "19997", "12705", "27346", "46201", "36527", "37617", "12872", "65590", "60898", "8217", "8926", "64070", "20120", "15928", "54543", "33259", "26217", "47100", "60394", "56263", "57560", "67797", "38901", "25003", "41999", "4395", "16554", "65622", "9374", "25865", "69166", "5133", "6586", "20727", "30232", "32975", "73006", "4079", "42716", "15647", "40873", "41550", "52195", "13131", "12378", "24375", "52576", "28698", "69734", "42100", "44867", "10214", "66071", "7854", "52063", "73024", "71096", "43567", "60087", "10232", "21963", "45316", "55322", "48117", "6845", "3865", "72138", "32887", "21070", "15504", "51923", "44788", "18496", "72721", "2974", "20531", "16823", "54146", "23141", "31445", "10173", "62312", "49128", "65696", "31462", "35519", "48297", "22610", "15589", "863", "50665", "35348", "55267", "53697", "2736", "37471", "26450", "9074", "18569", "7441", "18070", "23184", "14350", "56060", "23914", "70945", "1438", "71510", "58335", "43749", "7945", "6747", "33506", "10757", "19939", "31987", "38043", "64468", "57598", "4313", "28146", "21023", "5243", "17487", "4471", "14962", "29676", "28582", "11451", "22366", "43870", "62875", "40342", "12013", "52411", "723", "19003", "70058", "1686", "55932", "14495", "39203", "70310", "3893", "52336", "55202", "6981", "36019", "17890", "28706", "69687", "30996", "8603", "35844", "2212", "4169", "31193", "62813", "51057", "23747", "27397", "18522", "17972", "8433", "46463", "17387", "32392", "48299", "14787", "28965", "11471", "27667", "2601", "43234", "67378", "49764", "69279", "58697", "19138", "16319", "53493", "20869", "27038", "12664", "47499", "19108", "67006", "1600", "70250", "13616", "52736", "44709", "61179", "45284", "54946", "22818", "7664", "70289", "42578", "66895", "48022", "54697", "24068", "64550", "16887", "33172", "67613", "60841", "20541", "33106", "32234", "43696", "69153", "25807", "70599", "21939", "24378", "48445", "59376", "12234", "31204", "46090", "35695", "3238", "48036", "13762", "7495", "69541", "9744", "65664", "43777", "5618", "63453", "3581", "27361", "61550", "67155", "28187", "58913", "59416", "14055", "2929", "52643", "23092", "9053", "173", "68049", "12060", "47286", "4249", "13050", "66153", "51666", "53934", "63832", "9190", "45782", "71364", "11166", "28737", "66428", "68227", "16962", "33779", "497", "33223", "8201", "33197", "34700", "1671", "7652", "27897", "47725", "18268", "38477", "10439", "53840", "45989", "43452", "13742", "50813", "36440", "15196", "4589", "33219", "67319", "57099", "2488", "68312", "66464", "10515", "39209", "7138", "42175", "22270", "50331", "19278", "73213", "34870", "47240", "10369", "1485", "5224", "37861", "62483", "7111", "30315", "58236", "40269", "23265", "1314", "63889", "12387", "1437", "13940", "28764", "39093", "70534", "22245", "48126", "4", "20370", "15177", "62066", "28653", "6797", "72003", "18081", "34042", "48819", "67483", "18114", "32948", "70943", "27223", "66435", "71887", "28752", "28256", "47946", "45741", "68157", "46091", "7477", "14906", "62149", "3736", "47508", "49340", "10205", "27930", "21576", "37340", "3434", "43034", "43744", "31028", "46540", "71735", "12110", "66155", "30929", "32041", "34095", "50317", "52688", "73180", "62696", "57914", "34961", "59732", "48055", "73080", "64166", "51270", "62752", "50631", "11561", "25466", "63612", "2754", "28020", "34370", "43954", "46563", "23430", "38910", "66516", "58067", "22455", "32350", "69765", "35638", "67525", "26538", "30712", "3343", "14122", "43197", "8654", "67756", "28388", "7654", "25089", "39602", "31216", "16642", "41511", "53687", "29796", "45467", "65245", "14780", "69340", "71139", "25042", "13549", "29024", "34378", "14615", "38828", "24241", "42397", "57435", "51977", "42465", "13380", "23703", "71324", "44443", "36395", "21047", "67298", "16669", "6665", "68576", "43698", "53728", "64352", "25291", "54785", "36383", "64522", "42931", "23015", "4790", "55342", "68220", "21481", "2414", "12313", "25748", "48960", "25339", "72214", "72302", "58889", "39247", "49538", "14902", "33893", "7174", "71963", "60560", "14568", "60108", "40176", "41046", "15581", "58416", "26080", "2410", "64473", "5697", "10801", "28705", "49291", "50172", "47787", "7384", "20331", "57047", "49715", "62320", "18378", "48344", "53843", "19722", "21607", "73004", "69468", "1707", "29681", "69889", "20735", "68141", "12165", "38008", "54306", "64850", "68949", "7357", "9171", "49758", "44961", "69382", "7892", "61733", "30984", "8604", "32589", "36153", "63200", "36663", "11418", "35875", "52315", "24535", "37715", "17532", "56378", "42028", "54991", "50466", "58591", "4575", "41168", "8021", "46635", "7843", "7667", "39820", "4508", "10540", "67313", "893", "49516", "6446", "45263", "25923", "156", "66430", "32145", "8397", "45524", "38125", "24822", "44082", "35558", "13822", "66145", "11361", "43153", "59651", "55896", "22346", "33428", "41918", "56117", "67300", "70102", "45540", "56757", "20275", "5205", "70485", "29839", "43732", "26060", "4580", "15617", "46655", "33277", "44348", "47072", "13208", "1091", "25279", "66317", "4732", "27254", "21066", "25034", "48067", "7626", "29144", "25513", "43784", "2337", "55247", "26007", "50451", "44782", "35829", "47122", "62905", "14826", "8528", "51557", "19034", "10182", "5598", "71678", "43232", "65980", "62665", "3353", "70280", "26754", "34767", "21687", "3356", "34079", "50457", "52581", "11894", "64750", "50328", "43325", "22809", "12404", "62596", "34499", "43752", "21170", "33644", "21148", "32915", "12861", "30290", "27687", "25085", "25673", "13613", "1409", "52299", "70852", "61088", "13297", "19300", "34585", "2578", "495", "14738", "27289", "47222", "16067", "35002", "1888", "37723", "40056", "68764", "31149", "48175", "68046", "73188", "53039", "37106", "62390", "70372", "19866", "44975", "35126", "3342", "8184", "1182", "62912", "65514", "29200", "60536", "68030", "12109", "72044", "6614", "47692", "42283", "65039", "59637", "26031", "66808", "54298", "4495", "49871", "54067", "72938", "39580", "4699", "11059", "63305", "6880", "54797", "6536", "23951", "50818", "28072", "26260", "44999", "2084", "45402", "19994", "44738", "64692", "49546", "24854", "27387", "53224", "320", "36014", "54366", "3858", "71904", "68337", "52573", "49890", "56109", "31211", "26639", "72430", "35222", "17995", "70729", "49475", "57484", "33157", "18879", "6800", "28770", "52335", "32545", "55180", "39809", "15075", "17984", "3910", "33023", "47388", "39885", "11668", "27699", "33371", "35106", "58910", "70763", "22355", "12782", "71262", "56819", "31392", "50864", "32723", "30558", "69965", "26970", "12972", "14896", "62048", "7219", "28877", "69921", "65776", "42325", "41040", "22741", "68147", "19796", "20711", "2662", "70320", "38618", "48641", "41951", "55011", "17443", "28848", "22392", "53953", "27770", "49627", "47179", "3169", "56013", "45266", "39058", "40315", "17794", "71825", "2199", "5871", "5460", "63139", "6578", "63568", "46105", "72508", "11135", "28177", "6337", "3894", "67115", "27037", "38409", "64610", "11844", "40190", "14461", "64658", "29552", "47243", "8079", "48887", "17666", "46368", "405", "71168", "12719", "72702", "33767", "53519", "2354", "38147", "12973", "70277", "56715", "29726", "36004", "7704", "25358", "62323", "45856", "43701", "38241", "71641", "12418", "13002", "70721", "569", "49986", "70593", "71664", "34011", "20568", "5191", "24227", "10603", "3173", "57545", "32199", "42916", "25538", "64217", "28275", "27228", "62263", "20900", "50281", "30779", "14983", "54636", "45073", "49679", "19908", "67612", "45330", "10150", "70536", "20554", "3622", "40346", "23296", "55962", "60769", "70668", "25906", "44706", "20589", "65244", "25961", "71819", "63974", "8888", "70298", "19016", "27806", "65233", "48881", "26625", "44993", "39882", "15270", "7479", "42086", "53547", "68543", "59771", "6488", "11354", "26122", "64630", "49614", "26548", "66809", "317", "36569", "11699", "20527", "26878", "54562", "53125", "19412", "24603", "52739", "71304", "3103", "3064", "70332", "1666", "52327", "10404", "55456", "14946", "67108", "71559", "57215", "48264", "56552", "32717", "15742", "29718", "27826", "6486", "57688", "19926", "50675", "46820", "42563", "21167", "5769", "57520", "8989", "13519", "58631", "6985", "26675", "18395", "13632", "10462", "23067", "6952", "1312", "5020", "2968", "72303", "25313", "21825", "40226", "19204", "72072", "2433", "6180", "10551", "18534", "22860", "19353", "65983", "56123", "43091", "20356", "18382", "67434", "24698", "44476", "36198", "41196", "14385", "55824", "35860", "38603", "33651", "36265", "23480", "49770", "2384", "47005", "37867", "11384", "52751", "6464", "52585", "48350", "44967", "58014", "34372", "22775", "64935", "62060", "69256", "15092", "8318", "6414", "62281", "65902", "21546", "28623", "68357", "56297", "22687", "24661", "7142", "41257", "8611", "20739", "3043", "56500", "61826", "16383", "9068", "9037", "55220", "53261", "17111", "64626", "45476", "28561", "14881", "30068", "68113", "44978", "42542", "38936", "37489", "30193", "45298", "50436", "22201", "9688", "68769", "60276", "8062", "16606", "37547", "2317", "51586", "69896", "21432", "43125", "1225", "42333", "50088", "66315", "36667", "31963", "16874", "55674", "65944", "43157", "33939", "30196", "22967", "48666", "43423", "36914", "63187", "12330", "59275", "12634", "52841", "55879", "42573", "66619", "8325", "13150", "60030", "65557", "5158", "4235", "53309", "51748", "46775", "43456", "19812", "19023", "65647", "6563", "29950", "70101", "54139", "55914", "71592", "68025", "59137", "35675", "40919", "69876", "20168", "25547", "20065", "2856", "67250", "37324", "50996", "9553", "37428", "36255", "53500", "53896", "4828", "43816", "63361", "40562", "64589", "7399", "22191", "5617", "27768", "56576", "55719", "68814", "46433", "47700", "36833", "4155", "19814", "72596", "35694", "69034", "15280", "69031", "63265", "55678", "58394", "12066", "55560", "32787", "3745", "11577", "15108", "21996", "26369", "9901", "41398", "22684", "18118", "46497", "16864", "55923", "46860", "43605", "4865", "73050", "10123", "13698", "62999", "24621", "52914", "11408", "62231", "67550", "67412", "6482", "56553", "50523", "34210", "23264", "68008", "46145", "27868", "1565", "17826", "39299", "36494", "19118", "26973", "10790", "31578", "35741", "19301", "61437", "49173", "21248", "42664", "17631", "66579", "31052", "68840", "62553", "67342", "39159", "14436", "13426", "59900", "7884", "15114", "24253", "55880", "40433", "47316", "56229", "4143", "53611", "17338", "40441", "22565", "10881", "28396", "37691", "41916", "15051", "40626", "13766", "35091", "59968", "64596", "67840", "11950", "19037", "61922", "62664", "57532", "37727", "28411", "4720", "67657", "51265", "44707", "27540", "13950", "29186", "53858", "23270", "72157", "18283", "63602", "26630", "3021", "28013", "60234", "1765", "16360", "3300", "52562", "61825", "44008", "48788", "25578", "15392", "16110", "61170", "17679", "48459", "12416", "4887", "28222", "22856", "40620", "35961", "71485", "52021", "50175", "50493", "6284", "36214", "67238", "10774", "20917", "56112", "58684", "52130", "13741", "34276", "20943", "7894", "56638", "19841", "45639", "61428", "39704", "42487", "47011", "4105", "73107", "45711", "34860", "10999", "63442", "26805", "40706", "51875", "29046", "63975", "39846", "71689", "72239", "61930", "24262", "65091", "38747", "71144", "9366", "39826", "61035", "54941", "66085", "8122", "50204", "19006", "68074", "7730", "61102", "6596", "23784", "5921", "68823", "69514", "34631", "63010", "29759", "48334", "30505", "41284", "25090", "16893", "40450", "45522", "46922", "58492", "2824", "69642", "55808", "38892", "55418", "23771", "64625", "63370", "59988", "33255", "14262", "30223", "58050", "28069", "69559", "3303", "26546", "16004", "15739", "58189", "68336", "11955", "28230", "31579", "26656", "36932", "23567", "35630", "59471", "14474", "70866", "59263", "70821", "10286", "52282", "64308", "12228", "26248", "54230", "14125", "1288", "34313", "18413", "50753", "21584", "35024", "12247", "22255", "35105", "67134", "31481", "50247", "48008", "69736", "21355", "18538", "44084", "61999", "69820", "68461", "20740", "6174", "16584", "68481", "23511", "986", "69634", "34331", "15731", "12042", "58895", "61247", "46115", "63358", "61327", "47919", "20564", "34504", "7882", "18871", "45516", "15491", "40043", "49790", "57465", "38796", "2197", "61020", "42932", "25040", "71286", "31319", "27753", "48791", "37938", "31946", "34201", "9481", "36893", "52113", "63084", "49949", "60239", "60956", "32074", "55567", "16833", "19508", "50267", "31527", "64519", "68122", "53479", "49343", "72485", "18307", "5669", "54589", "61184", "51962", "18225", "26408", "65879", "23454", "64153", "677", "30125", "49429", "68799", "13052", "56671", "44350", "12904", "16528", "57889", "2203", "1627", "24371", "59939", "32172", "185", "47525", "3797", "44317", "21122", "15162", "61532", "55952", "43394", "46827", "26890", "71635", "70980", "61027", "53236", "67887", "30679", "43740", "22179", "51317", "35389", "72858", "68611", "70212", "105", "8155", "38789", "69500", "33676", "2770", "33826", "34364", "15002", "13543", "65530", "38799", "61781", "17677", "48792", "38099", "56148", "57668", "6291", "45078", "18576", "24297", "26255", "31855", "23240", "64954", "19219", "7278", "59858", "1757", "36810", "17347", "27019", "22545", "49654", "14852", "46880", "55381", "64393", "46220", "51446", "28002", "29879", "48391", "31308", "59680", "60782", "30928", "30661", "34662", "4390", "27823", "6191", "8553", "24446", "29206", "9527", "61008", "42041", "42059", "54867", "16790", "38943", "44619", "29137", "19337", "15620", "3496", "44559", "2101", "30139", "57112", "30296", "18888", "40070", "67076", "24693", "18883", "12360", "28330", "38602", "17716", "4649", "72955", "22591", "20307", "71847", "22596", "10364", "16481", "70765", "8627", "47275", "14713", "5320", "38256", "18002", "36627", "18944", "2515", "10276", "67199", "10288", "67433", "58070", "49055", "54879", "55557", "31436", "49084", "8331", "19889", "39830", "32831", "24529", "6458", "22386", "53390", "60552", "42650", "42663", "7784", "7185", "11061", "7504", "24541", "58185", "72577", "16863", "37072", "59812", "61627", "14909", "61173", "15784", "51422", "41260", "71808", "27614", "8179", "45388", "64479", "55993", "14992", "312", "57217", "21272", "72262", "71971", "59790", "46020", "57064", "15314", "58174", "65143", "5978", "38420", "69769", "62782", "2073", "8474", "63502", "21922", "20690", "4672", "64776", "24730", "32071", "43346", "29470", "48738", "4014", "57351", "8376", "13025", "60287", "54754", "46650", "70333", "37273", "34925", "9369", "15098", "71959", "39621", "40568", "7560", "23144", "65882", "50829", "52410", "27063", "10417", "32748", "36926", "189", "26164", "44038", "25518", "60193", "46092", "23425", "53578", "59765", "42639", "48185", "37923", "43981", "40261", "32871", "60229", "52218", "68409", "2650", "64418", "2640", "5430", "45952", "46919", "23166", "16177", "60227", "25206", "47593", "67087", "35229", "41269", "26681", "42023", "48671", "15173", "17228", "5292", "44701", "44936", "12520", "8630", "66424", "50691", "27414", "8427", "52328", "3663", "71160", "47320", "32983", "18093", "40533", "22482", "39451", "64003", "17739", "58809", "33412", "23513", "35392", "14802", "50789", "42602", "71209", "9231", "54109", "3599", "42147", "56125", "42201", "8348", "31443", "38286", "63122", "55644", "17671", "5729", "65462", "53330", "13533", "7490", "53407", "43558", "70912", "24192", "43926", "26486", "44281", "30277", "54704", "2673", "59831", "30171", "3933", "28266", "11097", "61935", "38441", "39440", "62678", "52180", "72722", "56575", "71941", "23366", "48921", "27333", "6290", "59665", "12038", "43340", "43303", "59865", "10334", "37101", "36338", "41060", "29446", "2140", "55473", "63645", "47513", "72378", "62268", "6347", "48790", "11162", "56406", "62082", "28189", "38136", "8819", "38608", "49968", "64205", "14160", "41634", "31042", "62284", "30883", "68520", "1186", "20105", "67293", "30354", "62023", "30592", "20538", "63092", "51330", "30641", "1292", "70151", "8231", "49692", "11091", "24575", "24883", "49168", "26999", "64933", "38438", "11600", "30939", "72670", "30042", "52907", "37373", "31455", "8946", "46330", "37883", "21302", "9017", "44747", "7729", "7173", "11035", "33709", "44981", "11252", "5067", "28214", "9462", "59726", "17981", "73036", "47", "34146", "19574", "37161", "28120", "10002", "34156", "67971", "63699", "46023", "23006", "54908", "37771", "47724", "21588", "22656", "57155", "65645", "47464", "37484", "70603", "29945", "68105", "26502", "36559", "17697", "7442", "30977", "72825", "19783", "43662", "65897", "6623", "20738", "56573", "67431", "10833", "43768", "6711", "55396", "49367", "19358", "68680", "71557", "60454", "15064", "56159", "57325", "49900", "8495", "61735", "70258", "28353", "18265", "60293", "59381", "22295", "2670", "12630", "67889", "20741", "40851", "42157", "23239", "44811", "66124", "59622", "71659", "44929", "17691", "20942", "30642", "72576", "16128", "19656", "4324", "17503", "35877", "10582", "19156", "71442", "44971", "38655", "12592", "12117", "49510", "14914", "16843", "61655", "24624", "15732", "20695", "51295", "18216", "63625", "59678", "53857", "65652", "21262", "58012", "25520", "10533", "54934", "33057", "63242", "19700", "58153", "48157", "63544", "33366", "11369", "37094", "61726", "52755", "52071", "53296", "20419", "67446", "2964", "20970", "6249", "19817", "3452", "64384", "37182", "57851", "41618", "16327", "22709", "47531", "24746", "5440", "32996", "18059", "52690", "32777", "67218", "67089", "62532", "46524", "25849", "18045", "30491", "3711", "65147", "51707", "49630", "44641", "20270", "19440", "7900", "44752", "63886", "69789", "63306", "68810", "20856", "69786", "29007", "45163", "27982", "41682", "18711", "49458", "41660", "15329", "8835", "60987", "41341", "23835", "5623", "1206", "25701", "64321", "56464", "3998", "22303", "35029", "37488", "29121", "32733", "46366", "53420", "55198", "34834", "8964", "19877", "24278", "32097", "55805", "47370", "10179", "34077", "24220", "26842", "18174", "19386", "14829", "70665", "70782", "4025", "55238", "2735", "66421", "42287", "61601", "58527", "25734", "23621", "64773", "37406", "9951", "70553", "59583", "45831", "66871", "40202", "15209", "49496", "6123", "17668", "51916", "47049", "60569", "50993", "66030", "28533", "71705", "14065", "36463", "53117", "9551", "4518", "70034", "10308", "25789", "8129", "65779", "32633", "35614", "36613", "13719", "42084", "11557", "59025", "47586", "40119", "17656", "48944", "14547", "38100", "28679", "71145", "64879", "23633", "38816", "1474", "65023", "9477", "52715", "46421", "12219", "17479", "39060", "55542", "21411", "22070", "50756", "62854", "27958", "18286", "22170", "13416", "68390", "50547", "64903", "35082", "25516", "51352", "1968", "59241", "22779", "69528", "20825", "71743", "64440", "42841", "40348", "33655", "5323", "28990", "63482", "33839", "61454", "26976", "24333", "25067", "22238", "3222", "29587", "23457", "64130", "12803", "36612", "24599", "45147", "48450", "27426", "41183", "1645", "30101", "24877", "34165", "28132", "45549", "46013", "65982", "24109", "66938", "32363", "67321", "27179", "40852", "18437", "6520", "37401", "67030", "18776", "17694", "6551", "2833", "49918", "71140", "31537", "69768", "41432", "8895", "5202", "51007", "61629", "32682", "24707", "9487", "55553", "65878", "10838", "13343", "26473", "25287", "24356", "36288", "51370", "5283", "7753", "23869", "9975", "57861", "542", "71466", "18758", "32470", "194", "53590", "38026", "51867", "37022", "63159", "55690", "21406", "65432", "362", "15464", "10323", "31612", "40955", "3385", "12585", "70121", "68435", "16865", "45207", "68060", "37255", "46097", "69958", "62984", "31557", "3970", "15586", "47608", "68064", "41223", "14032", "46356", "6876", "38727", "14837", "33403", "70411", "70317", "24132", "40685", "25217", "23071", "34010", "63393", "6962", "61756", "22306", "53197", "59563", "45878", "55415", "14136", "39302", "49145", "46191", "44510", "4963", "24264", "71362", "1971", "2138", "13331", "39634", "65503", "6790", "63349", "42804", "72053", "24671", "17066", "40402", "38141", "48644", "11546", "69276", "22151", "22160", "58342", "68685", "27054", "7947", "38145", "67203", "28014", "15426", "8541", "6390", "57932", "14263", "65479", "43550", "7643", "52201", "18853", "19137", "30517", "69579", "19233", "45798", "14792", "42189", "25771", "48134", "9322", "15266", "16695", "67794", "63079", "32763", "44585", "57881", "42590", "26300", "4774", "9294", "43400", "1277", "42183", "42892", "2749", "40995", "42179", "20226", "2619", "54311", "6118", "3435", "2625", "53540", "20701", "34727", "22475", "59995", "6833", "33118", "60652", "47162", "17108", "61644", "72357", "67665", "7472", "72390", "35712", "43578", "16976", "72566", "47834", "67167", "35555", "13809", "19434", "7257", "9124", "46112", "24177", "53824", "54470", "45092", "29459", "68326", "41111", "54506", "51320", "34711", "63057", "49114", "63553", "62146", "42592", "48512", "34975", "62056", "46900", "66696", "53381", "26123", "31101", "29924", "2671", "35688", "52347", "51235", "29968", "50437", "71492", "27617", "23261", "68370", "41457", "14182", "18700", "20289", "64385", "48499", "54202", "16366", "5785", "24681", "47476", "70524", "16241", "10023", "49478", "64332", "41989", "32385", "55636", "3199", "16879", "19537", "56859", "32400", "46624", "46964", "38399", "73212", "37029", "45216", "39260", "25910", "13765", "258", "70684", "60078", "60023", "18914", "6654", "67377", "26576", "48712", "10433", "49400", "66617", "43562", "37598", "56296", "23466", "65868", "65164", "34151", "4067", "64823", "422", "58872", "54462", "20658", "6634", "31779", "68665", "24915", "9594", "17113", "52123", "71702", "59809", "5993", "38502", "22520", "49507", "44697", "15435", "56626", "69997", "24722", "40949", "37822", "51785", "55299", "1826", "20440", "11194", "46198", "53818", "33302", "47250", "45706", "6670", "68020", "50008", "10684", "11994", "46674", "60138", "47485", "46970", "18275", "30668", "867", "43424", "6156", "40475", "13231", "32360", "9260", "23096", "48601", "19947", "58450", "34036", "12191", "42251", "46875", "32186", "45506", "4220", "17629", "38654", "18142", "54271", "64411", "37138", "28587", "36883", "69185", "35947", "4955", "40896", "41460", "62341", "58847", "50045", "41372", "26420", "72749", "45480", "46812", "60355", "32994", "46727", "15229", "47986", "24422", "32845", "20791", "21261", "63184", "36778", "12020", "64093", "43299", "40569", "16099", "41084", "71511", "10034", "65871", "27926", "31197", "12738", "24423", "780", "65919", "36933", "36035", "11603", "4628", "36339", "41187", "15215", "48184", "22956", "856", "46689", "29948", "43509", "26426", "14576", "33784", "18664", "65170", "65887", "9212", "32338", "40906", "828", "49466", "49750", "59729", "58553", "49567", "71155", "31627", "9811", "60842", "68863", "48577", "71232", "56289", "65442", "25363", "67906", "63610", "41037", "72764", "58550", "12701", "6194", "27265", "15944", "8183", "58685", "16196", "10858", "11144", "60760", "65553", "743", "28410", "42402", "55403", "21750", "26632", "34047", "53640", "9135", "43246", "55311", "34456", "51459", "21409", "38906", "18902", "25391", "38170", "50013", "8403", "38509", "14221", "37369", "52402", "51102", "30697", "33116", "57014", "20084", "67053", "45607", "41575", "38301", "57492", "10598", "31438", "70120", "48932", "13435", "51149", "25990", "8934", "59882", "41779", "60889", "38775", "64197", "26102", "71528", "42081", "68798", "25914", "1782", "49246", "352", "54009", "41226", "49070", "24972", "13037", "56727", "87", "3426", "40207", "5793", "2031", "19395", "66715", "23710", "30117", "30094", "72673", "18478", "3657", "64287", "17571", "31208", "58329", "49619", "48935", "14220", "19612", "57597", "22590", "38329", "15978", "72644", "20467", "58284", "67456", "66630", "41994", "6766", "43791", "2501", "9752", "66526", "29640", "3009", "63863", "10050", "35737", "65215", "70364", "44569", "42613", "28514", "67040", "65623", "69115", "16891", "32936", "61432", "24041", "39487", "9062", "19962", "58028", "60236", "67140", "46949", "1413", "5125", "27952", "64875", "25665", "54829", "44051", "13326", "42292", "40186", "27118", "39142", "67540", "20954", "65607", "60296", "24713", "31419", "27021", "29497", "64238", "51249", "11997", "63238", "65047", "20164", "58307", "41668", "72783", "11594", "37284", "31440", "8329", "48238", "31667", "11968", "40576", "50341", "64938", "47549", "27499", "27947", "38277", "34947", "51082", "27225", "47709", "28597", "1908", "21885", "26468", "12759", "60753", "10382", "27199", "5011", "20901", "23939", "43630", "56603", "60870", "33696", "16945", "27971", "12142", "36606", "7097", "20209", "67952", "66205", "12795", "30741", "19422", "59819", "33034", "63073", "15860", "61131", "14169", "39076", "27542", "11567", "31279", "69708", "24674", "11188", "5357", "13354", "68703", "46152", "2236", "18452", "10547", "29123", "53414", "20136", "1345", "32746", "70679", "44439", "71681", "49072", "131", "187", "51254", "3546", "71325", "36916", "32610", "7889", "27659", "65500", "5281", "9237", "56437", "23813", "8443", "52396", "54584", "34810", "5500", "33584", "32741", "67595", "37644", "70179", "30149", "38287", "24186", "39741", "71867", "63673", "15979", "64323", "9671", "66634", "977", "31603", "19255", "51497", "32262", "69975", "43107", "13256", "12467", "63147", "55341", "68318", "51273", "2622", "20668", "24940", "35113", "63982", "24531", "54832", "45295", "32055", "50138", "32852", "35579", "1592", "14284", "42477", "9939", "21338", "16645", "27865", "62160", "32986", "52903", "56085", "56453", "27794", "20438", "63280", "71256", "36066", "16757", "10556", "7880", "69993", "58086", "50634", "55175", "6783", "58325", "60869", "57056", "19057", "16817", "65297", "5722", "63170", "43349", "73098", "12299", "50100", "42730", "7217", "19595", "10754", "37694", "62159", "29777", "49623", "30003", "37957", "7011", "16961", "9576", "47244", "13500", "36443", "38829", "47022", "4605", "28399", "20714", "25130", "66319", "57898", "4427", "33009", "62087", "52404", "7228", "6899", "3423", "24825", "42160", "389", "54920", "53809", "58929", "55358", "8302", "32665", "6726", "28430", "27017", "8697", "47324", "68855", "68640", "46638", "24376", "27460", "11867", "24607", "2269", "68418", "32104", "65989", "29725", "11492", "46816", "23776", "44545", "47935", "58510", "27445", "69710", "64993", "12413", "54234", "25737", "8501", "31846", "63433", "53713", "1654", "10837", "50505", "4290", "22272", "54877", "23160", "44881", "59187", "63748", "58263", "30907", "59368", "44743", "23768", "61210", "25537", "40936", "73115", "30217", "48579", "38675", "44638", "30663", "57862", "63275", "9660", "65533", "36690", "18685", "32882", "62776", "69324", "59843", "68457", "47279", "43249", "30763", "25046", "58145", "57573", "35391", "1770", "53623", "13123", "42318", "11028", "61809", "43588", "31585", "51744", "56873", "62401", "42799", "23420", "2769", "7098", "52526", "15024", "69125", "23592", "52516", "62711", "67763", "11560", "11370", "2465", "8679", "24964", "53427", "34667", "52453", "15695", "58103", "39225", "51492", "13987", "58349", "68711", "8328", "26940", "26497", "51167", "11766", "70595", "10087", "23436", "70720", "35897", "49027", "9914", "9565", "53349", "52306", "7684", "61223", "31615", "52241", "66283", "47307", "26841", "32102", "23417", "66239", "67295", "38450", "42469", "29972", "66896", "10689", "13524", "65180", "54873", "17652", "66249", "5364", "32425", "20736", "55428", "71503", "18521", "4493", "66134", "27738", "10652", "71660", "29637", "54907", "31357", "18841", "38557", "64294", "22982", "44541", "31126", "61777", "33723", "1938", "72935", "61695", "47920", "43417", "66567", "56578", "66524", "53536", "67316", "64570", "29176", "61544", "5168", "23707", "60207", "58245", "12937", "47358", "36970", "4903", "38238", "65403", "44456", "2310", "275", "42948", "54", "25373", "46586", "64175", "62506", "15195", "39761", "6821", "57843", "8859", "4564", "64050", "70749", "17873", "56063", "42975", "50643", "49250", "41066", "65877", "22325", "42419", "26566", "62580", "53283", "4318", "25813", "31750", "28741", "28268", "58075", "693", "5081", "1330", "3104", "3028", "3407", "3804", "41443", "26425", "65698", "9614", "40897", "37238", "609", "60314", "56905", "14277", "12449", "30297", "17224", "40856", "37239", "62886", "4422", "61720", "61509", "40156", "61560", "57378", "42103", "41480", "22025", "14665", "25772", "53920", "51511", "38563", "15028", "68495", "3072", "25294", "71052", "9882", "19486", "29582", "21817", "24821", "51264", "20344", "44601", "12977", "27949", "71195", "28107", "68378", "44209", "50299", "45661", "11228", "50103", "42017", "41536", "10636", "68653", "69395", "43096", "43700", "33857", "72314", "35170", "28518", "35900", "70278", "36654", "46660", "24744", "1439", "907", "63115", "39536", "805", "61920", "47128", "3813", "37343", "49224", "48119", "60523", "5410", "33597", "50969", "23164", "4714", "65970", "62961", "23123", "33880", "58495", "50779", "25503", "69274", "67825", "63658", "71439", "63847", "29298", "62133", "64706", "17518", "56985", "29522", "6224", "2783", "61914", "54926", "4657", "34869", "18492", "44785", "44461", "12106", "9497", "18566", "56115", "63032", "39791", "6463", "19205", "47924", "8072", "39843", "67732", "16373", "22433", "69218", "11657", "2826", "32333", "30146", "45781", "4527", "22464", "24007", "64939", "67869", "22219", "63369", "24289", "35260", "37349", "70938", "36685", "9606", "43109", "31716", "54757", "22139", "21809", "18728", "7719", "72350", "5139", "67956", "69338", "2527", "17350", "30050", "55866", "69719", "63050", "13101", "6699", "67719", "32987", "24107", "37089", "40785", "27577", "48341", "39589", "30499", "24532", "49424", "39248", "12772", "28176", "19557", "13943", "5170", "9689", "11822", "71630", "45024", "66719", "1985", "31778", "12292", "54508", "68109", "26333", "51503", "65416", "47990", "60981", "8488", "2016", "30267", "1687", "20149", "43379", "33114", "57011", "62492", "26908", "3245", "5677", "29852", "11663", "49543", "46004", "25981", "7150", "5116", "62279", "23618", "33666", "51215", "51103", "64293", "14296", "12763", "34712", "67048", "24129", "68258", "45568", "51316", "22769", "55493", "56073", "31310", "25892", "16619", "55709", "51019", "23176", "46102", "26221", "9718", "15176", "25006", "72026", "33314", "12656", "33901", "34533", "51722", "380", "3816", "67408", "8180", "43724", "50715", "72829", "25420", "12230", "37944", "62582", "3716", "39715", "40630", "21741", "30927", "17323", "22410", "52407", "35861", "43876", "17559", "15379", "17217", "46870", "53256", "48621", "68759", "2519", "47186", "61545", "46737", "39585", "59242", "42409", "14466", "29834", "32644", "33894", "9513", "4026", "66364", "39560", "45898", "67309", "55065", "39833", "64247", "24849", "34256", "46685", "14030", "11915", "65380", "1817", "40672", "51343", "41221", "5862", "53688", "22611", "18839", "58909", "54324", "63536", "29088", "23536", "50089", "38527", "11167", "31206", "19025", "41933", "36817", "67093", "42565", "48971", "13375", "60789", "6618", "26698", "63356", "64195", "13355", "31049", "15130", "61463", "11888", "63671", "50886", "61891", "30353", "48043", "39906", "71690", "24873", "9391", "17670", "27103", "43313", "21901", "63649", "49825", "12668", "32892", "945", "56944", "65283", "6994", "28053", "2738", "59039", "31851", "52436", "37102", "36081", "9542", "16167", "4280", "38735", "64699", "31886", "22926", "27879", "64276", "64476", "26406", "68744", "48086", "5344", "50489", "67722", "2758", "46788", "30550", "54211", "26099", "52968", "3023", "64940", "33493", "52582", "11639", "60729", "6389", "14331", "68151", "19755", "67127", "48656", "45523", "20052", "48713", "67912", "69653", "701", "65977", "60667", "230", "29645", "17832", "17742", "25635", "15841", "35022", "43068", "56389", "10040", "9649", "11810", "61945", "18577", "12770", "18533", "52969", "65528", "29719", "39293", "52268", "62572", "5512", "50808", "12056", "52676", "52081", "16666", "29173", "9989", "9870", "249", "4910", "28305", "25322", "28151", "60145", "10722", "27994", "61361", "7514", "31934", "58054", "21931", "22701", "16140", "61344", "30931", "22751", "64392", "42467", "60095", "58296", "27867", "40291", "32863", "23919", "64757", "63460", "34016", "21940", "57580", "24901", "15479", "9679", "41420", "4101", "43017", "67551", "30257", "30818", "27338", "49921", "12512", "71634", "64634", "57380", "68907", "55696", "4985", "59752", "54315", "25573", "62650", "37025", "59611", "72893", "35622", "3764", "3376", "37162", "67769", "5091", "45103", "71287", "46163", "29377", "52220", "61722", "4513", "15678", "19526", "29025", "19851", "23485", "541", "6315", "15083", "69504", "160", "45123", "15666", "15574", "15299", "55039", "22574", "46227", "71636", "22269", "65221", "28880", "36426", "17905", "67773", "45692", "24899", "46226", "72942", "15344", "8790", "47702", "46504", "24728", "21320", "47962", "31546", "987", "68328", "54151", "64558", "46001", "41599", "3400", "50561", "46995", "67400", "29771", "30080", "43215", "70877", "31719", "40557", "935", "57691", "36106", "66012", "17791", "2431", "20802", "69149", "5534", "57777", "35394", "58520", "44791", "54737", "23329", "48059", "27073", "1081", "60649", "50379", "35912", "10137", "24564", "11779", "8404", "34549", "20086", "67105", "21437", "47994", "42003", "12347", "48007", "1922", "67357", "33070", "51078", "3651", "32696", "30662", "71756", "31595", "68373", "46705", "9740", "13079", "73118", "70134", "11341", "22735", "62616", "26810", "11303", "1659", "7998", "32853", "19577", "70123", "6355", "48163", "49085", "35301", "66648", "58291", "36322", "62385", "62944", "46364", "70580", "62846", "10448", "41378", "6826", "6701", "5435", "15225", "68465", "68059", "26924", "5526", "10940", "54117", "59958", "10065", "33940", "53705", "10571", "36490", "34693", "2775", "41890", "11290", "67786", "12500", "70842", "68019", "43609", "71858", "20983", "26884", "24428", "16005", "71757", "44370", "46715", "42962", "22951", "52003", "33088", "46179", "43499", "47260", "38189", "59548", "6582", "67237", "16231", "50149", "56507", "44776", "65376", "12882", "37576", "3286", "21316", "61758", "45110", "54945", "43657", "21856", "60320", "53730", "37224", "72998", "61094", "40868", "65762", "40171", "47175", "24176", "71270", "65855", "40737", "62476", "49439", "53297", "63705", "43120", "2728", "31912", "23388", "50201", "32061", "38482", "57576", "3260", "55037", "33136", "25980", "46992", "16109", "51444", "22477", "25669", "21789", "61996", "25361", "72887", "69090", "59577", "69510", "62262", "58652", "384", "9508", "71683", "43797", "65547", "57672", "8587", "9597", "5582", "10565", "21339", "67858", "31698", "58435", "3328", "33131", "60501", "49942", "63835", "14968", "35666", "69917", "17316", "72011", "24028", "58289", "4472", "9930", "29662", "30665", "39305", "68080", "62795", "28134", "41986", "17959", "8054", "5575", "47743", "53957", "16387", "25378", "28468", "64679", "34651", "54802", "14216", "37465", "9429", "51219", "24788", "58557", "23021", "36732", "52989", "42867", "11443", "33824", "42733", "56481", "57623", "41908", "43989", "50029", "51214", "16954", "60192", "55724", "6847", "67353", "6257", "16845", "17839", "16753", "2399", "44792", "65942", "67625", "61763", "3476", "50097", "70409", "62533", "2911", "31980", "26991", "8001", "65019", "59099", "56244", "27850", "32235", "16489", "14575", "34318", "60221", "32190", "22066", "69386", "72036", "72080", "9963", "38834", "18333", "32951", "3085", "11428", "21010", "70790", "9340", "61901", "63119", "62289", "34074", "21075", "21734", "50828", "17315", "12217", "10768", "7484", "54937", "67893", "65680", "44171", "48402", "26770", "61347", "62067", "56068", "32403", "2385", "2069", "29304", "61333", "37336", "5151", "34796", "42795", "22841", "10059", "57562", "55138", "69476", "12216", "31387", "19906", "67381", "11622", "13522", "62641", "73142", "69295", "49904", "39841", "24096", "4137", "38951", "41808", "24416", "35955", "70098", "38939", "5804", "52167", "17197", "67262", "19321", "18880", "28302", "9574", "58756", "60995", "26623", "24843", "9005", "28549", "25982", "34114", "22736", "63071", "53669", "13631", "1772", "32944", "28618", "40712", "17119", "62477", "9702", "33013", "27558", "18039", "28985", "2126", "11809", "67497", "14714", "28110", "1555", "26954", "64896", "59166", "12781", "28865", "17842", "18519", "45667", "55384", "56482", "56101", "39935", "49852", "23319", "62555", "62531", "6522", "3654", "12678", "43097", "55681", "65358", "50152", "5760", "68748", "69632", "49293", "56805", "46416", "53755", "72906", "48758", "36094", "21745", "20552", "54382", "57919", "16244", "42503", "31830", "42177", "19666", "21937", "9504", "45303", "43206", "30671", "34380", "858", "45837", "41383", "32906", "28465", "11359", "6305", "46843", "57488", "14186", "38555", "38402", "7728", "50032", "2024", "61508", "43683", "58326", "63416", "1425", "61648", "8077", "62728", "47467", "18073", "66529", "67063", "61378", "34227", "60279", "26220", "23810", "35995", "27244", "50900", "34009", "14585", "59380", "62904", "16000", "22045", "66656", "60829", "23531", "58248", "61970", "51008", "42007", "18244", "4960", "40607", "59605", "20220", "2189", "40874", "73238", "26532", "18032", "47797", "13728", "42205", "48389", "56942", "54162", "5028", "72653", "69688", "32069", "60477", "70796", "66297", "65566", "57980", "70872", "5021", "6261", "37151", "56364", "28696", "7906", "26564", "8766", "65182", "10782", "17015", "60378", "6", "68885", "21324", "14491", "11856", "2873", "63043", "64064", "71366", "45580", "49074", "38326", "41851", "27122", "7647", "59742", "16168", "72908", "68730", "65550", "43888", "60450", "53685", "48261", "2593", "13538", "25198", "70963", "46369", "63771", "7776", "62635", "5750", "28195", "72824", "62379", "29213", "23452", "66241", "22546", "10675", "9762", "52836", "64889", "33066", "19742", "40695", "56554", "37649", "64688", "20019", "20916", "30719", "16363", "9795", "3535", "8720", "47832", "19492", "56732", "3734", "32803", "5647", "53712", "60656", "28958", "42244", "9386", "61818", "71610", "34898", "54121", "40604", "35219", "49494", "34454", "39747", "49305", "36548", "59309", "47764", "43123", "23031", "49926", "7429", "40322", "39726", "71558", "49360", "26222", "47460", "56543", "54722", "14425", "45157", "70571", "4445", "47276", "10301", "36652", "70444", "12211", "9375", "54883", "38051", "1496", "33263", "68772", "16406", "15714", "63224", "49651", "137", "17868", "44877", "23970", "5639", "11110", "52487", "22696", "25896", "64859", "17598", "72769", "19375", "69248", "53937", "56853", "20463", "43733", "51202", "4624", "55514", "21309", "34630", "60602", "568", "70719", "43519", "59853", "53095", "31978", "46601", "60615", "34982", "32903", "31819", "17867", "11568", "17805", "41379", "34874", "37668", "18743", "40768", "46505", "23685", "43932", "50893", "7176", "5707", "9431", "32274", "62085", "57732", "35658", "34755", "33385", "24814", "71999", "13558", "227", "5457", "21564", "69838", "14549", "47755", "70717", "52456", "33560", "4890", "17936", "64191", "50044", "64736", "18316", "64318", "71370", "7500", "22254", "38835", "19630", "4144", "24880", "21679", "62114", "53161", "19370", "66813", "13835", "14051", "31080", "31244", "20844", "31106", "12575", "16228", "41953", "70135", "55684", "47849", "28178", "18769", "67960", "15637", "38811", "37733", "50745", "17796", "11798", "44179", "4882", "36166", "27777", "61527", "51578", "42096", "29970", "58250", "48936", "46454", "427", "26526", "27282", "629", "28657", "30520", "33811", "58397", "41930", "28932", "7520", "43963", "30905", "63304", "62574", "11122", "43442", "62272", "55109", "49665", "42853", "50304", "37312", "8809", "15779", "25286", "59253", "5380", "69165", "66387", "50776", "55227", "32459", "17046", "35846", "29395", "13449", "5508", "15914", "14419", "67608", "69384", "53247", "25035", "56210", "58116", "58839", "36355", "68892", "41184", "8764", "47632", "14932", "52089", "46728", "20099", "42822", "16374", "63257", "35073", "35038", "49484", "43747", "20018", "72895", "8853", "52153", "32293", "11309", "40200", "14128", "17746", "47238", "38695", "68085", "47035", "2067", "38460", "35730", "33158", "16838", "21250", "59168", "15901", "9329", "33442", "59432", "32918", "15628", "60627", "19342", "49609", "35673", "53914", "14489", "8401", "22589", "50180", "51023", "13481", "55083", "41106", "10102", "71311", "8730", "35550", "51737", "111", "30397", "3027", "22036", "9514", "29471", "32541", "14317", "66441", "5967", "44716", "39254", "3846", "57852", "29191", "42110", "32615", "27342", "1569", "42043", "2883", "12828", "23605", "17419", "54064", "13205", "36318", "10532", "2584", "43335", "67538", "15594", "38202", "59969", "34888", "59334", "10763", "6121", "1050", "718", "32340", "66394", "4747", "16803", "38928", "8976", "865", "52747", "44216", "35562", "72946", "44696", "7395", "25952", "66020", "65405", "51980", "14142", "45306", "67279", "35300", "62450", "49841", "21505", "68982", "46973", "38519", "2207", "56548", "58671", "9390", "24540", "9118", "50112", "67682", "4209", "24510", "47265", "73030", "71862", "17660", "71670", "22737", "55874", "25062", "27963", "52221", "41415", "26265", "10975", "41339", "1414", "64509", "10212", "18936", "7755", "58892", "44234", "72100", "59760", "63653", "66638", "63569", "36487", "12311", "1646", "65086", "47125", "64016", "39645", "57052", "59673", "49936", "57808", "71556", "44215", "27071", "11074", "36461", "26772", "47689", "31428", "9303", "1298", "40317", "41896", "54655", "4838", "41050", "21828", "7314", "40786", "4045", "65986", "842", "63850", "55731", "21361", "27340", "11526", "20771", "1454", "13371", "65739", "33918", "43670", "8138", "18877", "56332", "19421", "6773", "14081", "64386", "70825", "53203", "31844", "8157", "59370", "32847", "38840", "55982", "12997", "33604", "66459", "71290", "11026", "21045", "39779", "66310", "56960", "13357", "51577", "21140", "39518", "17221", "67345", "70816", "70122", "57728", "56945", "5504", "34", "55048", "67562", "22765", "65084", "21898", "5429", "28631", "65679", "39719", "56335", "49", "52718", "51299", "6796", "29979", "35510", "66087", "7318", "4920", "33942", "56133", "35954", "27060", "38508", "62534", "2533", "41044", "7765", "15171", "13598", "28590", "32957", "65144", "61791", "35233", "66624", "51369", "67235", "69048", "70555", "66340", "38763", "2141", "59268", "35211", "32321", "62630", "5826", "17062", "66481", "35976", "60147", "38669", "52976", "28426", "42681", "1951", "23059", "36021", "34268", "36280", "14893", "8310", "33632", "3966", "20261", "13470", "33540", "9163", "58072", "36056", "72589", "14767", "21598", "29288", "66542", "29391", "26892", "15456", "7360", "68522", "22484", "34600", "11357", "45605", "59992", "25372", "66788", "58681", "5083", "31599", "61737", "51971", "67808", "17554", "53066", "56475", "32506", "53684", "21854", "25732", "21522", "51635", "65876", "45481", "30509", "43040", "29387", "13285", "10991", "2763", "62671", "13", "7035", "37785", "24515", "62931", "67207", "10395", "58548", "7541", "30057", "8966", "68946", "21401", "51569", "48541", "27241", "27191", "27707", "12277", "20441", "18352", "14239", "49132", "3241", "47108", "58239", "59064", "1956", "59692", "42057", "15373", "72317", "23451", "53638", "32222", "9151", "71754", "1714", "9182", "35001", "50550", "47440", "9432", "35534", "20004", "13284", "20920", "6503", "36793", "63287", "2719", "29561", "39816", "61546", "65912", "49130", "2319", "67767", "35203", "71574", "18549", "19178", "14059", "10914", "55823", "61013", "8864", "3881", "7300", "46101", "58207", "16653", "18051", "408", "50862", "12511", "58819", "8333", "29199", "64315", "45511", "58880", "36404", "47314", "16102", "46630", "72264", "23104", "48693", "55571", "47552", "60775", "31020", "48222", "26712", "31245", "7073", "51394", "7309", "68570", "12921", "61355", "51325", "8192", "50480", "45591", "67231", "19957", "7159", "4951", "68684", "7096", "58945", "72014", "43008", "8121", "18494", "44725", "29833", "4539", "7350", "72704", "873", "33027", "34713", "6532", "30436", "52725", "42765", "46134", "52785", "24454", "55089", "7853", "19179", "58410", "2778", "56753", "30954", "30772", "13911", "67920", "25550", "1630", "71396", "1255", "31309", "59403", "51391", "49624", "30814", "34678", "28212", "6903", "40799", "64239", "46982", "47498", "71063", "11225", "16259", "67777", "65849", "55882", "38900", "1397", "62680", "2847", "22099", "30487", "52852", "36083", "57248", "26588", "29706", "40380", "25882", "70623", "48959", "48464", "41630", "17736", "40194", "18018", "40289", "6136", "60259", "14952", "11620", "64142", "1812", "59695", "51674", "18105", "26472", "5894", "18744", "53143", "56178", "18021", "25156", "53846", "52637", "63414", "68403", "31099", "59768", "11405", "72438", "68228", "69246", "30628", "55281", "54410", "18137", "33782", "38680", "11044", "53666", "12718", "45839", "35162", "21841", "6973", "48194", "28509", "59997", "12987", "35944", "10716", "18574", "9854", "34754", "40802", "18206", "58757", "61803", "46246", "40110", "71700", "70692", "40316", "33117", "17512", "47413", "34379", "31931", "44375", "70224", "51318", "15372", "2322", "5316", "54083", "15167", "5799", "56750", "15066", "72286", "61167", "51220", "60601", "72487", "59396", "17351", "58387", "13368", "37374", "22707", "29003", "19510", "52667", "47776", "7154", "42064", "43099", "69482", "64952", "7677", "57535", "63117", "5619", "44091", "28158", "38923", "73246", "72787", "47395", "47083", "20909", "46465", "85", "34935", "3925", "27320", "8621", "59434", "61452", "16023", "18460", "67521", "4893", "46520", "43964", "53189", "51884", "5062", "56570", "22241", "68035", "19711", "30291", "7942", "16385", "7134", "42748", "65584", "25873", "43223", "29188", "20108", "46521", "49907", "21838", "62823", "23157", "10568", "58", "29912", "10409", "66911", "31083", "25254", "58441", "24617", "19122", "1655", "36907", "23735", "27935", "38052", "20852", "14965", "33291", "21653", "5681", "22549", "35482", "14116", "72490", "11358", "65217", "58211", "23922", "34638", "40071", "394", "29837", "71868", "35169", "7421", "38959", "71278", "15762", "29831", "11967", "31772", "72020", "41678", "3630", "5902", "5413", "35600", "4719", "35577", "23570", "51441", "7748", "37248", "48070", "29774", "2149", "73211", "8957", "36478", "71905", "71374", "16816", "6533", "11607", "69795", "36851", "32841", "70469", "61124", "880", "27190", "31147", "68313", "14362", "5079", "31070", "57118", "68175", "58286", "15471", "34229", "38916", "14077", "48416", "67591", "67246", "61162", "43524", "61657", "44784", "36868", "48446", "31600", "38408", "29559", "36846", "53881", "46070", "36756", "56259", "56679", "19075", "29116", "45586", "38760", "42531", "38818", "29697", "54530", "64210", "30304", "68808", "38765", "13607", "10553", "26783", "68302", "72165", "28071", "57169", "34199", "52133", "2817", "56143", "29429", "11874", "467", "61612", "17557", "72729", "5189", "51985", "69293", "57970", "3042", "64520", "54681", "19981", "8711", "10014", "4578", "27980", "55714", "60838", "47077", "32547", "61703", "24675", "71469", "41193", "665", "9117", "11646", "19940", "41061", "35834", "17565", "63354", "59193", "64748", "8194", "72920", "35471", "65101", "61541", "53324", "61736", "8918", "66841", "20592", "72975", "36084", "18718", "46009", "63871", "56463", "45598", "66505", "11878", "16509", "12453", "34900", "64068", "7049", "37404", "4070", "54720", "4700", "63162", "11662", "22932", "21511", "56938", "46247", "25393", "25919", "29091", "25597", "28864", "6165", "39546", "13353", "31082", "25686", "35464", "27894", "13288", "36714", "46941", "6658", "2854", "32166", "65197", "33099", "346", "50630", "39326", "29757", "13938", "7137", "66837", "3331", "133", "13778", "4185", "4824", "34889", "4432", "47287", "37518", "18252", "51495", "7016", "38860", "71228", "43063", "58488", "1907", "18030", "55188", "68184", "52980", "13506", "27871", "49749", "26323", "10320", "33868", "1807", "15417", "49328", "47922", "55761", "18676", "18400", "16391", "37750", "54225", "64492", "40870", "17058", "27390", "54007", "43962", "22551", "16862", "29389", "30601", "11170", "43391", "24702", "6134", "32067", "3547", "41261", "463", "4962", "59292", "39411", "28156", "5501", "58201", "33041", "8801", "7891", "72256", "13510", "22954", "44105", "24113", "1584", "61932", "15446", "70678", "45317", "20779", "11464", "6960", "16918", "51734", "7839", "61123", "24828", "61257", "39194", "65252", "68876", "6604", "69046", "58998", "68596", "25584", "19078", "51925", "49929", "22738", "49387", "10082", "52117", "16747", "19076", "23880", "15712", "12011", "67522", "18592", "14620", "53065", "70922", "28257", "43019", "61503", "43497", "73012", "12912", "61917", "33056", "35302", "63826", "40370", "61656", "29376", "10338", "35050", "68597", "37991", "43023", "64804", "16926", "32947", "47383", "15710", "19890", "6307", "71955", "1335", "59734", "3758", "9070", "64080", "35224", "47944", "55411", "57965", "29817", "25844", "7499", "58719", "54726", "1697", "5225", "20258", "63209", "47968", "70303", "8876", "20316", "68972", "12629", "41903", "44312", "10365", "55123", "4503", "13367", "20975", "14167", "22632", "66490", "10699", "58196", "25384", "55632", "5469", "42246", "27792", "60437", "34897", "20219", "71307", "3047", "68547", "63567", "53625", "7347", "1882", "10019", "38667", "66035", "39725", "54876", "25791", "54631", "47147", "42301", "54529", "35078", "9057", "43207", "44827", "48796", "45594", "72537", "47075", "41024", "43401", "25853", "23856", "57493", "54072", "65529", "45337", "56916", "73094", "43283", "41772", "62579", "70375", "32101", "10656", "27341", "39955", "59905", "61438", "49828", "11393", "54195", "149", "47714", "6270", "55201", "27985", "23720", "18788", "53949", "72698", "29888", "43694", "58525", "41333", "52964", "45142", "12784", "50727", "66900", "27575", "72889", "64194", "31190", "23535", "15831", "2524", "66046", "58428", "5090", "19013", "65691", "70517", "17436", "42837", "4268", "38280", "51513", "20191", "61686", "70241", "11438", "73201", "72617", "36403", "37573", "3231", "46551", "34621", "64236", "42245", "5915", "8283", "34696", "10458", "55701", "3228", "5338", "28202", "6340", "35710", "9577", "8952", "71413", "70248", "5759", "55374", "29776", "45241", "40888", "42688", "11629", "2693", "23068", "12692", "68230", "43321", "26936", "7692", "61865", "28506", "49612", "7414", "25545", "31136", "7984", "42312", "51997", "57127", "59206", "2372", "27671", "31322", "58354", "32510", "22395", "64327", "4516", "14652", "5075", "63778", "55937", "28766", "46310", "27144", "8715", "54436", "53113", "72684", "58089", "42210", "68517", "25785", "13903", "69943", "47850", "56952", "47853", "64780", "38186", "61604", "18259", "61487", "2967", "60814", "35865", "27260", "62378", "59842", "62605", "36639", "45846", "46434", "15127", "1027", "8961", "33251", "30518", "34448", "49657", "47739", "47733", "71549", "61974", "14556", "71190", "19806", "69999", "30163", "72970", "23222", "46303", "45487", "55587", "14373", "36415", "2315", "70793", "65654", "67680", "64937", "63232", "65137", "61120", "5428", "72151", "51742", "57830", "20134", "67160", "71112", "27110", "14137", "22493", "33144", "31039", "39651", "8088", "39777", "34582", "45486", "27357", "31340", "32485", "28182", "51805", "45342", "64726", "58120", "59965", "33184", "70050", "53458", "61730", "68811", "57883", "18855", "64189", "21157", "7952", "47846", "53214", "45315", "29779", "72392", "33230", "40353", "7068", "53804", "19128", "33725", "45652", "51179", "13232", "37765", "70062", "21765", "23034", "36529", "54278", "30998", "53766", "4501", "5953", "23412", "54320", "54782", "5278", "55720", "19842", "64193", "8237", "36898", "19123", "22777", "41270", "70305", "65534", "20747", "64788", "37762", "15503", "48941", "47880", "59561", "7411", "58040", "51160", "46988", "41645", "21490", "13907", "34514", "26660", "52875", "27634", "71030", "17496", "47044", "41635", "52614", "48151", "34817", "34642", "9185", "60880", "14989", "61497", "46107", "18642", "36674", "69092", "69935", "34866", "32381", "38139", "51628", "58482", "23629", "24517", "23595", "22509", "59998", "37899", "70645", "34722", "52432", "32815", "56871", "2336", "35167", "64150", "49509", "42331", "59271", "69523", "28532", "44186", "4984", "38657", "21137", "23063", "52584", "43501", "36834", "5391", "54561", "10737", "67395", "9245", "42037", "52669", "14998", "45102", "33771", "22230", "7567", "12599", "1429", "59745", "24582", "29102", "29957", "14621", "30796", "4177", "36867", "28001", "13167", "61846", "21319", "13908", "42894", "12426", "37036", "72615", "20503", "4227", "72917", "4966", "49633", "5831", "50812", "10711", "36969", "63916", "56106", "50155", "1386", "54842", "71734", "34022", "53750", "45262", "71954", "19040", "39973", "56888", "20652", "30787", "66351", "45190", "70501", "21083", "20145", "34510", "18431", "70481", "62731", "22369", "8608", "68196", "11092", "70060", "55185", "20251", "11551", "21700", "62257", "28576", "60630", "50860", "27162", "53951", "15366", "14973", "38415", "46539", "9209", "10706", "49903", "51152", "38248", "6258", "3695", "126", "1266", "150", "48220", "23252", "17963", "5164", "46278", "12103", "12530", "23418", "39473", "24350", "72460", "69557", "44243", "16107", "11601", "8158", "18290", "9979", "5121", "19147", "49034", "14366", "6631", "55734", "26786", "27973", "41107", "29433", "30252", "40277", "62956", "61624", "45406", "30482", "34668", "22773", "11770", "9661", "55794", "45848", "36524", "53864", "27319", "4226", "69332", "40747", "1062", "20459", "18164", "51984", "60535", "14183", "34421", "51131", "760", "6608", "28234", "65603", "29368", "70760", "53454", "28350", "72186", "42722", "11454", "9518", "65079", "57943", "9013", "45151", "63743", "4497", "28475", "6384", "9403", "61140", "2387", "64049", "30686", "16566", "64914", "45432", "59002", "33575", "35308", "44453", "31482", "23874", "51851", "24139", "70663", "8429", "29438", "38542", "68451", "71189", "45279", "19240", "47338", "18371", "6392", "70218", "48909", "60142", "38201", "43778", "6892", "49626", "37006", "48271", "27779", "20653", "10732", "8049", "17514", "30411", "31016", "46122", "24215", "71206", "4591", "63498", "17955", "65250", "9099", "26533", "18570", "28215", "5706", "58996", "32230", "1304", "41056", "9804", "47631", "53086", "35173", "25831", "35441", "57441", "8149", "6541", "70764", "53558", "50799", "57679", "4346", "36936", "39598", "44259", "70733", "52066", "57021", "9318", "56162", "68406", "13148", "60217", "28659", "11960", "31041", "69006", "5087", "43195", "44289", "42997", "21506", "22864", "15129", "21404", "12802", "50620", "5112", "23253", "45859", "48801", "281", "5274", "45145", "19141", "49524", "5362", "52454", "33809", "36400", "69944", "36485", "52632", "24175", "17233", "70913", "54190", "51155", "3887", "48146", "27057", "33523", "58990", "68579", "32318", "30302", "15291", "50616", "71568", "45007", "64245", "64921", "30623", "73227", "56628", "59034", "25639", "62151", "1582", "18169", "18418", "69950", "44658", "68504", "14344", "27679", "61067", "41546", "18926", "38902", "43117", "25925", "39609", "41900", "25341", "43190", "34740", "24286", "71470", "9880", "32683", "61352", "30248", "51285", "3470", "14096", "26937", "42197", "41036", "26286", "43054", "40537", "54255", "47578", "69226", "72776", "40143", "50397", "27068", "40534", "12593", "26128", "41593", "35079", "55759", "8337", "22028", "25104", "44382", "61747", "54405", "6138", "36095", "17539", "64855", "50819", "34130", "29403", "1498", "70455", "13554", "8900", "5248", "22700", "57088", "4161", "52449", "70753", "47660", "2596", "7403", "25216", "7121", "65171", "35343", "32140", "49696", "5289", "70172", "39280", "18439", "30133", "14553", "41697", "35312", "11887", "27298", "40981", "45461", "73008", "32300", "35189", "36534", "34133", "28995", "67276", "59521", "54375", "64836", "57838", "46283", "35180", "68255", "15023", "13279", "25031", "13725", "5394", "763", "54023", "47925", "1059", "51277", "22783", "35337", "57794", "62745", "19196", "10969", "55939", "45888", "53723", "71345", "31339", "42501", "36782", "12166", "42667", "14047", "51104", "59515", "3176", "25387", "27521", "23729", "50495", "41445", "39592", "5709", "18483", "36361", "39623", "21950", "1009", "49898", "45764", "12501", "44367", "22358", "28961", "51132", "69331", "53177", "38836", "58845", "19401", "67188", "71400", "72846", "24929", "6220", "41428", "52419", "12615", "21412", "4124", "18194", "61481", "20448", "35153", "72859", "37289", "14662", "9952", "60766", "56368", "30274", "70146", "25781", "12879", "37199", "45633", "11571", "44679", "26178", "10633", "22133", "34575", "11236", "65493", "19922", "22552", "1142", "30070", "47188", "27900", "62351", "11953", "57782", "4488", "43361", "49221", "9997", "61943", "52794", "9650", "26371", "61195", "23030", "40749", "68708", "70410", "22719", "12382", "13096", "22256", "22750", "21029", "19556", "42491", "10928", "42027", "62370", "26648", "7015", "14701", "71560", "29442", "65574", "21224", "46041", "25307", "12206", "32750", "2678", "19414", "960", "48622", "59792", "64428", "57913", "6364", "71737", "71945", "15736", "58209", "69663", "63299", "65127", "17798", "54746", "7680", "27188", "26641", "5327", "54523", "7486", "54597", "61675", "8031", "22875", "22924", "70837", "33405", "36736", "59000", "56005", "50366", "69650", "55044", "70367", "34633", "49266", "35678", "1870", "26827", "53216", "1821", "62165", "48254", "63957", "70921", "55606", "55181", "64669", "65859", "25293", "71384", "5131", "21867", "35593", "11547", "36519", "36149", "45128", "47928", "65163", "37492", "63288", "22732", "61529", "54390", "67995", "40671", "64642", "57983", "31478", "9884", "1531", "59908", "40781", "63796", "3882", "493", "52296", "56074", "48236", "24151", "63156", "51150", "63217", "17491", "31607", "6293", "8267", "46336", "34747", "37144", "53575", "71940", "16423", "22896", "66533", "64192", "57743", "4619", "30081", "68860", "71122", "32183", "43336", "48880", "66603", "55156", "71391", "43644", "51463", "23902", "36366", "69973", "9617", "21839", "50600", "4795", "31580", "8197", "68681", "38269", "35284", "32008", "53455", "41893", "25315", "37712", "31199", "29840", "55261", "49029", "47916", "7060", "67455", "66471", "65114", "26262", "8512", "26224", "4257", "18094", "39356", "7297", "73000", "35008", "30973", "53137", "63958", "13183", "68704", "36057", "66158", "40412", "48594", "26388", "13308", "71021", "21028", "9959", "15077", "42441", "24386", "24019", "41088", "6207", "433", "70809", "72818", "57596", "55335", "68501", "18827", "48651", "62273", "59175", "69673", "12422", "11204", "50083", "69689", "44146", "24981", "36672", "9667", "36312", "46331", "35276", "67051", "27852", "7246", "50751", "72767", "31834", "18741", "851", "68686", "59113", "50513", "71555", "69433", "60996", "29329", "33211", "51697", "31201", "2201", "5539", "47783", "43871", "44652", "8695", "27062", "8729", "55676", "64004", "40838", "14626", "33607", "42941", "5906", "2816", "67457", "33692", "52399", "39852", "47066", "12578", "47341", "1703", "1094", "57647", "23821", "65651", "48999", "71762", "1301", "40153", "17152", "62578", "59587", "54914", "62995", "34212", "46359", "42387", "39799", "63729", "45369", "27798", "31285", "59462", "32285", "36281", "1028", "19902", "1932", "6051", "44542", "41140", "30698", "18507", "63142", "52409", "44815", "18254", "6189", "5085", "7712", "23855", "39212", "68518", "19426", "54476", "14464", "73029", "71069", "72362", "6530", "44248", "601", "58733", "8509", "67904", "50841", "3588", "31151", "44682", "14990", "22899", "39972", "48857", "66138", "19524", "35152", "56235", "52625", "59552", "46291", "58038", "22145", "31864", "3753", "71675", "44803", "10517", "30615", "28747", "52323", "65896", "37529", "44893", "50095", "25507", "10335", "52749", "8780", "11905", "19329", "15596", "34880", "58084", "12063", "68191", "29054", "33645", "68928", "48955", "45018", "50354", "63121", "50518", "2743", "66626", "23585", "70276", "68930", "60796", "11043", "16152", "50592", "56860", "47825", "39289", "19665", "60746", "3422", "15026", "72012", "32337", "61905", "35120", "13856", "41504", "6621", "27339", "51646", "38790", "8691", "33574", "19847", "1310", "4713", "23455", "18485", "11731", "52059", "64813", "23424", "41222", "7158", "54159", "17038", "3090", "3965", "36871", "72463", "67428", "5161", "46171", "4377", "64591", "50217", "46021", "70681", "32443", "21182", "30702", "6819", "59276", "51504", "52186", "58832", "33658", "24226", "37423", "12203", "23732", "25383", "60879", "45740", "29596", "58424", "63292", "57482", "13460", "57461", "3294", "38742", "6115", "71593", "19334", "31811", "30023", "3814", "7888", "52126", "58987", "3402", "29864", "36803", "26020", "70846", "13248", "20862", "4549", "11112", "55113", "1688", "62900", "63212", "18890", "25396", "66028", "26982", "7995", "46754", "15838", "48368", "63404", "14727", "43429", "26848", "27525", "14498", "66962", "46832", "2445", "3935", "26930", "35406", "37587", "56438", "54011", "18828", "39412", "68857", "37755", "42605", "47219", "63134", "12151", "33955", "28099", "38300", "21454", "59410", "40372", "33390", "62386", "53836", "23534", "51498", "37553", "51313", "36313", "66645", "30011", "61906", "3101", "41311", "5134", "2216", "46481", "56812", "67943", "4746", "35779", "33482", "63882", "61561", "62157", "25876", "60693", "26157", "60688", "64009", "63269", "38626", "69335", "49314", "6547", "8417", "17761", "54952", "64022", "4635", "52722", "20590", "64061", "8469", "72613", "18659", "11640", "38730", "12076", "61468", "47591", "1470", "357", "33105", "45090", "19011", "48189", "24719", "43478", "27443", "41506", "53094", "33543", "31601", "34419", "3524", "10362", "62303", "5110", "27914", "29663", "54197", "29672", "19527", "54002", "2257", "60499", "67829", "42856", "47486", "32123", "23656", "908", "60146", "57195", "18945", "40827", "25066", "37314", "51817", "51059", "53425", "57106", "14737", "9673", "1566", "66339", "32688", "22585", "2740", "66450", "70026", "30031", "62884", "41331", "56311", "43341", "28573", "25676", "2377", "6026", "20185", "23154", "53298", "49004", "49036", "56042", "5957", "67032", "53694", "69731", "55038", "26047", "27869", "59909", "44410", "45069", "30446", "6849", "22961", "70741", "30829", "30789", "14557", "12003", "63098", "45708", "56271", "51454", "58358", "45427", "57136", "27988", "44362", "55231", "43035", "11634", "12760", "32509", "48810", "34506", "34066", "47036", "33568", "69782", "35870", "5493", "54554", "4335", "10496", "19894", "46999", "13861", "6763", "46431", "9283", "12623", "22850", "59557", "20208", "41200", "65108", "71975", "59821", "51399", "41391", "31483", "48176", "5389", "59973", "63898", "52120", "2392", "25978", "381", "39383", "44908", "19190", "58513", "35980", "58739", "63786", "42071", "39834", "29238", "65742", "8786", "69151", "5203", "66192", "40488", "6893", "71895", "16899", "23505", "36900", "66771", "41902", "33814", "65888", "60974", "24856", "20680", "25171", "43304", "30667", "51840", "12594", "6679", "30916", "68904", "43960", "42260", "52641", "57696", "55239", "7742", "5449", "47257", "57274", "23942", "61553", "72974", "44085", "50677", "23546", "63229", "64082", "66475", "63991", "70423", "23499", "6298", "17476", "32194", "4330", "39711", "23971", "28633", "64151", "69315", "26539", "72724", "37217", "32527", "71992", "28508", "3051", "14945", "46060", "28376", "38306", "63524", "3464", "35732", "18953", "48483", "9535", "19345", "58400", "68425", "26225", "65058", "38858", "33757", "16820", "33289", "28776", "35566", "26404", "47157", "54834", "12334", "41136", "50759", "17265", "10640", "22662", "35588", "71885", "70612", "50219", "67816", "23112", "50454", "5660", "36299", "60809", "617", "8605", "22419", "45504", "31913", "46211", "57515", "4733", "47434", "27269", "29941", "69885", "71", "2958", "19", "40532", "31998", "58734", "21114", "33719", "35125", "17218", "4107", "15076", "1756", "54158", "60793", "28029", "39478", "4940", "1348", "72094", "23122", "1253", "2081", "9505", "25020", "73207", "58246", "40871", "67216", "55232", "9566", "15938", "69233", "17567", "22408", "5971", "63282", "26869", "15194", "17843", "41481", "55045", "26066", "56778", "3394", "53768", "1828", "28167", "30060", "59569", "12751", "59198", "27948", "5446", "56119", "6327", "52701", "62345", "54461", "10373", "57681", "66151", "66063", "26285", "57558", "41057", "67985", "8799", "30258", "44956", "65834", "42135", "56839", "6234", "35291", "23368", "35451", "54351", "20887", "11658", "12307", "12273", "19232", "47254", "17774", "6053", "5988", "8975", "43796", "53235", "61764", "56655", "11352", "40649", "3364", "70833", "5300", "70703", "51965", "39158", "53165", "1501", "54051", "49459", "30092", "62885", "64120", "12722", "10654", "10148", "2071", "25087", "44917", "8048", "31617", "35490", "12891", "20838", "43141", "33215", "60617", "59816", "3572", "42184", "52784", "62993", "13733", "8391", "26507", "25973", "58105", "17917", "3270", "4271", "26763", "45036", "50931", "33208", "26809", "10663", "5058", "31936", "35271", "66907", "64810", "66998", "27846", "16826", "35422", "1553", "12621", "16033", "57609", "65025", "17785", "3171", "3202", "21245", "38512", "51687", "70956", "40360", "46782", "5841", "36049", "20256", "70405", "71912", "6730", "18044", "40642", "41739", "6222", "21281", "61767", "15205", "19972", "14979", "63675", "55935", "40504", "8468", "10442", "66379", "51000", "28550", "61128", "52493", "36768", "49685", "26967", "70996", "6353", "64705", "17285", "57945", "43454", "7169", "64861", "45133", "72002", "8038", "3872", "53419", "39351", "61990", "20640", "44087", "27569", "20543", "62564", "16497", "64219", "49487", "50995", "14740", "42001", "57025", "62175", "37115", "31004", "66148", "27182", "64838", "45727", "47676", "27810", "46525", "25176", "31881", "70525", "5089", "31737", "62909", "51726", "71627", "33829", "44945", "72359", "72948", "40727", "27573", "13080", "38432", "59541", "48339", "69499", "44290", "15485", "47103", "64105", "3943", "61951", "29476", "5774", "39461", "55605", "70093", "34092", "39485", "54820", "27111", "9756", "1940", "5573", "22367", "49395", "52332", "34182", "37447", "1203", "40006", "13573", "47124", "69221", "65726", "31318", "72640", "65012", "45268", "12969", "45493", "13307", "22121", "49138", "28345", "20678", "4744", "24838", "58689", "17163", "26422", "35371", "70451", "5908", "10882", "50583", "47410", "17626", "6031", "49972", "43471", "65070", "11284", "32679", "63691", "54545", "32963", "28219", "38641", "51567", "55875", "68937", "26230", "11517", "62788", "48202", "61012", "58332", "57801", "43245", "11095", "15877", "35255", "35731", "18537", "13557", "32629", "33578", "19792", "62222", "15832", "164", "47411", "66097", "70124", "16494", "55627", "51331", "3593", "66128", "55019", "44727", "10036", "7289", "59001", "1549", "35971", "7262", "57675", "41992", "3451", "50005", "2692", "61280", "43070", "55182", "33825", "25968", "5484", "12888", "12761", "60339", "25546", "11908", "35907", "36174", "49435", "6470", "39426", "55406", "7037", "51457", "3377", "68716", "37359", "24274", "65612", "2502", "70770", "62041", "12411", "67957", "27135", "53829", "37367", "63930", "25634", "23333", "30597", "26118", "37095", "24509", "24569", "14759", "3731", "1881", "40053", "788", "54322", "67273", "44048", "14795", "23760", "7107", "33536", "72064", "1040", "19383", "46779", "53792", "19149", "3010", "56492", "28064", "53480", "5115", "29255", "56547", "3247", "53463", "28515", "20501", "14318", "4285", "40986", "71984", "7863", "67384", "18732", "11984", "11721", "9213", "11812", "16247", "16235", "46229", "64176", "48239", "9208", "46517", "55955", "19699", "25033", "14435", "65495", "28940", "21295", "35179", "36896", "2789", "19432", "18973", "20320", "63815", "19207", "28241", "13591", "21195", "51107", "18781", "35085", "16463", "22630", "7426", "13899", "29947", "50168", "29339", "58391", "42496", "31026", "70382", "44849", "70081", "72048", "45265", "56239", "28253", "48797", "46082", "6314", "15384", "53135", "50081", "18103", "66890", "28486", "63911", "58608", "63101", "24231", "55713", "26004", "19897", "11817", "46612", "1502", "28497", "17317", "43210", "21040", "33792", "72971", "29286", "5250", "48785", "34401", "48427", "71493", "6762", "62824", "40765", "10705", "59452", "62248", "15606", "53643", "35668", "17022", "41039", "36351", "10874", "51607", "62950", "71254", "69388", "65960", "8521", "60347", "15035", "55715", "61178", "26606", "28036", "34673", "37433", "25400", "28432", "52501", "38368", "36951", "13255", "68700", "60951", "3859", "60235", "4269", "66964", "12976", "35698", "59676", "50938", "71803", "14487", "61853", "5566", "37178", "13008", "18932", "12447", "37383", "1441", "21999", "26389", "8619", "65646", "1855", "1178", "9304", "2218", "24238", "15980", "57340", "9918", "44352", "50035", "72632", "46718", "32207", "60484", "72696", "22132", "62802", "4989", "18967", "67328", "29967", "24396", "55466", "15019", "65626", "19804", "39439", "6322", "5819", "46604", "15157", "8035", "48358", "58877", "31446", "41686", "9831", "1750", "50143", "15810", "38061", "40997", "56188", "43284", "67054", "28630", "32248", "24729", "21934", "50240", "67240", "30439", "15365", "72991", "2586", "34887", "37754", "15259", "3852", "316", "70512", "69028", "49595", "19021", "20655", "51919", "12137", "69844", "45802", "32891", "26956", "66668", "49338", "48095", "68774", "55068", "35376", "69322", "7550", "73250", "58073", "35722", "33321", "9181", "12064", "33849", "58260", "14581", "45276", "38172", "17475", "65134", "29517", "63682", "62722", "67371", "28083", "16525", "38945", "24236", "65314", "49934", "72367", "41736", "56799", "61959", "35633", "67850", "26434", "26358", "41052", "39393", "29766", "45861", "50189", "2544", "6534", "3466", "31121", "50556", "41017", "13820", "9835", "31069", "45553", "29374", "42768", "30903", "51353", "28034", "21108", "68045", "43607", "42745", "48519", "1656", "11473", "66174", "27009", "42959", "55047", "45313", "24292", "29357", "56243", "31275", "23558", "9532", "38762", "56882", "17169", "40980", "2467", "13967", "3386", "39170", "61518", "62446", "2406", "55784", "30993", "61660", "67311", "54762", "10514", "38391", "59485", "9361", "17485", "41151", "60686", "67596", "37085", "4279", "654", "73079", "36579", "56164", "16001", "57498", "67536", "10480", "48283", "44970", "29482", "19119", "12493", "20883", "6580", "46272", "7493", "48360", "24202", "22540", "62358", "20292", "38285", "48885", "13606", "7451", "51175", "365", "9942", "25885", "25093", "31005", "69606", "52005", "25904", "29932", "52867", "18666", "47335", "2941", "58135", "15378", "50642", "49422", "65318", "29292", "40820", "51850", "13095", "66653", "65900", "67326", "14994", "72408", "37410", "15554", "22082", "53651", "68438", "36645", "11186", "15346", "53130", "42660", "55127", "8372", "36421", "45249", "1912", "41234", "11237", "36708", "61407", "65620", "46276", "63138", "36590", "31089", "26177", "17542", "19938", "25323", "59949", "42002", "16290", "52212", "55219", "45213", "10038", "13103", "47831", "13920", "44246", "18798", "57145", "41239", "49399", "66578", "7179", "72812", "8884", "42995", "68178", "27790", "50902", "11985", "33519", "70046", "70014", "22125", "41733", "3180", "32749", "61213", "16927", "18383", "15805", "60884", "48678", "53267", "9001", "66223", "8839", "7689", "12863", "31000", "50701", "63072", "1448", "3567", "10854", "24451", "10909", "62953", "53215", "53753", "55059", "46585", "66705", "68334", "50182", "31162", "52491", "63947", "4833", "52943", "7551", "68331", "32065", "57734", "40286", "36126", "32955", "69898", "25351", "3880", "5554", "31010", "18756", "59245", "5339", "52892", "48156", "53436", "41359", "44534", "41463", "32790", "37728", "53739", "50795", "60635", "37365", "51392", "14114", "38119", "66454", "46167", "675", "35351", "37703", "29221", "60009", "72187", "32548", "19409", "65987", "37163", "65630", "2214", "23911", "47218", "23114", "7857", "35524", "10795", "65733", "13810", "66253", "33107", "9494", "1799", "66330", "22263", "47001", "64439", "60000", "47843", "30897", "69104", "29493", "40505", "30607", "31087", "35156", "34932", "15837", "19721", "22807", "9201", "5409", "43136", "222", "41870", "69360", "69928", "71771", "66774", "11870", "66999", "15572", "62270", "1080", "55756", "4262", "23193", "60911", "59129", "7934", "43658", "71219", "42042", "34325", "21907", "43636", "54585", "29428", "54983", "60558", "55097", "69012", "15562", "58479", "55706", "63426", "58690", "45826", "69522", "34117", "12296", "60610", "53108", "51164", "34749", "19658", "31229", "55733", "34997", "51898", "65540", "69841", "20205", "61", "54793", "62343", "71648", "22377", "31639", "52586", "21581", "33884", "42286", "33973", "3744", "15011", "15785", "59115", "61606", "23943", "62668", "31105", "49537", "64609", "40445", "30468", "48672", "3302", "22115", "65875", "23791", "770", "68599", "37135", "60834", "52514", "1287", "46958", "29014", "62054", "26187", "29005", "9421", "22537", "66405", "65175", "48595", "44266", "64598", "29463", "24127", "51126", "5219", "44689", "22888", "19197", "43912", "30102", "15213", "18182", "31964", "39857", "53744", "71958", "71800", "17751", "35205", "62906", "13237", "16260", "68755", "16483", "796", "42536", "61896", "53637", "11858", "50741", "17212", "2517", "4538", "44787", "42020", "52862", "57480", "49456", "61252", "72526", "46869", "63443", "492", "29673", "25045", "11455", "11801", "23244", "44633", "63056", "1191", "50510", "14612", "56246", "38381", "3372", "40403", "7335", "36024", "22217", "15616", "19347", "69998", "10979", "17327", "32200", "54269", "58414", "41194", "11453", "32068", "41786", "32351", "28588", "25815", "72368", "42240", "53527", "7293", "25850", "40683", "17960", "52952", "32502", "45041", "38520", "15420", "6159", "63401", "33866", "34564", "37656", "24947", "16936", "44551", "32672", "49672", "31590", "25223", "36726", "13659", "12403", "1942", "18160", "35956", "49996", "38386", "9913", "28686", "38505", "24922", "21479", "38977", "40514", "49267", "70342", "1193", "29991", "22022", "42819", "650", "28478", "593", "67599", "53591", "4480", "43776", "38303", "14803", "45749", "15826", "55904", "53863", "26064", "14320", "53785", "27787", "2998", "1111", "744", "1467", "7890", "5620", "61160", "72701", "73215", "57357", "51336", "56375", "67111", "12407", "10584", "5044", "40373", "2922", "31956", "355", "41424", "49165", "60160", "6774", "60774", "31915", "1794", "54030", "49867", "33521", "9435", "72982", "7144", "59320", "38590", "17847", "44263", "16316", "19803", "31569", "35369", "17596", "27353", "48534", "9228", "7860", "52693", "31573", "29849", "4729", "19254", "1725", "31582", "50976", "47058", "579", "23712", "19139", "41370", "4967", "21706", "6308", "46736", "38175", "20998", "57205", "45719", "73139", "70998", "52925", "26044", "37099", "1463", "40513", "4664", "45675", "18465", "31486", "32052", "29275", "45380", "44958", "12685", "43566", "49414", "46471", "37631", "63496", "67449", "70032", "15120", "32990", "32716", "57747", "31776", "31640", "43066", "48457", "53979", "22119", "55431", "57579", "48842", "21629", "15760", "10235", "15696", "46865", "42449", "16784", "15133", "72245", "20345", "29951", "31129", "41495", "6814", "13143", "68654", "11780", "58372", "37185", "54853", "69667", "4543", "67297", "1660", "42433", "41155", "68411", "15208", "29962", "2494", "6454", "32521", "70175", "1919", "23471", "63178", "7556", "6723", "3637", "20178", "22940", "8992", "27653", "29216", "60338", "9896", "2965", "25019", "8268", "62301", "18571", "39604", "63885", "43905", "27380", "45070", "27436", "18775", "6831", "45254", "61967", "55533", "69542", "10508", "45587", "40208", "50419", "37572", "23408", "65021", "11485", "42271", "38732", "3471", "57408", "36089", "41699", "25629", "41674", "32898", "34173", "71248", "67980", "17606", "30056", "52287", "63400", "65261", "16807", "63368", "33334", "10694", "28395", "23523", "25478", "67632", "47158", "35761", "60541", "22910", "15697", "5442", "29799", "30150", "12869", "17965", "45705", "36074", "70961", "29781", "17365", "72680", "36730", "47109", "375", "71407", "13981", "709", "4456", "10230", "68587", "19561", "32002", "51620", "9306", "41422", "49345", "72451", "14783", "64778", "97", "47594", "70859", "57406", "29384", "70110", "36737", "17129", "16694", "28966", "70746", "32971", "48351", "24434", "21652", "21042", "2037", "3448", "32813", "55477", "68100", "692", "29461", "26583", "47234", "62786", "54633", "8870", "63252", "60700", "65644", "71810", "21135", "43639", "41482", "37191", "37821", "56163", "41263", "68569", "44656", "29171", "41435", "37354", "13413", "40951", "30228", "8288", "15627", "34566", "32920", "40638", "480", "28690", "59708", "40853", "36850", "30727", "13605", "20972", "52910", "26137", "59291", "22503", "57185", "62139", "62116", "32353", "63430", "20080", "51682", "58164", "5417", "60257", "37574", "55460", "11015", "33783", "31967", "42278", "64999", "56504", "57424", "9235", "20042", "38190", "72246", "23343", "70358", "45346", "35129", "7725", "62075", "28860", "67351", "30827", "158", "11618", "65846", "67695", "21433", "68841", "14017", "52689", "55976", "32670", "6129", "44669", "33198", "49015", "28952", "28322", "60429", "67731", "65613", "24334", "61137", "71097", "45609", "43896", "62369", "45762", "63768", "38006", "25956", "16153", "46182", "1845", "70889", "59769", "12161", "8463", "29135", "68990", "11258", "52828", "60848", "14307", "70575", "67708", "15921", "63112", "52577", "54804", "37910", "60126", "28342", "40267", "15097", "36186", "54623", "42836", "72551", "24062", "62516", "33991", "24462", "34988", "32018", "5682", "12366", "71259", "17255", "44284", "24807", "14873", "70210", "42173", "31029", "8531", "27520", "27232", "22161", "3096", "24527", "16443", "33693", "54764", "5903", "71919", "68540", "65202", "52591", "56044", "46298", "69747", "31448", "37114", "179", "39756", "71301", "4297", "4676", "72171", "7445", "57817", "30618", "16557", "23139", "37486", "43296", "2560", "21886", "61566", "31093", "47045", "13483", "48555", "41122", "36031", "12356", "22785", "44094", "41964", "73136", "32616", "71676", "68844", "57222", "57023", "68651", "5864", "52592", "26817", "18681", "8508", "23012", "31283", "64899", "44698", "64582", "41295", "32901", "24033", "14709", "54374", "21669", "17669", "33459", "25424", "61542", "20772", "925", "34604", "55548", "65966", "30424", "25123", "7406", "7067", "4683", "25621", "36561", "29487", "17902", "13679", "52329", "16814", "45055", "54981", "31207", "377", "31011", "63846", "17179", "34455", "73025", "42224", "21968", "15184", "38345", "45510", "51340", "2479", "38926", "72078", "40650", "16597", "59208", "26090", "27233", "12588", "69771", "15873", "2579", "36980", "65761", "3319", "6911", "53109", "44341", "10187", "53009", "34396", "29299", "53275", "63781", "33353", "929", "45246", "61562", "57455", "47384", "7481", "51014", "543", "64314", "73016", "36961", "36277", "73130", "68754", "37252", "47160", "43278", "27316", "47024", "62562", "57940", "26474", "62673", "64834", "68138", "59273", "53142", "55801", "29990", "11921", "1493", "35480", "15119", "26920", "21342", "29261", "38449", "69525", "23444", "30933", "33324", "59893", "50156", "59504", "21086", "1899", "61083", "8380", "31957", "38089", "37078", "13178", "1575", "127", "18140", "55307", "18652", "55581", "55168", "50099", "38967", "27645", "26933", "17178", "8883", "31175", "33203", "33979", "5473", "21502", "62016", "64986", "10529", "58355", "35355", "19268", "10844", "27467", "59756", "20468", "35118", "47757", "50861", "12711", "11265", "11977", "26939", "5163", "26968", "58682", "33873", "46054", "53466", "72747", "25794", "40501", "26969", "37240", "63572", "28650", "24469", "23600", "70119", "50837", "72732", "36948", "9230", "5530", "14879", "5443", "36189", "43289", "31379", "8707", "38437", "55988", "18010", "46667", "50192", "44244", "49539", "28466", "32649", "48268", "18471", "54430", "17560", "22664", "41450", "11558", "61370", "47861", "24637", "48816", "70056", "2903", "33896", "21240", "2208", "2923", "25522", "48439", "26642", "16613", "6627", "4038", "19374", "47511", "27536", "44530", "40564", "20536", "9618", "23175", "51053", "71594", "43726", "25753", "3239", "22710", "5767", "38405", "19392", "471", "20710", "50917", "59189", "21801", "39434", "44566", "50588", "20111", "68909", "70440", "71480", "18341", "55722", "23672", "42692", "53645", "32261", "71394", "6919", "15881", "33717", "8907", "8452", "44546", "33451", "13710", "42076", "1681", "13994", "39455", "57552", "70369", "26312", "52163", "43015", "42229", "42400", "64383", "63712", "33398", "1893", "1796", "38340", "48507", "18963", "33214", "41782", "26571", "33858", "48496", "31799", "27649", "15396", "34008", "8460", "13485", "61493", "49454", "42823", "22412", "38515", "261", "12716", "36874", "64758", "1167", "43767", "43514", "8622", "47042", "37073", "34059", "15468", "17376", "10888", "51893", "50504", "201", "69886", "40518", "30937", "44554", "25636", "37851", "23892", "57180", "59911", "21423", "30334", "51218", "72900", "52558", "56422", "35797", "55422", "47546", "60704", "23243", "7244", "22069", "20073", "62201", "60327", "44153", "40384", "15286", "6027", "62353", "34522", "12209", "25297", "6739", "65083", "49137", "72158", "60466", "39196", "31255", "35637", "46991", "70662", "25410", "46003", "34962", "9842", "66093", "64448", "15086", "47251", "15407", "35790", "30032", "66984", "55439", "21722", "12683", "36566", "48372", "751", "54377", "48064", "38033", "29687", "48556", "53991", "49574", "44333", "5063", "37143", "5033", "57953", "57583", "73155", "33592", "9338", "12070", "30726", "36731", "32970", "50661", "26635", "30941", "8850", "66393", "3102", "11828", "11220", "55620", "60139", "5119", "47642", "40971", "20577", "34645", "65981", "67755", "45626", "65853", "35711", "42901", "24121", "46314", "69363", "11964", "41041", "33961", "27626", "69270", "570", "64307", "48370", "55118", "40901", "65426", "66982", "31754", "25453", "40314", "7433", "38039", "13480", "62772", "44925", "28718", "49634", "28951", "22253", "55510", "71603", "26033", "927", "9965", "62207", "53167", "30966", "46909", "53740", "5929", "8206", "56030", "27403", "10984", "70658", "45980", "2461", "38800", "70000", "28694", "15448", "47193", "62342", "25651", "54810", "66488", "40283", "16547", "256", "42637", "71763", "25002", "16323", "40072", "72275", "72046", "48992", "42471", "19861", "13327", "62727", "29017", "53563", "58443", "63794", "58610", "71315", "60148", "68931", "35452", "12696", "51212", "32278", "37242", "58194", "51449", "41909", "29132", "14202", "58828", "71860", "667", "28970", "1138", "8205", "47934", "7388", "39728", "29578", "55643", "23190", "38133", "34106", "15080", "44504", "7023", "70042", "8867", "20619", "69723", "33883", "20839", "34785", "20734", "39397", "35370", "37330", "63750", "16668", "13340", "19739", "67972", "39863", "8252", "58649", "58676", "47337", "12302", "11939", "72672", "18996", "63051", "13277", "26867", "39018", "60200", "68693", "7295", "3997", "47793", "9003", "1987", "72885", "30176", "16579", "55981", "54796", "58695", "39131", "46956", "58165", "921", "10027", "31037", "67771", "36499", "22042", "24200", "65905", "52633", "62211", "52010", "55215", "25349", "17747", "31214", "19236", "49142", "7883", "34175", "26046", "50087", "12034", "38337", "16842", "17287", "72022", "45193", "24199", "9521", "62445", "50953", "18714", "70498", "1777", "16991", "51975", "57561", "25235", "25388", "54199", "18367", "46164", "18778", "70180", "52762", "61477", "9315", "23830", "29667", "64735", "15755", "64648", "67757", "53886", "38990", "28594", "33331", "22653", "18133", "53717", "12935", "12876", "25155", "12851", "51639", "71977", "2119", "3812", "52421", "63656", "67822", "7701", "23073", "52477", "40940", "3942", "31423", "8413", "16123", "45689", "49712", "6910", "17786", "1823", "36082", "25251", "13552", "67337", "60167", "53995", "40356", "16479", "41266", "67872", "1099", "73191", "7693", "31043", "266", "67164", "65062", "22541", "14929", "47114", "9520", "7168", "6020", "44922", "73028", "65071", "70682", "7062", "35738", "67493", "10758", "17109", "64460", "7879", "14388", "60333", "23477", "10719", "13076", "61269", "37992", "42261", "5066", "47782", "25190", "44500", "53078", "65438", "11963", "12725", "6338", "4576", "31604", "48742", "34891", "47767", "16305", "1943", "14661", "15885", "43323", "39040", "21252", "52941", "9525", "31391", "69775", "51268", "65308", "45651", "18319", "28080", "16734", "6583", "8575", "44948", "52040", "28221", "23647", "52031", "22519", "55196", "25878", "25252", "38487", "34322", "58559", "68040", "44035", "3230", "48743", "72364", "37004", "47589", "40903", "33264", "28457", "65524", "60249", "62888", "25966", "65809", "10398", "38312", "6949", "42668", "43754", "54758", "72690", "37088", "55205", "26419", "9823", "54565", "71141", "70654", "30067", "20676", "54184", "57619", "54056", "41205", "8779", "4689", "62073", "22798", "54076", "9619", "5352", "34041", "70294", "61672", "64764", "8664", "27193", "25751", "55111", "54783", "39516", "20613", "50778", "17112", "60359", "22645", "7937", "67143", "31727", "23433", "3929", "57663", "2504", "50975", "42840", "27120", "7717", "69286", "53360", "64435", "21451", "64562", "30126", "31566", "46422", "24017", "66772", "31321", "12014", "3205", "71447", "30051", "51836", "24507", "24312", "62260", "60708", "61169", "29380", "56549", "45459", "2251", "47882", "55765", "37729", "50714", "21079", "61619", "4540", "53026", "20417", "4938", "63497", "3145", "17788", "58707", "48636", "55825", "46073", "30997", "18374", "24792", "48276", "49320", "49583", "18488", "50555", "8496", "48557", "687", "4529", "29558", "35378", "61812", "15862", "5496", "41293", "70024", "6922", "41963", "69400", "32021", "26054", "48423", "54237", "66352", "44942", "47266", "10307", "9234", "41532", "2932", "595", "55747", "49889", "56336", "72933", "19734", "39151", "64467", "11764", "71415", "65309", "8499", "57572", "23612", "28726", "24846", "3782", "10387", "27883", "13720", "12537", "44214", "5039", "30938", "54185", "15152", "68431", "39232", "59684", "2205", "39008", "22761", "38712", "8020", "18750", "23369", "1511", "47816", "44425", "54421", "6690", "1789", "60266", "52691", "46354", "2102", "14034", "42870", "44489", "36564", "16886", "6357", "521", "58369", "19022", "69548", "43734", "53408", "6603", "5244", "49906", "68758", "53486", "17413", "32773", "28379", "2947", "30463", "627", "32880", "4881", "64702", "33181", "24622", "869", "63672", "39257", "55348", "42785", "53008", "54074", "23360", "19627", "59869", "58447", "46536", "23878", "11210", "53999", "14524", "4125", "35920", "5529", "26354", "39959", "19459", "16569", "53901", "17685", "16553", "10319", "56767", "58308", "68029", "26430", "66377", "13064", "8421", "47672", "4830", "62676", "42776", "30392", "18505", "68483", "9139", "58799", "42223", "21767", "19602", "33678", "44469", "37714", "7657", "39616", "60518", "40165", "2022", "8710", "26814", "61424", "68623", "41799", "24664", "72086", "65145", "54475", "72413", "4931", "16794", "68343", "26089", "54222", "68673", "70748", "33356", "8772", "59157", "39154", "17041", "54096", "63389", "51974", "44307", "27359", "26994", "41806", "69621", "54880", "27277", "17897", "876", "38271", "65335", "48705", "37300", "52611", "52879", "6399", "68355", "3824", "65158", "17931", "8978", "6141", "30677", "53384", "52610", "4859", "11392", "38092", "31824", "29385", "20754", "69148", "33406", "30958", "1499", "23695", "54579", "54805", "59191", "66313", "7052", "18971", "39774", "41262", "15874", "33999", "6947", "40020", "24968", "69668", "64576", "53671", "49417", "23103", "47667", "62092", "15561", "40014", "59204", "17471", "65841", "66212", "15267", "22582", "64391", "18100", "41352", "20000", "41228", "2196", "61549", "19689", "49980", "47544", "11461", "62566", "26235", "46631", "20919", "1563", "36975", "43455", "16771", "37335", "1755", "2617", "29821", "66768", "34771", "65835", "53426", "54167", "16871", "67469", "9371", "38649", "26790", "65565", "28504", "56705", "30972", "67901", "15156", "589", "29630", "38809", "58844", "70910", "25178", "41562", "28842", "9113", "58379", "32861", "41434", "35612", "65772", "62883", "39300", "40101", "25671", "59987", "40077", "607", "21988", "61112", "29654", "59114", "15907", "50896", "33843", "6613", "64083", "42069", "19801", "57557", "55146", "36241", "4066", "24058", "66733", "39273", "2786", "2237", "34577", "30725", "10587", "30614", "16377", "63624", "6904", "4300", "12469", "66899", "68713", "23172", "64786", "9499", "15165", "9498", "59683", "8338", "22529", "13384", "33338", "18241", "14508", "54429", "14440", "15815", "54105", "65159", "43295", "50353", "10672", "42493", "38222", "4436", "35083", "15746", "66736", "8414", "57250", "35013", "37263", "58944", "729", "43961", "17701", "27270", "393", "69348", "4382", "41358", "69479", "22102", "1604", "38693", "65270", "46443", "21755", "18162", "66883", "13750", "73033", "13968", "19903", "17977", "57193", "52705", "39423", "31761", "54626", "50981", "57738", "63584", "1962", "62170", "9832", "38324", "69167", "49193", "32714", "14021", "621", "53315", "38150", "50255", "43586", "44655", "59400", "66398", "68209", "52511", "16723", "4702", "30159", "54600", "68216", "46482", "10746", "71682", "21965", "63398", "14710", "26709", "10642", "70259", "34587", "3937", "5887", "26218", "36087", "29235", "32315", "68164", "34050", "52061", "35194", "67229", "26629", "30330", "10761", "2867", "30729", "37766", "46322", "28593", "64862", "42352", "71751", "32", "18222", "7291", "64742", "55223", "13634", "25374", "54479", "10950", "48698", "26387", "64858", "44124", "70867", "32419", "45527", "69715", "367", "67903", "39818", "1100", "54480", "20210", "71245", "55629", "45517", "23750", "30545", "61136", "3806", "40679", "51140", "26806", "17076", "64114", "19377", "49807", "57467", "17077", "56580", "45815", "48673", "22095", "58781", "30327", "42537", "56466", "58241", "3541", "37209", "8520", "22675", "2176", "50786", "37439", "48245", "38376", "56493", "37682", "56823", "13893", "12482", "27465", "46580", "39540", "28583", "16222", "32647", "37152", "54153", "50933", "39851", "62757", "13370", "67978", "32706", "38645", "39963", "38338", "20212", "13528", "66995", "43188", "24465", "15030", "45822", "46873", "7837", "51843", "19174", "1783", "42035", "20787", "8091", "12671", "55139", "43861", "40904", "39288", "33260", "12622", "41608", "27979", "50287", "63902", "21156", "69358", "6687", "70624", "33240", "10623", "66600", "7700", "931", "66972", "69420", "1469", "520", "22730", "53343", "16585", "11566", "6785", "70407", "44322", "25050", "49816", "39694", "53282", "18115", "53571", "13753", "47974", "32979", "67507", "71677", "12280", "33724", "14138", "44433", "46378", "70070", "34573", "5491", "23467", "49777", "50984", "47902", "62052", "3754", "52619", "34330", "6754", "64985", "68519", "13997", "42645", "50441", "72925", "2874", "51118", "29277", "58152", "61582", "65220", "27946", "3373", "34588", "38561", "38412", "2610", "69746", "49563", "51188", "49930", "43748", "58287", "67186", "6427", "1400", "5187", "55306", "905", "19266", "31316", "3564", "58506", "13309", "31060", "62962", "50696", "19569", "6656", "40892", "24862", "13651", "321", "41594", "14729", "69623", "46776", "15730", "56183", "18938", "32372", "17517", "35674", "45864", "68630", "49202", "8596", "3323", "34652", "13222", "57329", "14850", "8454", "41713", "54448", "29862", "63083", "41126", "63373", "50374", "46800", "34310", "3558", "10349", "27511", "7069", "51415", "72788", "19649", "63866", "37940", "34126", "25071", "57608", "13477", "2503", "55371", "66207", "45803", "10660", "39586", "18342", "1710", "17123", "28319", "12113", "66208", "3349", "10209", "17014", "15664", "49668", "8970", "18246", "57204", "32512", "1232", "58958", "54439", "23811", "61779", "39539", "68506", "3598", "57584", "26824", "33382", "10630", "41672", "25776", "44972", "57305", "1554", "47570", "50198", "33531", "9258", "17065", "1842", "14413", "32658", "7811", "51672", "62853", "20449", "42861", "47010", "52981", "72606", "9243", "39053", "39978", "33147", "45929", "66679", "65149", "6177", "30538", "49693", "35802", "27052", "61305", "55211", "30489", "66956", "71882", "66756", "29963", "61146", "41207", "61690", "66560", "70448", "8162", "379", "68600", "62171", "23997", "17470", "36489", "2219", "40352", "48168", "45091", "70402", "3098", "42475", "31848", "65700", "1022", "55748", "52472", "44678", "61859", "47008", "14730", "4458", "28538", "64123", "32473", "31693", "13877", "71257", "31368", "43113", "32039", "20253", "36592", "70952", "35794", "41244", "9224", "14520", "6865", "3672", "37961", "48860", "24260", "22323", "66301", "9707", "897", "9347", "9558", "1061", "27094", "53243", "64001", "18232", "7754", "58904", "38838", "47641", "25404", "51721", "41150", "63910", "13845", "50334", "72155", "61457", "52374", "63425", "25983", "29062", "57787", "36530", "23490", "6630", "56242", "10142", "71943", "17257", "41807", "38637", "6889", "165", "5651", "30143", "61673", "59279", "45601", "70255", "50082", "26281", "17522", "34303", "13512", "39095", "5325", "26441", "46046", "20830", "46969", "73199", "60511", "12030", "50086", "33852", "47788", "18405", "3896", "56280", "54210", "9894", "18555", "6410", "43389", "60975", "18745", "36026", "1512", "26081", "46106", "12887", "34521", "27004", "4665", "6987", "1804", "10267", "11851", "67865", "68987", "42679", "10938", "16506", "22913", "4316", "39722", "42625", "48000", "55770", "19736", "5958", "45492", "3450", "11171", "9599", "57677", "64388", "7870", "47483", "1798", "28474", "53477", "19749", "20821", "37683", "7547", "56523", "4764", "34144", "60565", "54765", "51631", "42346", "69281", "53386", "51591", "17326", "35450", "62761", "16958", "41732", "10964", "72224", "20451", "39308", "23560", "69016", "64569", "26911", "19647", "1952", "9103", "53263", "21100", "51585", "2975", "34940", "72969", "30909", "10596", "43396", "506", "14842", "15850", "18981", "58840", "53387", "58833", "9798", "10287", "10659", "23159", "25691", "25542", "13886", "35077", "4486", "49734", "31243", "26191", "26852", "456", "12140", "4492", "57759", "71426", "47781", "36374", "72534", "33659", "46025", "9503", "11038", "37848", "17226", "61122", "36767", "28242", "51357", "13509", "35571", "26861", "65507", "46104", "36586", "72458", "67511", "30962", "62917", "41389", "57111", "35062", "14276", "69304", "3748", "56363", "67292", "27131", "13126", "72737", "64822", "34163", "53788", "42666", "56919", "1724", "35915", "69285", "27412", "20139", "36719", "56320", "22226", "18077", "37914", "23188", "22657", "2796", "45180", "21491", "54058", "49604", "19206", "10026", "5794", "64227", "32106", "63076", "15269", "21538", "50137", "26760", "8024", "53222", "69539", "61279", "20721", "25566", "28337", "36743", "16596", "10951", "54760", "28469", "17947", "17625", "1284", "48273", "27726", "36539", "55054", "2045", "58448", "52621", "14956", "68381", "28443", "12180", "22506", "16743", "64072", "18872", "14938", "37049", "23576", "36762", "56204", "71680", "71117", "27029", "10330", "14333", "4877", "15998", "21291", "28625", "17709", "42832", "47845", "49097", "49143", "11523", "49174", "16347", "18882", "67941", "43042", "60351", "10724", "63165", "28062", "7494", "44231", "47930", "27908", "40510", "6033", "42680", "28180", "18831", "67215", "66022", "44924", "72480", "66303", "1752", "72658", "33508", "44767", "22", "11126", "19779", "30866", "6279", "58926", "10408", "52397", "38795", "53234", "71090", "31598", "15813", "63055", "22839", "20755", "25398", "40821", "35232", "38676", "23748", "40966", "54547", "67480", "7625", "11449", "37041", "22030", "71518", "58122", "69448", "39682", "18127", "44419", "37333", "32757", "42951", "49760", "39644", "11230", "66576", "49601", "31427", "55333", "13313", "38751", "6394", "72611", "65854", "46169", "17427", "63708", "4208", "53337", "44818", "58554", "37441", "46679", "42718", "38315", "51483", "35734", "2135", "59402", "28621", "41400", "17259", "28762", "51383", "42896", "66967", "24740", "17286", "28239", "6810", "65838", "9445", "35845", "6682", "2078", "17439", "24377", "19686", "49842", "57918", "30026", "20980", "14715", "54611", "10005", "50968", "44398", "16689", "20096", "64553", "30828", "58270", "8153", "33410", "5868", "3312", "69651", "68534", "69126", "40803", "23321", "46348", "58399", "6543", "7734", "62545", "13171", "35767", "58179", "50269", "71106", "51061", "19900", "37398", "12825", "42546", "23857", "22182", "9690", "6864", "37724", "5216", "72090", "41883", "12405", "61623", "63573", "36709", "6101", "55781", "48315", "4200", "41011", "65731", "50939", "8370", "29551", "16248", "54433", "13695", "56828", "53926", "42516", "48730", "63542", "41824", "63511", "33152", "24294", "28666", "2858", "44630", "27581", "69772", "54560", "4240", "34665", "13883", "19418", "18922", "37093", "60685", "69652", "1200", "20078", "69217", "11004", "43072", "55167", "8399", "21895", "35942", "69851", "68976", "31472", "68474", "72222", "24267", "16376", "21802", "63933", "55529", "30460", "53323", "70131", "25194", "43745", "56206", "55699", "63981", "14067", "15197", "8661", "71679", "39221", "12769", "34326", "12812", "56849", "53322", "35488", "43531", "12085", "37585", "68081", "26414", "9011", "43788", "61552", "13802", "35092", "58076", "59754", "18237", "8607", "69483", "60035", "70657", "62597", "41369", "66445", "54330", "61889", "31297", "24909", "32155", "17958", "26779", "32651", "12574", "68521", "11042", "51523", "4085", "538", "13403", "4808", "35246", "46363", "71457", "28290", "60803", "69300", "8996", "63431", "68119", "7642", "49115", "17988", "1176", "6243", "44763", "57022", "34124", "5378", "16450", "23017", "733", "11185", "30541", "61837", "43428", "33079", "23569", "65938", "25300", "34861", "24826", "21935", "52874", "30220", "10741", "14630", "7656", "44381", "67917", "51804", "11545", "59581", "45730", "51588", "20858", "68917", "65287", "21134", "66357", "16989", "12762", "56170", "59392", "19388", "41446", "24095", "51953", "46755", "70515", "37763", "65332", "70861", "57273", "38319", "25010", "48198", "33550", "43412", "6744", "3094", "63271", "52098", "750", "55512", "18615", "25875", "16706", "34911", "24216", "50375", "36754", "3718", "33006", "37831", "24197", "52885", "70884", "17545", "388", "37778", "7618", "48474", "67193", "65035", "45784", "29732", "26661", "13283", "27561", "73171", "31708", "17328", "4517", "32560", "57822", "64258", "62695", "17133", "16662", "3081", "63296", "51162", "31079", "53786", "53954", "58477", "5962", "23891", "1915", "56327", "43736", "39897", "39633", "68305", "60390", "10022", "11489", "40288", "35949", "2010", "31940", "8435", "55572", "61293", "45370", "43080", "36079", "22502", "29264", "4500", "27119", "55476", "27989", "68505", "72118", "38182", "60395", "139", "57284", "65213", "20391", "15723", "40817", "64703", "23999", "63842", "3766", "24370", "31113", "26928", "30479", "17206", "37564", "68428", "57613", "47957", "55534", "48998", "28785", "40908", "35589", "70531", "25661", "47326", "7438", "23101", "45787", "25055", "42576", "53806", "48591", "40560", "40910", "64969", "11286", "30174", "9325", "17056", "32263", "27121", "46093", "1606", "12067", "19993", "63294", "15141", "55594", "34835", "58643", "4049", "36496", "58453", "44286", "5741", "62473", "51826", "45916", "36114", "68736", "26813", "57029", "23393", "40107", "24229", "963", "34826", "34208", "38514", "72456", "26962", "15412", "54748", "25411", "72728", "53433", "30666", "44023", "24219", "19832", "7862", "36746", "67086", "27481", "12857", "17674", "60019", "43259", "6517", "4409", "21675", "64108", "62329", "4758", "69892", "33059", "19657", "26902", "27829", "69581", "33760", "58264", "54993", "26436", "52859", "4852", "72799", "40252", "55985", "11090", "12984", "54323", "55389", "35032", "66157", "45610", "25077", "67645", "62410", "43167", "21345", "9548", "16002", "59946", "387", "37713", "33167", "6997", "19253", "42585", "48928", "48908", "72122", "49016", "49648", "69547", "27205", "10154", "38230", "29425", "66101", "64528", "32974", "13680", "29058", "40725", "52424", "40192", "25976", "50520", "69301", "67910", "34906", "3830", "31862", "34159", "37457", "37130", "31810", "57459", "38403", "20876", "39998", "33242", "67263", "68958", "46575", "33623", "10851", "66716", "61385", "34689", "34266", "51890", "11927", "37894", "35597", "27928", "22404", "20273", "10634", "5626", "26079", "49908", "31949", "8309", "614", "22307", "60557", "20311", "67169", "54496", "44319", "62838", "12927", "58363", "8893", "68311", "35384", "49020", "25000", "13993", "55186", "30456", "69209", "61010", "52618", "43650", "73108", "17758", "52458", "34105", "62932", "52572", "24659", "42551", "34741", "66298", "50285", "68955", "53769", "15783", "40293", "67159", "54416", "65827", "53993", "15583", "16721", "18822", "19461", "63495", "40425", "51159", "66543", "2464", "41633", "32505", "22103", "26529", "45081", "42006", "66686", "27177", "68088", "12956", "8624", "72816", "16957", "9295", "71034", "55995", "18768", "2561", "55532", "45716", "19874", "64161", "67746", "68979", "66392", "65913", "69637", "28893", "70302", "10628", "7227", "45396", "28307", "46494", "24270", "68165", "60858", "36020", "58080", "26511", "54492", "16059", "3446", "10429", "24353", "10113", "68360", "27351", "55272", "71065", "65928", "703", "45373", "44122", "46608", "48691", "22944", "66415", "16543", "17592", "18925", "32931", "15982", "63665", "33803", "47311", "33108", "35733", "57241", "5946", "3467", "68392", "32312", "30386", "34452", "28453", "27975", "26274", "7982", "46932", "25967", "29099", "16172", "35019", "47617", "17911", "71586", "8357", "11496", "36749", "16602", "36680", "26715", "7833", "13612", "67870", "48747", "45277", "16205", "62852", "13336", "54844", "29330", "56047", "68895", "21936", "69089", "11120", "66222", "52887", "29016", "39410", "38332", "14937", "7886", "50986", "16970", "3271", "44568", "587", "70887", "14028", "40090", "16006", "55092", "22820", "1545", "51479", "26781", "28441", "36662", "40798", "68261", "24926", "35285", "3038", "43270", "41626", "26175", "13007", "52320", "62144", "20264", "48763", "59007", "63791", "71405", "35182", "55274", "59879", "62412", "37717", "72815", "23405", "29820", "67019", "72503", "14198", "49891", "24379", "43387", "24553", "46961", "65818", "72176", "18313", "5863", "59047", "48649", "35522", "63029", "43445", "28429", "30946", "8304", "18417", "45838", "41777", "45721", "13317", "13254", "28608", "41639", "72778", "58742", "25479", "27720", "72395", "53879", "22568", "39375", "3915", "1642", "18121", "66990", "508", "68342", "53149", "68143", "63295", "62914", "65427", "17583", "19670", "21242", "42788", "31137", "17063", "39315", "56922", "71873", "599", "2608", "46506", "47837", "11436", "26688", "53270", "22298", "128", "35565", "5321", "8578", "34037", "21634", "9725", "53543", "47155", "16010", "51678", "50884", "6085", "9796", "16677", "16795", "16965", "55071", "66510", "16380", "21467", "21346", "71233", "48775", "3380", "9879", "47734", "20808", "28507", "72316", "46438", "63457", "45891", "45983", "31035", "70136", "31451", "70166", "14141", "16312", "71087", "16018", "70787", "26415", "40887", "46959", "12549", "28912", "36806", "3788", "39559", "9449", "20294", "63858", "38849", "55441", "9164", "15625", "21008", "21408", "46267", "62435", "34987", "32504", "39004", "14757", "44646", "31899", "36570", "8699", "71180", "25484", "29505", "61222", "31238", "47996", "2798", "5976", "11459", "5659", "31485", "28327", "46824", "60881", "10009", "17281", "73216", "1433", "30347", "8271", "3719", "26341", "29866", "55007", "43420", "42398", "22060", "60420", "50342", "40699", "20717", "42670", "65450", "53079", "37868", "28535", "49371", "26723", "39960", "66045", "35098", "2695", "27376", "66403", "10868", "70083", "27522", "10673", "59614", "57116", "21910", "17682", "9254", "3662", "16697", "11730", "54749", "60241", "2623", "13584", "44841", "23941", "28297", "37863", "20031", "1049", "273", "56973", "70103", "18202", "38350", "9785", "5897", "21221", "23223", "55626", "5024", "10610", "57524", "21757", "12050", "43122", "35099", "51083", "25406", "54498", "72867", "24394", "55967", "24522", "10619", "6402", "36160", "20753", "43611", "51993", "36439", "17428", "11190", "8910", "38394", "9651", "59009", "22128", "27305", "17278", "37522", "42868", "48598", "21419", "18923", "53642", "54576", "54777", "58710", "45589", "13502", "63696", "66803", "49512", "4423", "26011", "37935", "12340", "39169", "12195", "26096", "5132", "66557", "41605", "56449", "12505", "24346", "39811", "17050", "34977", "11605", "1721", "1751", "71783", "48280", "31346", "69656", "12400", "49813", "29473", "73078", "37594", "53104", "41977", "14397", "16390", "4926", "5006", "62505", "31833", "59027", "4457", "63471", "14927", "32441", "32542", "55881", "69984", "50042", "8118", "16663", "49940", "72034", "37019", "1384", "50238", "58716", "71546", "28795", "34904", "28046", "55857", "73218", "16331", "8106", "31400", "38881", "20999", "2285", "56019", "37654", "30722", "1289", "2931", "22524", "31336", "40884", "5635", "70568", "69905", "21312", "26695", "49723", "48572", "50356", "44129", "63592", "35149", "68778", "37249", "47675", "64876", "26665", "28314", "62510", "7075", "19372", "55523", "63520", "10434", "47252", "42404", "21923", "33135", "13905", "41275", "71828", "28551", "16310", "45503", "271", "63693", "27745", "14691", "26801", "47685", "63634", "46026", "46722", "46172", "3660", "20119", "16897", "14185", "4496", "66795", "68193", "32305", "25281", "13428", "19941", "21519", "15996", "35921", "61248", "59207", "51870", "38582", "49803", "5608", "47152", "51991", "46935", "40606", "35796", "18898", "582", "24066", "41600", "73007", "53895", "10891", "44390", "19793", "33209", "58023", "66913", "56958", "72506", "48574", "18461", "70109", "58365", "20555", "44503", "68352", "40762", "8566", "49517", "32226", "30452", "51296", "6710", "56088", "6951", "3994", "32905", "53230", "28181", "37782", "13415", "13915", "12481", "13249", "34300", "17106", "13070", "14101", "29233", "59421", "12706", "50452", "61108", "39419", "71402", "70561", "14724", "2302", "48121", "47750", "60085", "38879", "65554", "51250", "69544", "36204", "46528", "69826", "42344", "16565", "1533", "9848", "65873", "50671", "17374", "52982", "10594", "27689", "57992", "34955", "13571", "66184", "33499", "67876", "28456", "57707", "62245", "19289", "68981", "68835", "37173", "49181", "32456", "68335", "39005", "71032", "53832", "59546", "40167", "27466", "39650", "54488", "70712", "10042", "17030", "66406", "51598", "51967", "17138", "3206", "28359", "18610", "22959", "46487", "16449", "41364", "67968", "4959", "19568", "59054", "1662", "57958", "72150", "12515", "220", "27092", "16237", "18854", "171", "2448", "2210", "45377", "41606", "13169", "54349", "67743", "73051", "10853", "7501", "18512", "46596", "47664", "52520", "21064", "42328", "51677", "40937", "67500", "415", "63674", "68234", "972", "40387", "49585", "10823", "61992", "17025", "21421", "32515", "22912", "19260", "54637", "16986", "37458", "58006", "72228", "36444", "72149", "48486", "62908", "62084", "5335", "24891", "15974", "30888", "3517", "24437", "49555", "58295", "10998", "15295", "14483", "56918", "61402", "50923", "38390", "42095", "48404", "60224", "64491", "69911", "66183", "20813", "52807", "22731", "48475", "30854", "30385", "52097", "3060", "42504", "48639", "11641", "4019", "71356", "39458", "27574", "25276", "71620", "41206", "22450", "66004", "67495", "69037", "61459", "39544", "59685", "57142", "7741", "61049", "55766", "43194", "57567", "26594", "56458", "47987", "56723", "65735", "21593", "10822", "62411", "19067", "45256", "54759", "57797", "45196", "1183", "19697", "36196", "50816", "14686", "708", "43621", "47144", "56256", "45141", "29521", "49264", "27837", "16370", "19064", "59928", "13761", "34336", "60855", "32277", "55753", "10295", "11456", "4035", "10220", "145", "22705", "68345", "25476", "37620", "8570", "31560", "69801", "62102", "11399", "68787", "28003", "45032", "25787", "42393", "55235", "180", "38869", "9175", "64698", "39176", "49592", "5178", "49044", "49056", "21466", "5147", "37901", "38349", "6198", "58514", "30236", "37028", "40152", "19433", "948", "64870", "10412", "874", "22429", "30599", "32095", "48586", "16911", "23842", "67688", "51430", "26918", "31484", "1813", "33668", "5916", "27954", "58731", "19986", "39311", "7537", "45660", "55552", "38367", "70194", "52797", "24279", "64204", "40787", "1017", "67083", "29027", "3189", "29076", "25605", "38115", "8776", "30988", "40722", "24942", "36733", "32563", "225", "54215", "56299", "68731", "50347", "25765", "49212", "683", "2801", "23094", "46419", "4159", "17081", "59301", "50906", "61154", "63481", "10176", "18712", "71965", "735", "67097", "50535", "30024", "42215", "39331", "40977", "71637", "26958", "42165", "28554", "54364", "41889", "35161", "14698", "6042", "54021", "28127", "68589", "46344", "24908", "66946", "68367", "37494", "26830", "58769", "45946", "23347", "51365", "41292", "17793", "12097", "29589", "70193", "53798", "52279", "22037", "8023", "67930", "44376", "13158", "5966", "65292", "34067", "67527", "34752", "6137", "69337", "59565", "23668", "51692", "6425", "6757", "6196", "5924", "16218", "65744", "34108", "3592", "50231", "6232", "25907", "19473", "33752", "21386", "19211", "4888", "46889", "68923", "72528", "17967", "69976", "32233", "15247", "9799", "42299", "16571", "59317", "73052", "8132", "28123", "26334", "32677", "18739", "47627", "4832", "13507", "56358", "43755", "49506", "2178", "18159", "25540", "5286", "4304", "33671", "45656", "23964", "35528", "46450", "4388", "14453", "11999", "8202", "65734", "26198", "71693", "51203", "54089", "31550", "1408", "54831", "16461", "19004", "65768", "14401", "56496", "21019", "8704", "53415", "21219", "7000", "34412", "71543", "21530", "22744", "26731", "12082", "32916", "3826", "8442", "34405", "47101", "53373", "69622", "6200", "62185", "1008", "50197", "11595", "6882", "54521", "56714", "51275", "5672", "48930", "50025", "37686", "72047", "63888", "43081", "50368", "56436", "64934", "18211", "19540", "28367", "42633", "70424", "70634", "39409", "1305", "35228", "28782", "40235", "62310", "41005", "412", "14597", "34998", "8899", "65116", "68042", "44262", "60042", "50177", "25997", "58821", "64309", "67463", "36978", "35644", "3152", "51358", "41721", "46178", "53049", "64791", "3086", "64666", "25803", "6351", "27811", "45901", "6972", "33346", "30571", "63444", "6761", "23979", "15931", "41629", "56724", "2320", "47367", "41471", "50945", "2175", "12425", "63065", "3127", "1211", "38485", "67372", "35624", "44073", "39687", "6830", "58217", "49599", "59128", "9122", "7670", "15252", "2184", "260", "19799", "52050", "47076", "13145", "39866", "51920", "64670", "50404", "66950", "32291", "54348", "67928", "36323", "51533", "24927", "68077", "28287", "16083", "67976", "21438", "31558", "32275", "3298", "32687", "46643", "20343", "38614", "43741", "57181", "19914", "27781", "33319", "9910", "27997", "58856", "40240", "52764", "26831", "51225", "23383", "13513", "45713", "37875", "20796", "10411", "20631", "39950", "15852", "52414", "37402", "9014", "52924", "28269", "59451", "10825", "63646", "60859", "71261", "56270", "65338", "66878", "13709", "35016", "2635", "3198", "20562", "50220", "1675", "21061", "62789", "42604", "69418", "30556", "57369", "60752", "19463", "24374", "15672", "38683", "65075", "51461", "21807", "25583", "55481", "19061", "60755", "57650", "40392", "22331", "47007", "57571", "36517", "9468", "48972", "13322", "16578", "49919", "20742", "11365", "9955", "16785", "62455", "15734", "17011", "11671", "32807", "26385", "16400", "31664", "17307", "49064", "33967", "2015", "32574", "19763", "8988", "52154", "60109", "57259", "46400", "68724", "46735", "45127", "10531", "72965", "17227", "41211", "49180", "27176", "29378", "62952", "59031", "1129", "51356", "43409", "55321", "49423", "39304", "60553", "50214", "70344", "60067", "12617", "15602", "3699", "35447", "24568", "35275", "26051", "34097", "63554", "13832", "7785", "58282", "5265", "29247", "24052", "32011", "6106", "22799", "37338", "27969", "9067", "1648", "51182", "55309", "3321", "7604", "21800", "51385", "21427", "35777", "39235", "68647", "32991", "33616", "57473", "13965", "62945", "6376", "72583", "53986", "41248", "29100", "57445", "12327", "37820", "45800", "39164", "55771", "48592", "6104", "8191", "58646", "3807", "29196", "58605", "45906", "53747", "22166", "8069", "48710", "49670", "9860", "51862", "59098", "31290", "7895", "8878", "63720", "12747", "58587", "49857", "15680", "39133", "45563", "39570", "43540", "58345", "15110", "8803", "49006", "7172", "35358", "69925", "52890", "42294", "13811", "40824", "69108", "53496", "17663", "50210", "62785", "22459", "61156", "53594", "54645", "58624", "48301", "45101", "19217", "29708", "3795", "24466", "62864", "13029", "5658", "71696", "9839", "54111", "4981", "41875", "28091", "47674", "55852", "55550", "12820", "17120", "1570", "18364", "30701", "56644", "65457", "16570", "55390", "66271", "56768", "50446", "44072", "24015", "47519", "55773", "14327", "35200", "27701", "2066", "924", "20020", "18295", "24122", "7322", "13708", "38912", "31562", "63385", "62982", "71099", "59409", "25821", "14389", "53781", "28942", "16304", "4724", "32497", "48327", "24078", "57436", "4093", "16437", "25529", "23688", "11169", "28490", "29653", "56776", "29770", "14423", "6201", "3235", "57595", "52408", "10639", "2558", "57731", "28404", "56650", "19694", "22638", "37596", "11982", "33418", "63341", "38714", "52307", "48440", "20352", "37962", "39876", "6472", "35660", "31271", "66370", "27366", "57314", "70290", "56632", "3739", "60549", "69614", "72916", "46376", "48803", "25222", "56086", "3842", "42024", "1849", "15578", "3675", "37853", "32427", "21639", "72883", "14907", "29008", "30312", "25954", "29342", "26531", "69903", "68252", "3419", "25321", "57019", "71773", "25724", "2896", "28251", "21416", "19797", "64066", "57778", "16809", "22953", "43003", "5061", "60187", "12847", "33220", "18281", "63448", "10361", "22651", "28104", "51282", "2412", "48740", "30568", "23125", "6348", "3776", "52915", "25898", "69047", "47715", "69249", "4993", "39466", "60891", "52368", "17454", "50763", "17084", "38062", "20151", "63938", "70508", "19815", "47866", "72114", "15143", "2575", "55136", "955", "27587", "37075", "71422", "39228", "56103", "16230", "56053", "64337", "2253", "70438", "72548", "6135", "35363", "10064", "44130", "72902", "26456", "53745", "45080", "72752", "34781", "51740", "4435", "36801", "72040", "28676", "31386", "72229", "29175", "33171", "47048", "17992", "44960", "13976", "14305", "22425", "53098", "45040", "64306", "36259", "55871", "5359", "1921", "39527", "71318", "54956", "44183", "32136", "69832", "38914", "30886", "13785", "34497", "67699", "11816", "50894", "8753", "52530", "56488", "69644", "44213", "55498", "30578", "39692", "45625", "54512", "34623", "56230", "34288", "14886", "28165", "66065", "9915", "29868", "36937", "5784", "51191", "33278", "55625", "53114", "13153", "29953", "14000", "7206", "37635", "41", "40764", "63069", "66270", "5035", "62518", "8299", "5459", "46519", "57928", "45302", "21758", "58121", "7258", "21572", "6453", "11376", "27736", "66742", "64387", "39252", "2192", "53357", "39340", "18703", "3449", "6812", "62512", "16375", "24456", "8540", "39402", "33935", "52054", "21998", "40916", "30782", "61705", "66066", "69327", "50517", "27348", "18311", "52443", "8977", "62049", "51382", "52171", "35814", "24440", "860", "69203", "3557", "57570", "40496", "36271", "58259", "19973", "50300", "10564", "5646", "12908", "11115", "35476", "22248", "54150", "65724", "754", "14171", "18597", "49957", "8958", "18639", "55502", "23547", "47040", "49419", "39048", "72407", "69982", "68052", "5603", "9546", "36497", "36873", "4094", "18738", "19553", "18550", "4139", "64356", "45761", "48303", "17889", "27847", "19472", "55051", "16612", "66704", "13671", "21127", "49763", "21265", "49548", "2275", "8146", "41345", "42950", "28662", "41967", "36142", "53462", "18339", "18435", "7299", "43938", "18786", "68628", "43333", "23577", "63458", "5596", "6561", "2235", "33878", "47738", "16253", "17523", "59930", "49330", "55987", "48617", "36544", "737", "31762", "18916", "58810", "33457", "72334", "13466", "45478", "54676", "10", "16587", "49575", "44440", "28299", "37157", "53743", "17274", "36508", "52060", "24614", "68851", "56621", "3256", "8550", "11751", "14163", "64128", "59458", "59860", "40157", "54544", "24393", "56043", "45843", "6735", "23327", "59333", "4007", "52758", "32451", "54071", "32366", "58318", "36947", "21590", "37139", "3948", "347", "50442", "13462", "63666", "67569", "31418", "15001", "7552", "56480", "2976", "24071", "68670", "18719", "53598", "7592", "27411", "56354", "28171", "3738", "39990", "55491", "65346", "217", "52623", "65207", "2534", "2304", "50263", "35693", "67344", "5847", "48172", "66889", "65296", "46844", "31502", "1608", "51685", "64645", "24316", "21784", "8944", "72293", "49814", "29702", "40047", "14702", "40329", "39825", "6011", "30207", "43951", "10872", "16487", "3921", "61047", "14420", "18545", "26160", "52121", "10488", "31432", "4661", "20353", "40391", "48320", "25719", "65765", "41319", "29454", "37034", "21168", "36122", "47414", "67842", "23653", "52351", "61939", "45963", "13013", "62935", "34890", "16379", "44326", "32744", "15296", "978", "40462", "17185", "26027", "41927", "39612", "67310", "17525", "37187", "38673", "65348", "28147", "41988", "31459", "49309", "63244", "42642", "67441", "22526", "63522", "52575", "50135", "29916", "70932", "70673", "37259", "27309", "25202", "53388", "4594", "49396", "572", "26094", "21636", "37325", "21773", "9897", "58814", "14899", "58249", "48347", "69241", "16522", "24125", "10745", "39778", "40005", "54497", "20663", "37158", "11983", "16671", "11159", "66136", "35399", "65122", "16716", "6931", "56822", "44342", "67369", "60570", "22822", "43950", "56058", "22633", "2698", "8754", "4263", "17844", "46940", "43593", "67296", "57368", "24952", "67506", "11785", "68845", "27266", "55555", "36085", "23516", "28871", "32770", "47397", "42572", "39966", "51671", "27686", "1744", "26416", "63684", "65962", "52540", "25441", "64487", "53302", "10788", "67622", "55451", "51954", "43699", "69717", "22167", "21712", "70522", "45170", "24978", "64961", "52009", "72206", "15006", "55300", "51763", "61296", "23993", "46305", "44734", "22207", "50396", "3076", "54335", "47509", "18502", "21350", "38870", "8791", "55585", "31033", "32276", "15818", "62742", "60660", "64575", "58255", "70804", "34529", "3504", "45023", "20674", "43983", "46030", "56933", "72402", "56568", "70178", "57735", "48873", "6915", "19878", "40678", "63168", "59514", "11118", "63061", "2660", "9211", "36430", "11773", "53344", "40716", "51054", "39804", "69783", "50284", "35508", "61505", "51713", "45883", "36631", "66660", "17060", "6331", "58917", "14437", "34269", "40182", "68641", "43490", "34750", "69038", "52380", "56931", "6124", "29167", "57018", "65710", "51597", "20507", "63449", "45732", "40481", "62199", "48834", "9777", "28024", "10096", "65778", "38452", "69745", "65516", "32596", "45902", "12939", "15571", "23409", "16027", "39624", "55489", "34553", "17208", "7166", "4120", "38856", "55524", "15495", "30812", "4843", "61543", "39898", "31023", "69915", "30645", "8932", "68171", "66361", "19150", "70228", "32304", "52682", "38366", "123", "72952", "5886", "10900", "10558", "63313", "65081", "5671", "51419", "63249", "64880", "50590", "44167", "43640", "35121", "46915", "50321", "48352", "28144", "32655", "15923", "981", "2934", "28981", "11158", "13578", "1399", "65582", "54943", "9180", "69495", "838", "49095", "3497", "54472", "40559", "30025", "13531", "58364", "71320", "44344", "50770", "58566", "44229", "48143", "26615", "36236", "53584", "12391", "6933", "51590", "70697", "50865", "56972", "40869", "24709", "69194", "59152", "25203", "6516", "72570", "5757", "39226", "18050", "29333", "65157", "20829", "73071", "68749", "5634", "31067", "70464", "43203", "30209", "2104", "29710", "53941", "32181", "35344", "47914", "20956", "60595", "21222", "14642", "53803", "67602", "55977", "16338", "12071", "37784", "72569", "8594", "56034", "44331", "24668", "47056", "3641", "49430", "26393", "55204", "22901", "53452", "11734", "17252", "36248", "11319", "10273", "20240", "51480", "37108", "25608", "37031", "64203", "17593", "44438", "51321", "59915", "6765", "19502", "41113", "18983", "61227", "37740", "23661", "31351", "61687", "14204", "378", "23992", "41625", "66836", "9220", "69198", "44235", "66815", "10195", "58125", "68783", "22727", "26201", "11446", "68500", "15451", "12317", "53209", "53375", "46842", "18830", "38174", "421", "29481", "49144", "44749", "3678", "47282", "2933", "32478", "15301", "48306", "25245", "51669", "50187", "62241", "23731", "49606", "62679", "5249", "20167", "3623", "57335", "30536", "4460", "66918", "46583", "67388", "48963", "27018", "71312", "39640", "56961", "23224", "43038", "56009", "68879", "36536", "63716", "68156", "55042", "45290", "53813", "52001", "10109", "61551", "66534", "25620", "38407", "37570", "30000", "58951", "62714", "53014", "49249", "3126", "49878", "42972", "55519", "7956", "59955", "69970", "24230", "54961", "29695", "70950", "65440", "15187", "37657", "55407", "65635", "56360", "41251", "5645", "20383", "6954", "45074", "51465", "48165", "21960", "22175", "39917", "24703", "17416", "55783", "63435", "45825", "62156", "71143", "12131", "63363", "20511", "25334", "43965", "29712", "59188", "44522", "14127", "6068", "26929", "43426", "9255", "44580", "48124", "14288", "4685", "72250", "66221", "18436", "68752", "1381", "14680", "23208", "64181", "58871", "140", "12817", "71314", "12278", "35817", "64256", "37254", "13556", "32886", "48311", "68887", "44166", "1065", "20375", "64027", "64659", "14874", "33085", "2249", "19413", "12813", "48147", "58912", "48850", "52937", "47301", "48089", "42107", "8556", "56788", "35397", "46221", "67233", "43369", "58784", "61880", "36143", "56181", "67532", "67689", "5546", "22487", "32249", "7256", "4884", "62844", "60856", "65129", "38027", "52569", "58386", "42735", "5332", "24160", "57692", "20948", "12967", "15945", "38725", "36445", "54884", "34492", "38291", "6448", "46374", "19072", "62167", "24203", "8527", "67512", "40760", "4855", "24748", "51774", "3545", "21536", "35045", "64465", "19244", "65317", "16698", "22193", "4697", "44834", "75", "35607", "11379", "14592", "42349", "36636", "13419", "26610", "10579", "69626", "48794", "2396", "52182", "66606", "6479", "64687", "47735", "24537", "16912", "57903", "60413", "27974", "8843", "10470", "12058", "37904", "70202", "11317", "50435", "69686", "43065", "65481", "30530", "34814", "40324", "38385", "68921", "66033", "5955", "50468", "34191", "15942", "13790", "26725", "60663", "69931", "65785", "20262", "8449", "16850", "12610", "57343", "69529", "25211", "41677", "22259", "66914", "43535", "52649", "1541", "13688", "62035", "3756", "22902", "28831", "30449", "15816", "4338", "38050", "38434", "34728", "72318", "51773", "47146", "60177", "3834", "16178", "2751", "52190", "57367", "49511", "52134", "34789", "3073", "19887", "64059", "17849", "5434", "29851", "27482", "23825", "68213", "43681", "69716", "44833", "15925", "9101", "3601", "8369", "53128", "10399", "31224", "55376", "65632", "8530", "25689", "15289", "25342", "46864", "58803", "62821", "49091", "8423", "10527", "31686", "17013", "43467", "63822", "34637", "59803", "35136", "15206", "71796", "42940", "21528", "25036", "21660", "55623", "23350", "22416", "29457", "13421", "21383", "32221", "64973", "22698", "33273", "61113", "22320", "63529", "8739", "11647", "17590", "3841", "68056", "27584", "71715", "32367", "16302", "42621", "14972", "57544", "61371", "55290", "36015", "55255", "43598", "29163", "28052", "3684", "51561", "51747", "55328", "34442", "15457", "21977", "40025", "67718", "59620", "6653", "36860", "45390", "50485", "72424", "19155", "30368", "21246", "1210", "43852", "40054", "30971", "25809", "38815", "11368", "25692", "45177", "70427", "40031", "39184", "12074", "19393", "608", "7736", "47942", "73219", "37707", "34119", "7013", "43570", "21271", "60273", "41561", "71653", "31471", "21556", "64902", "65588", "43047", "18260", "35503", "62150", "65", "36635", "46294", "53770", "1281", "41429", "1702", "17401", "51261", "31840", "3214", "32771", "25714", "71579", "5337", "64532", "64241", "5754", "72554", "68485", "37430", "24546", "43516", "40628", "61760", "26171", "24983", "69111", "61641", "34348", "25038", "60144", "8761", "32091", "42774", "72743", "17329", "54666", "15493", "53870", "69346", "59996", "49208", "42378", "11630", "40416", "64572", "52164", "14818", "41711", "49167", "52202", "30130", "58314", "7313", "36157", "38871", "9280", "35021", "53522", "23822", "63454", "69298", "24725", "10617", "57982", "30913", "21472", "55905", "33143", "70990", "65531", "48049", "26528", "44884", "26433", "21893", "13848", "5950", "26821", "64047", "13792", "48094", "26543", "26159", "662", "64932", "32395", "58590", "67370", "43468", "10647", "15069", "26636", "73121", "2435", "4020", "30447", "30311", "9290", "11756", "53499", "41448", "19539", "46407", "20385", "15473", "30680", "14173", "29012", "62334", "14348", "45182", "72872", "4452", "30678", "12881", "27740", "43668", "43715", "2520", "29356", "30801", "34239", "13714", "48923", "7549", "43556", "39313", "52176", "5627", "49671", "20241", "60930", "46916", "10930", "30579", "36684", "13464", "29856", "39752", "36292", "6206", "22297", "61063", "38648", "9345", "10435", "2912", "55357", "28616", "11373", "64886", "10733", "26266", "31331", "16504", "33855", "10575", "22550", "36258", "25941", "4210", "23916", "42231", "71812", "2704", "40195", "48340", "19583", "49931", "72208", "19404", "33970", "5145", "58216", "29873", "56136", "26555", "36367", "55444", "18842", "59084", "21299", "55859", "28724", "59411", "1543", "57307", "16444", "54833", "11030", "29713", "8470", "59513", "62692", "69810", "56722", "45294", "36437", "69866", "10345", "15875", "1069", "42307", "18052", "43198", "48120", "47318", "32864", "13879", "66649", "20355", "33639", "69899", "7578", "28804", "3326", "32035", "38318", "38758", "48744", "55619", "72751", "46765", "62957", "20032", "9778", "20519", "43230", "37956", "30724", "4301", "44544", "30410", "31576", "41110", "50584", "21976", "43676", "28519", "153", "4582", "59711", "15113", "38822", "42406", "29861", "14975", "25301", "9773", "10921", "38013", "65602", "69616", "36113", "44354", "16840", "54850", "35677", "14074", "45441", "53056", "56605", "48472", "35608", "21956", "17288", "7855", "14298", "69705", "31180", "24758", "39775", "7136", "15507", "35699", "33736", "6535", "571", "12355", "58859", "26680", "1593", "13673", "43911", "70926", "72683", "48082", "54481", "59493", "42497", "8750", "67541", "65836", "44473", "12743", "11655", "71744", "22991", "28059", "6545", "37294", "1966", "36702", "10707", "9682", "70600", "17648", "50064", "16636", "29467", "23423", "73236", "38065", "39889", "43737", "34017", "68650", "53876", "49213", "26452", "3883", "29568", "47655", "26849", "38025", "60028", "38935", "20713", "67630", "45195", "10879", "63923", "40661", "1356", "61021", "64444", "12546", "60802", "61731", "21964", "1914", "62428", "39919", "35895", "67894", "1601", "6850", "59468", "24993", "35869", "54770", "14663", "42728", "39149", "30437", "46527", "57246", "59520", "4629", "56066", "5848", "6412", "71745", "34596", "22870", "11500", "41716", "29624", "29557", "39723", "33162", "61048", "63548", "60303", "10003", "2700", "18886", "68969", "47705", "65532", "32817", "6748", "803", "35815", "10889", "39111", "40160", "64262", "70412", "27870", "28980", "31794", "6280", "27321", "15839", "59697", "66328", "30873", "13704", "1109", "46825", "28447", "64223", "31163", "16484", "11045", "34160", "66152", "20944", "54015", "69263", "71866", "60736", "63215", "56683", "27049", "4004", "62138", "34519", "56455", "39814", "48035", "1426", "37804", "73244", "12423", "15754", "40393", "23606", "42133", "15624", "13737", "36071", "70072", "29051", "10170", "51", "67450", "36061", "55365", "43949", "32017", "19523", "35901", "51088", "64058", "31470", "33810", "49976", "55953", "8692", "37528", "351", "26303", "46123", "31447", "4312", "58458", "32520", "66894", "70841", "21599", "57785", "58869", "60155", "48407", "26488", "56194", "10942", "8120", "53196", "33859", "6407", "53940", "56597", "3136", "34188", "48504", "58940", "60410", "42754", "69569", "12089", "49442", "27972", "12611", "32561", "40253", "50868", "20792", "13667", "51664", "58472", "13496", "5913", "67325", "72365", "10130", "72898", "35051", "52816", "71891", "46072", "46185", "66478", "59850", "42067", "62125", "45160", "46637", "33610", "6725", "49052", "33355", "63685", "8406", "68760", "41996", "72836", "9200", "42906", "27993", "31063", "48364", "57909", "9745", "69253", "41178", "34878", "1691", "62313", "68301", "26003", "13195", "15591", "31624", "11708", "26659", "53704", "29153", "52489", "14895", "49997", "41191", "29510", "18609", "57077", "63499", "65478", "27910", "21013", "35864", "71872", "27650", "53624", "41664", "59125", "19821", "29032", "41147", "59989", "53488", "36509", "61129", "58311", "67849", "70813", "55347", "16223", "15331", "46100", "8498", "43622", "1235", "10422", "65860", "9397", "9348", "440", "12096", "27763", "10787", "22601", "9747", "55221", "57060", "13202", "59322", "44386", "69075", "72328", "31237", "72298", "65167", "16404", "31692", "55003", "61913", "11394", "60101", "31962", "21682", "60171", "42123", "44306", "12455", "37519", "62112", "7199", "39479", "2703", "19085", "65656", "14360", "24360", "43520", "57952", "13429", "8240", "52568", "35080", "6727", "45814", "46230", "57612", "44156", "54790", "29154", "52222", "59295", "41722", "1764", "20496", "58404", "22094", "71614", "63681", "34150", "19442", "13752", "6506", "24347", "38627", "62825", "22871", "54395", "52716", "66232", "72017", "44637", "41753", "43437", "35808", "68244", "42966", "4918", "41175", "13777", "67679", "28364", "13924", "36231", "20153", "51311", "56314", "48615", "5498", "19833", "55088", "62549", "34025", "28792", "37947", "69192", "27559", "31707", "24483", "47801", "32581", "54897", "64295", "13197", "16647", "20644", "3014", "7864", "63110", "9988", "55001", "48772", "33593", "15288", "68767", "20407", "52990", "57869", "38219", "64618", "63667", "67368", "61904", "70414", "25891", "20013", "55959", "19845", "63008", "26627", "23305", "3284", "24794", "14439", "13596", "1416", "26295", "60079", "24753", "69114", "18370", "38169", "32973", "60098", "7198", "67077", "25007", "44962", "40259", "71201", "14671", "28383", "47648", "41447", "24438", "59951", "23213", "18887", "10219", "34176", "25860", "7303", "7899", "62716", "67407", "49640", "60036", "49427", "53695", "15790", "65421", "20997", "59653", "24967", "1558", "1054", "28196", "6695", "62831", "53797", "18328", "39982", "31937", "46805", "45445", "25829", "67242", "58444", "66701", "28817", "3732", "44258", "24025", "33551", "51378", "32130", "7978", "29755", "52717", "35580", "61427", "3628", "32802", "57634", "40343", "33001", "61500", "11301", "17801", "71076", "30275", "63725", "44404", "37059", "32953", "4287", "34218", "30608", "36371", "9936", "12045", "49026", "64601", "72061", "46289", "8585", "36365", "57724", "11598", "48697", "65280", "37491", "64911", "43060", "59095", "50370", "60017", "34775", "22845", "37926", "69538", "49883", "14593", "29509", "23377", "2437", "31889", "26748", "3178", "48100", "7607", "44356", "48371", "43995", "15163", "1998", "38191", "17352", "64267", "62990", "41144", "64341", "48726", "1723", "35591", "12582", "70558", "169", "41083", "22321", "3350", "31265", "23183", "39639", "7043", "44490", "47909", "12283", "23283", "9", "26161", "71297", "66176", "37047", "11765", "16203", "61034", "14192", "14640", "24871", "68658", "2963", "31241", "66457", "66149", "2111", "3975", "23197", "67753", "20406", "67044", "28276", "69000", "47002", "25843", "40408", "66224", "38171", "28300", "15123", "37228", "35141", "11727", "48116", "2438", "5181", "5423", "13189", "1515", "69713", "11757", "42142", "63038", "47821", "57775", "37422", "39105", "23162", "20050", "46929", "38517", "46324", "25654", "64336", "48560", "53391", "63905", "52830", "24305", "8980", "34558", "13387", "26696", "19750", "58147", "35613", "70491", "37561", "64905", "46996", "13581", "16108", "9864", "39408", "48112", "32543", "50761", "9495", "22573", "19229", "43156", "19017", "13682", "34850", "50752", "24301", "15016", "4873", "55015", "62249", "54602", "27515", "2916", "1679", "22287", "12864", "46621", "44275", "70752", "54061", "21294", "64373", "4925", "9453", "22767", "36340", "59808", "44492", "49185", "68750", "48259", "42862", "41067", "59306", "37897", "61716", "1358", "21501", "47431", "26674", "7808", "36110", "25682", "62593", "24774", "34995", "39013", "49096", "44457", "33455", "61558", "30643", "41662", "38054", "44274", "20007", "4731", "48516", "424", "70576", "21382", "71797", "16069", "9985", "57133", "44379", "15741", "20089", "6226", "23519", "44789", "61281", "62266", "16130", "66616", "66912", "44227", "855", "33864", "5510", "995", "54515", "6982", "71200", "25733", "23233", "8045", "29574", "17441", "60006", "70783", "47518", "24536", "46219", "70619", "57223", "43135", "42574", "30120", "57220", "71613", "25336", "34195", "40624", "33501", "72127", "30994", "55637", "14765", "27028", "4976", "26006", "53417", "65349", "11096", "55956", "60188", "54294", "15089", "12541", "29272", "35653", "5004", "49008", "45611", "7448", "6401", "32266", "36453", "17166", "13361", "57304", "20248", "58348", "21512", "44501", "62255", "48395", "9954", "41320", "11909", "15934", "14098", "32139", "46306", "59785", "27490", "49751", "46402", "31338", "57120", "62307", "71861", "62258", "52131", "25146", "11480", "35751", "71732", "4728", "31280", "39087", "21521", "23733", "11411", "67074", "60419", "20509", "55667", "5562", "70267", "8004", "53172", "36593", "21306", "33062", "61775", "151", "29350", "14659", "50091", "26868", "71477", "60063", "24252", "67831", "31352", "71241", "69161", "61961", "34278", "17762", "69379", "72372", "60821", "55206", "51937", "50253", "42011", "6373", "50330", "36326", "22815", "11823", "16335", "41886", "28807", "48531", "59526", "34920", "13036", "64069", "9921", "11592", "67150", "60966", "47551", "29485", "49247", "39201", "3138", "24038", "24223", "10895", "1616", "58188", "57876", "9191", "33950", "50337", "8160", "1256", "1192", "27287", "66607", "7402", "28179", "63638", "17222", "6077", "62611", "605", "5815", "16738", "40313", "59121", "44901", "57290", "24602", "60149", "1831", "46002", "19159", "42412", "3184", "6938", "64499", "64116", "25704", "22173", "17767", "26961", "22960", "49793", "7829", "1774", "23014", "24802", "14278", "32408", "50389", "62669", "52137", "46036", "40381", "57137", "60161", "66196", "17357", "39264", "39193", "26854", "30586", "2276", "29390", "48968", "42337", "70835", "60367", "51604", "24380", "58762", "41694", "1005", "359", "60025", "38583", "13294", "67504", "8795", "43863", "31303", "49781", "41436", "54308", "21194", "34804", "7028", "61216", "38553", "55194", "28665", "20687", "11143", "65823", "24114", "198", "41724", "425", "33905", "10461", "40238", "3213", "57338", "51520", "72973", "17469", "38426", "22942", "6212", "10756", "32042", "11745", "15880", "10200", "61287", "758", "29803", "39376", "8924", "29069", "70447", "68825", "19365", "51337", "34703", "15430", "64317", "18844", "51877", "69376", "39422", "60733", "22087", "67987", "72860", "13565", "41053", "8759", "39712", "27751", "20340", "10166", "57093", "55648", "21477", "40166", "50680", "36670", "29072", "48559", "1415", "51759", "46568", "55693", "2063", "63015", "55669", "9583", "45593", "11912", "20604", "6048", "24391", "55687", "56843", "9138", "66575", "28150", "26578", "21606", "9737", "58753", "56151", "13837", "67911", "10880", "2877", "446", "186", "44654", "51464", "270", "33244", "68973", "68831", "18476", "32855", "61356", "19221", "14697", "17238", "36638", "61492", "68097", "44989", "34952", "52970", "19299", "67982", "36537", "14395", "1995", "50640", "53783", "15864", "43104", "42066", "51987", "53012", "66639", "31132", "16464", "5745", "69235", "9771", "44021", "22040", "8135", "14472", "46645", "4526", "51481", "13910", "36124", "16688", "8291", "17699", "72375", "41928", "8464", "7040", "66683", "5032", "9416", "64797", "12057", "27506", "51787", "2627", "58095", "31109", "39071", "54658", "43886", "18900", "54563", "44029", "12689", "62173", "48746", "54630", "34240", "44980", "49899", "69496", "52288", "40619", "34523", "22396", "59553", "6661", "23345", "16493", "20312", "64638", "40039", "25640", "16206", "28442", "45835", "48325", "13567", "36092", "11000", "28496", "26517", "46333", "7078", "62216", "69860", "26753", "27651", "55125", "45851", "43677", "42113", "26368", "46472", "72432", "55806", "57456", "19945", "42364", "24559", "7564", "2885", "14062", "48170", "36267", "61226", "69312", "48398", "1668", "7911", "31408", "15691", "5711", "39520", "1185", "61854", "71583", "23798", "33891", "51869", "70529", "7517", "32440", "31122", "31143", "17360", "67187", "50904", "14392", "484", "39347", "19464", "19573", "60001", "30282", "56135", "65123", "56114", "61692", "18513", "58794", "26135", "45107", "63731", "11834", "27956", "47806", "44065", "51571", "37112", "21582", "56347", "62062", "9998", "7661", "28675", "33715", "65360", "68837", "7750", "19688", "14526", "40613", "61488", "16482", "42214", "39576", "39702", "890", "57213", "4523", "42468", "39631", "41412", "33654", "7964", "16261", "10077", "43988", "58947", "35672", "47563", "12192", "10135", "47796", "62968", "7413", "61766", "14822", "58866", "24714", "66926", "3092", "10585", "20058", "57974", "57086", "6306", "1653", "37462", "30774", "27830", "66165", "6431", "42934", "16145", "57401", "555", "68818", "58381", "15111", "27504", "42581", "35155", "6975", "22604", "37603", "21449", "66539", "29989", "17838", "10747", "37625", "18906", "50153", "12255", "53644", "42361", "135", "31150", "54135", "57055", "50710", "54612", "38847", "2342", "70260", "67487", "6650", "31572", "40811", "5898", "54213", "65940", "8513", "6382", "69478", "49618", "7202", "3645", "3251", "57172", "37020", "24934", "62585", "2748", "1290", "2202", "35821", "24658", "44558", "55087", "25998", "40219", "31123", "20647", "53250", "58337", "29217", "30136", "31836", "40715", "49148", "59793", "28789", "1188", "14980", "39050", "35687", "5150", "16098", "30799", "18934", "56663", "34346", "32359", "39500", "41346", "35785", "3941", "22033", "18594", "29094", "49075", "28721", "5865", "58104", "19056", "44933", "62703", "7976", "40251", "22445", "44141", "14218", "32474", "6310", "39561", "40376", "23115", "11586", "48489", "70149", "54583", "68125", "21799", "37003", "17854", "55892", "22763", "50257", "37051", "13546", "65575", "18309", "31594", "13447", "54063", "66968", "39783", "20485", "12873", "15526", "3315", "28033", "33595", "14622", "29398", "69936", "44343", "60465", "30114", "64734", "50549", "67327", "46468", "9461", "33067", "32420", "43353", "44175", "59423", "56341", "13844", "14894", "27725", "34260", "33969", "33060", "25017", "66007", "64298", "11176", "15385", "37091", "9054", "663", "48716", "47363", "4839", "42414", "53975", "109", "36495", "10677", "202", "45200", "37179", "20912", "28682", "37942", "34584", "35342", "32247", "35256", "66741", "930", "44536", "33103", "3724", "70406", "35", "32837", "47769", "28224", "49529", "71365", "6971", "14479", "33821", "60616", "22975", "8051", "21072", "56033", "152", "28950", "453", "32154", "12009", "32452", "62439", "24171", "57784", "25144", "46986", "30754", "60544", "18867", "44704", "63531", "7722", "33608", "8056", "6712", "19723", "7362", "67037", "68003", "68619", "20488", "8825", "42395", "31544", "42360", "31739", "53245", "2587", "37640", "66281", "1843", "36616", "70181", "24409", "45166", "25672", "13992", "66609", "6926", "13601", "38176", "57834", "40924", "71818", "26499", "72915", "64165", "47967", "23528", "2228", "39784", "69612", "48996", "61984", "23432", "9663", "19958", "12686", "8665", "69118", "71969", "44135", "5996", "15360", "16859", "44560", "65901", "32236", "71016", "64577", "21085", "30271", "20336", "2590", "34829", "19891", "72427", "48066", "2375", "50926", "43651", "28527", "44940", "34862", "32961", "62350", "48564", "41535", "13933", "12126", "70381", "39859", "69985", "63102", "21060", "55535", "10184", "52106", "26173", "37086", "33646", "10860", "45954", "41738", "55830", "32175", "69800", "56559", "32296", "2836", "20744", "26851", "711", "1451", "31453", "56319", "63983", "51134", "45267", "10207", "68594", "36130", "30484", "42070", "30369", "37355", "54894", "67535", "64691", "39883", "67144", "39345", "23056", "1153", "2756", "47120", "42054", "27800", "25888", "34200", "11075", "65032", "17283", "34139", "34184", "19808", "42314", "30349", "50994", "8995", "49806", "20107", "54798", "38075", "55455", "72814", "59649", "37315", "13456", "26210", "20548", "47489", "71902", "49232", "69803", "25317", "21952", "45941", "46971", "68359", "43205", "7002", "36247", "48990", "7850", "20535", "26738", "21084", "72148", "20573", "56692", "47082", "31763", "64344", "11282", "54661", "36246", "26042", "56097", "52343", "41279", "15716", "40004", "41025", "19978", "7521", "46248", "56208", "43216", "60496", "58826", "57324", "67348", "40599", "43837", "44796", "22829", "18166", "50587", "22863", "6176", "35990", "42601", "26002", "32642", "64953", "27044", "16189", "72630", "58743", "64109", "16311", "49109", "47241", "26332", "5979", "47804", "28495", "40164", "56305", "43142", "47271", "27821", "51502", "46447", "62760", "3036", "60469", "70831", "35529", "43106", "48021", "57280", "44408", "41938", "41710", "26690", "37924", "35837", "62220", "27449", "2305", "55537", "30564", "32809", "14599", "66879", "51760", "16185", "40248", "8787", "47323", "63844", "32066", "66875", "49384", "51782", "6017", "44555", "11444", "31823", "41780", "46768", "55848", "21631", "56713", "52425", "60722", "23684", "31929", "59358", "32786", "21149", "43025", "19306", "55591", "57820", "60603", "36839", "43114", "29289", "2760", "21753", "28472", "52023", "40590", "4135", "21055", "48724", "56901", "70052", "70546", "73203", "64816", "26684", "45844", "23581", "33761", "42702", "47321", "22583", "53726", "61683", "1378", "33652", "67733", "21666", "23935", "63078", "42389", "46936", "70043", "43363", "3774", "22576", "19055", "35242", "48606", "32430", "6953", "56717", "64521", "55442", "58935", "66527", "4857", "52074", "21159", "50295", "65201", "37064", "2146", "33218", "44855", "55401", "35544", "50444", "38143", "5192", "47892", "40745", "61354", "5798", "66599", "6184", "24324", "26550", "69818", "31877", "31529", "18974", "37544", "11064", "16292", "67189", "32328", "46217", "13734", "33534", "1613", "49237", "33092", "27986", "26021", "27876", "20688", "32600", "59229", "45925", "28372", "20872", "36605", "1871", "68248", "61196", "12130", "10771", "53520", "62384", "34392", "39642", "61710", "20112", "10262", "24201", "53446", "34262", "34345", "66237", "14166", "57652", "15180", "34820", "11199", "31119", "10199", "55700", "28815", "47426", "67015", "57350", "25151", "48684", "20157", "14884", "5322", "31072", "12259", "49282", "62326", "34021", "35978", "47484", "20815", "51510", "31645", "58732", "39322", "66230", "66514", "51543", "55327", "43229", "14758", "41096", "19991", "66798", "49110", "15551", "68159", "21909", "13738", "48832", "27432", "48138", "7093", "64731", "29572", "56639", "214", "40834", "37503", "47416", "22003", "69766", "15380", "55424", "26853", "5714", "12492", "25623", "36181", "48447", "2382", "36314", "11992", "36230", "46429", "67411", "30340", "44831", "52883", "65722", "63193", "29251", "17154", "1427", "44586", "46613", "26082", "55861", "17675", "11553", "17461", "37170", "17296", "29129", "2945", "33473", "2510", "10597", "31497", "51849", "70714", "14172", "33104", "46582", "35654", "36268", "39864", "67132", "61834", "946", "28629", "69617", "7038", "64214", "6585", "6304", "30097", "49066", "993", "13681", "11123", "51125", "71826", "39224", "5172", "56513", "71260", "21340", "57183", "34190", "43827", "31415", "9784", "65371", "6827", "9759", "67607", "49368", "61741", "4078", "23925", "16907", "22900", "51852", "34209", "3540", "38956", "702", "57582", "69078", "46480", "34471", "53260", "39496", "72446", "35500", "55041", "71527", "42641", "38825", "38355", "20534", "37701", "72112", "18913", "65811", "40035", "28043", "33000", "69666", "46476", "31222", "70", "42047", "47121", "72871", "57933", "11541", "70127", "18528", "53816", "58746", "8805", "8174", "29451", "1901", "61201", "10885", "30524", "72247", "66231", "23191", "22076", "55027", "29458", "51326", "47270", "54665", "44430", "40589", "54698", "19834", "19705", "38068", "8972", "67349", "59847", "12494", "64041", "36376", "37116", "68493", "2540", "22908", "5740", "65235", "1872", "5673", "43717", "18948", "34478", "33971", "65535", "55797", "34430", "19198", "70516", "32686", "66827", "20550", "3196", "68061", "35408", "38226", "26609", "25273", "61238", "15046", "56840", "28868", "42937", "65580", "67008", "16251", "59598", "7491", "8641", "40636", "54710", "61907", "55124", "47366", "34001", "46068", "28081", "72995", "51066", "4914", "73127", "576", "64402", "32899", "18665", "18019", "51033", "54236", "58765", "37460", "32825", "41238", "53567", "33795", "52371", "47680", "62860", "15514", "13109", "58849", "43534", "48570", "48396", "37207", "24036", "15565", "2289", "19117", "70164", "33813", "7507", "32457", "5783", "33445", "37583", "55285", "68066", "36350", "71379", "11245", "58701", "58789", "45223", "20672", "44003", "45111", "24445", "69900", "46300", "20940", "12393", "64556", "20386", "9382", "20177", "23158", "60710", "57045", "19410", "358", "58455", "46750", "24943", "65843", "22465", "12577", "58822", "46913", "2955", "67739", "37357", "7909", "29280", "66473", "46014", "17824", "4437", "35031", "35598", "40295", "63733", "39296", "9840", "3905", "7465", "5217", "40865", "39593", "20602", "46380", "11991", "28243", "24596", "33195", "51757", "48868", "68140", "34086", "20553", "29939", "46902", "67171", "59151", "63739", "33097", "39662", "38720", "8868", "23472", "48302", "13735", "22432", "11033", "19708", "12149", "26605", "60163", "66531", "15532", "22177", "68608", "14970", "9217", "27240", "11086", "64781", "68001", "7735", "36811", "6082", "38746", "332", "20047", "72495", "2168", "45252", "67618", "51197", "34669", "64621", "20745", "71332", "23489", "42428", "48733", "2677", "54928", "65771", "2429", "9941", "35701", "25646", "8598", "13028", "32029", "23540", "40956", "23793", "67950", "50498", "68068", "4221", "971", "53445", "3898", "29038", "50173", "19496", "11898", "51770", "59126", "53610", "61864", "32487", "27059", "7731", "61313", "1370", "11733", "36550", "33204", "7932", "62137", "48588", "54533", "37599", "42913", "22644", "49895", "65073", "56539", "23378", "13869", "40273", "5030", "56920", "66261", "27872", "21844", "3845", "58622", "11754", "2982", "63922", "24167", "18420", "47272", "13030", "36940", "41691", "28400", "61531", "44616", "49741", "38019", "63718", "68372", "6729", "52628", "68436", "23150", "35774", "39006", "20743", "44089", "27804", "42772", "54200", "33126", "32673", "11954", "51944", "40718", "53223", "33310", "28237", "17181", "56237", "72964", "29784", "30985", "8956", "35525", "7568", "46511", "50716", "56703", "42290", "57485", "59321", "13323", "4654", "47633", "63906", "18066", "42878", "3253", "17855", "53405", "23248", "68174", "54241", "72242", "71360", "34535", "43573", "26728", "10685", "22903", "15737", "5813", "34219", "35557", "9430", "47897", "27683", "71976", "22229", "6084", "596", "23169", "71498", "36583", "67478", "17729", "3502", "69740", "56743", "19684", "32870", "19987", "18517", "36831", "11434", "23980", "14777", "24997", "61345", "22090", "68488", "31299", "41487", "61050", "29760", "66021", "52630", "67734", "55323", "17937", "22811", "65578", "6345", "4483", "13703", "49591", "67821", "36396", "71230", "32138", "994", "50221", "6696", "43472", "6052", "25021", "59547", "59556", "9096", "24526", "54784", "7996", "56710", "68443", "56690", "25001", "3381", "27125", "29824", "50364", "29291", "62687", "41719", "45086", "30975", "21603", "11719", "2120", "63700", "38114", "44202", "72531", "72782", "18318", "13402", "57306", "21065", "52636", "59185", "8362", "13872", "45374", "2973", "49896", "45671", "64929", "21074", "19544", "65567", "1395", "21258", "64328", "71524", "25116", "55014", "55872", "45810", "42805", "13275", "37748", "21878", "8485", "23927", "32785", "17269", "50768", "22595", "64507", "7613", "24932", "19859", "66706", "50666", "47361", "52966", "64628", "1244", "53376", "8693", "73125", "43544", "62402", "45483", "48211", "16289", "26219", "50067", "69232", "39807", "2613", "60761", "61718", "62302", "29639", "33962", "63095", "9867", "45518", "39831", "71306", "2048", "1953", "72405", "2825", "55004", "13971", "21685", "20549", "37307", "4302", "34397", "61078", "43263", "27173", "5160", "66228", "46215", "1742", "35603", "60788", "19964", "2676", "55645", "26678", "35906", "19862", "53418", "44208", "6440", "22478", "40361", "59635", "69995", "7943", "2549", "3137", "40027", "5983", "72992", "69677", "20291", "36342", "23580", "21747", "11518", "10161", "6073", "35553", "63236", "22234", "73085", "65055", "17601", "11154", "64959", "60240", "28440", "36477", "43172", "66133", "45987", "14375", "28956", "5839", "52079", "59176", "8003", "34270", "48781", "49386", "22588", "50878", "34666", "39358", "4712", "31191", "48596", "40842", "22038", "15442", "3333", "48127", "6871", "26070", "30693", "1663", "10591", "35184", "39341", "24592", "65681", "56376", "30694", "38969", "65637", "1108", "68287", "62624", "49146", "11795", "20982", "62701", "7076", "8652", "34221", "3565", "8524", "39108", "66690", "26454", "19068", "44862", "11391", "46543", "44512", "42757", "42912", "2391", "47378", "24806", "44147", "27996", "47027", "30690", "44315", "27030", "14473", "56615", "58640", "18777", "70999", "49433", "7555", "10046", "51552", "5005", "45546", "31284", "49366", "50157", "48414", "69107", "10927", "47292", "65366", "33528", "22326", "67801", "41115", "25657", "69175", "32766", "37758", "21641", "72195", "35935", "61575", "38093", "46541", "71382", "17393", "13319", "284", "41486", "47951", "37066", "18980", "37838", "50080", "68516", "60121", "4154", "41551", "60598", "20783", "17245", "47795", "69264", "3909", "62095", "8757", "7515", "64910", "31215", "53848", "66727", "3597", "38698", "49646", "48711", "40797", "37919", "15661", "1560", "4557", "23008", "26955", "15248", "70017", "50481", "16801", "46006", "51458", "54687", "25837", "466", "57589", "12653", "47185", "51559", "60795", "511", "52812", "21997", "50988", "57000", "35479", "66401", "68572", "44854", "3550", "52590", "58726", "24425", "11364", "24987", "29898", "18618"]], "valid": ["int", ["19077", "7881", "39793", "10218", "8741", "19548", "26067", "28100", "54917", "1550", "745", "68884", "35383", "6639", "33487", "22921", "67750", "786", "12794", "51517", "40197", "12239", "25902", "6087", "20403", "9469", "44143", "13264", "18479", "36541", "27212", "71854", "26965", "63978", "366", "59640", "70039", "58886", "53639", "52112", "51625", "68229", "22485", "32712", "55147", "68629", "50932", "32875", "17647", "24888", "71703", "22986", "19846", "4376", "46439", "51122", "19248", "43537", "67555", "40016", "61574", "53917", "45899", "39120", "54445", "989", "22491", "15534", "10764", "26986", "32820", "42350", "10910", "63415", "32608", "55786", "52580", "39989", "11682", "27079", "13716", "70292", "19065", "35783", "63329", "17190", "27082", "13049", "48169", "20525", "28704", "71497", "7237", "65541", "51466", "56982", "16993", "10047", "3480", "29104", "34069", "4252", "71427", "32801", "69243", "38029", "57130", "54527", "53862", "56858", "24304", "24697", "39605", "13772", "24359", "20298", "42595", "15008", "55437", "58617", "60893", "32215", "28939", "48896", "63590", "33627", "70252", "67573", "45488", "23299", "28162", "16084", "34844", "69924", "827", "32552", "64096", "17945", "33032", "19653", "18943", "16050", "53307", "32782", "24042", "40807", "68027", "42345", "45186", "66122", "72320", "16898", "41179", "53091", "67405", "52445", "62543", "52731", "18958", "31865", "68968", "39880", "34852", "24554", "55867", "8702", "12281", "28759", "38129", "68172", "49184", "60104", "30560", "42139", "47768", "8203", "17242", "24173", "14525", "25557", "20014", "64783", "62405", "50603", "1152", "21329", "42543", "66206", "66015", "52376", "56820", "48825", "37698", "22950", "49681", "5009", "44291", "42025", "22361", "9716", "40528", "62014", "60623", "3259", "53866", "26882", "67184", "36692", "14560", "22985", "48750", "63180", "48752", "29897", "42438", "33831", "40401", "18520", "63445", "22597", "16359", "64513", "15029", "7051", "26894", "32926", "48376", "2900", "15570", "58312", "11094", "14779", "37087", "54171", "18662", "51914", "64280", "47878", "50236", "19564", "50934", "63912", "71532", "48282", "43600", "40500", "52984", "2580", "20200", "682", "22997", "46953", "40028", "31574", "66565", "59298", "30731", "23583", "56746", "49951", "18740", "51530", "11066", "24686", "41014", "16015", "10185", "23131", "57986", "24161", "64942", "30071", "14833", "61471", "31086", "12943", "48206", "47064", "72477", "68575", "71656", "35983", "51253", "56367", "69743", "30396", "71813", "59687", "40591", "47621", "8629", "37292", "23681", "44372", "11343", "45076", "45710", "14058", "64920", "25846", "1602", "24088", "60052", "42526", "61318", "11350", "9051", "25154", "70992", "12897", "25446", "71809", "32330", "42845", "52250", "20500", "30215", "66307", "30647", "33329", "21464", "39941", "28238", "54848", "27439", "56070", "33019", "25884", "50954", "42172", "38483", "33051", "70677", "72131", "14964", "36410", "38705", "56930", "6714", "41726", "14756", "47374", "45326", "6043", "71452", "47006", "2114", "15146", "47331", "24329", "21560", "60075", "14951", "39246", "59868", "56691", "66102", "45640", "51474", "42707", "63719", "72753", "14611", "48614", "63040", "32949", "40766", "30234", "62393", "16630", "21944", "29929", "27627", "61462", "21164", "45608", "32577", "12903", "14830", "65935", "23163", "29317", "8082", "18546", "42530", "36432", "1810", "24021", "31630", "40551", "71473", "71048", "33553", "72181", "51234", "20461", "50487", "55440", "12102", "38574", "5407", "6514", "20933", "55631", "65799", "67213", "46206", "12934", "38124", "1352", "13494", "27005", "23047", "23001", "40503", "62141", "27697", "20347", "31662", "25247", "50320", "65001", "19633", "50098", "51615", "40509", "24141", "13000", "18715", "14100", "31521", "57042", "56153", "47151", "31944", "69158", "338", "65430", "45520", "7311", "11769", "72826", "43297", "19008", "8503", "37304", "37799", "68652", "72199", "33206", "61712", "26906", "32237", "22276", "33848", "21637", "62204", "41940", "5051", "29764", "40814", "47380", "51223", "32806", "70676", "37825", "309", "17993", "51996", "14429", "20329", "56317", "39985", "41008", "72185", "67106", "30664", "38868", "60409", "11826", "71147", "68344", "68041", "61153", "61090", "51176", "10843", "68709", "63876", "33927", "33025", "52234", "47089", "56104", "71719", "40848", "48379", "25385", "34189", "37118", "20431", "41453", "32085", "45686", "72834", "54531", "33384", "25163", "54487", "29914", "45319", "1850", "55345", "49528", "41224", "17585", "7605", "70246", "50827", "41588", "56315", "22803", "3980", "51403", "25911", "54695", "72343", "24969", "12122", "8009", "40989", "23078", "2187", "12931", "44134", "65210", "38428", "13670", "21823", "1534", "31605", "38807", "50463", "3934", "41466", "71601", "58782", "66217", "67635", "15053", "8973", "30285", "70718", "53475", "58157", "66011", "1978", "474", "72009", "69257", "29619", "1711", "62589", "15747", "67425", "58580", "30287", "39494", "51011", "45202", "21115", "19731", "5878", "8386", "8909", "29364", "36200", "49486", "13541", "63663", "60614", "18687", "44041", "31393", "4077", "20520", "44676", "62653", "49053", "56120", "44755", "55999", "35569", "72251", "4167", "56780", "36232", "749", "9008", "66532", "69459", "64622", "28883", "2823", "16628", "27886", "29679", "38442", "25368", "65276", "65914", "649", "63327", "37669", "48123", "752", "11024", "41501", "59967", "62021", "56997", "58585", "48602", "67473", "25632", "12145", "43306", "36065", "64503", "14241", "43592", "14178", "21379", "58720", "37601", "45275", "17610", "61342", "40309", "34402", "5517", "3750", "10581", "19237", "5585", "53752", "32449", "71545", "56606", "27507", "27882", "31362", "13873", "23038", "72482", "34931", "10484", "52484", "27766", "36744", "7706", "11919", "48997", "33096", "59401", "47682", "19153", "5427", "61401", "11441", "59173", "14781", "38254", "41767", "4237", "1778", "25157", "50824", "23651", "29236", "29273", "4875", "4936", "17362", "43459", "56331", "28601", "67693", "27335", "38321", "5497", "29254", "22421", "17406", "64354", "23663", "21533", "72864", "45647", "12596", "5046", "19270", "6211", "4607", "20890", "71389", "34579", "5936", "44606", "6782", "43064", "60354", "7958", "52452", "18679", "43795", "71694", "16805", "53814", "69511", "60047", "23307", "36305", "70647", "48953", "6528", "59588", "60190", "21678", "36599", "60896", "3246", "1216", "5499", "34178", "56006", "11116", "10149", "38465", "28858", "61017", "37972", "69449", "60519", "41882", "42108", "65196", "54709", "31417", "37705", "47206", "36185", "47026", "51395", "55085", "25124", "15806", "7156", "67514", "33626", "58655", "31952", "31", "5390", "70667", "26586", "48545", "63541", "4982", "23129", "50689", "65390", "9972", "36315", "30464", "10898", "36576", "64664", "1911", "28236", "25823", "52025", "57432", "40074", "18688", "30027", "24900", "27902", "27089", "68129", "19586", "22883", "25834", "3499", "28988", "44160", "30831", "16925", "61713", "69155", "29740", "42907", "45215", "65316", "33801", "42122", "62850", "12707", "56202", "60383", "39261", "54101", "34214", "68420", "38845", "45149", "29300", "48441", "14673", "34622", "31444", "14521", "31344", "12745", "22567", "27334", "1844", "58215", "44150", "53501", "19767", "58064", "20252", "38984", "12111", "55018", "15757", "13727", "44387", "18197", "36199", "28008", "6064", "39171", "991", "38908", "20725", "57182", "20639", "11109", "49905", "38342", "52207", "69697", "41572", "16657", "18626", "32984", "51345", "29579", "50961", "39748", "29694", "59787", "61662", "3790", "38427", "25458", "73018", "44898", "4641", "60018", "11132", "54519", "38581", "35341", "10115", "69596", "67715", "5832", "4485", "36319", "18816", "38305", "37495", "11079", "29928", "16085", "3432", "58973", "38187", "11830", "5586", "26287", "23330", "2179", "7467", "6694", "6704", "25565", "52872", "33148", "49460", "59801", "13082", "13635", "17543", "18621", "36394", "22054", "18463", "70898", "64355", "41449", "45908", "25501", "2902", "29550", "4408", "47736", "253", "28736", "60103", "22939", "73021", "7371", "67962", "65238", "52314", "49866", "25065", "38017", "49488", "55246", "39767", "64168", "35938", "46385", "65536", "50943", "5276", "8436", "44444", "69971", "46203", "41584", "61299", "54825", "51163", "28284", "64178", "8931", "69879", "14001", "54918", "15536", "28102", "10189", "33446", "40769", "52823", "1071", "4115", "34063", "47242", "328", "62524", "29773", "46158", "64304", "30762", "54108", "9405", "26313", "533", "20068", "43530", "61285", "64805", "53828", "56175", "35237", "37796", "19452", "5441", "59662", "44583", "53601", "23100", "40579", "1690", "99", "9417", "27983", "21732", "46258", "21181", "41794", "46113", "69927", "46618", "50425", "11625", "52355", "58570", "39573", "23192", "106", "27647", "5238", "36664", "61158", "63736", "59038", "52423", "34832", "30470", "19837", "67138", "24101", "34358", "48688", "15367", "21917", "22530", "50798", "14982", "52542", "43261", "23119", "60777", "23065", "3398", "26018", "4092", "32639", "22791", "53602", "19870", "11774", "888", "60007", "31264", "9523", "15513", "51081", "52002", "32851", "26587", "17825", "50802", "17277", "14669", "12662", "28217", "67068", "69639", "30887", "3211", "60860", "46108", "58137", "1972", "27189", "46647", "17894", "46702", "46011", "70655", "36641", "36328", "66932", "72708", "73187", "53400", "2418", "6491", "36107", "8746", "20192", "25100", "63504", "19200", "11583", "72454", "1525", "5424", "21674", "72029", "30889", "49643", "14295", "38722", "11616", "62155", "22068", "22258", "12367", "69045", "31628", "24916", "71833", "62395", "26585", "57950", "24590", "68706", "45349", "34238", "39241", "46264", "40668", "54555", "65253", "65648", "42988", "8794", "29381", "39446", "32157", "50667", "45812", "64434", "22015", "42586", "12633", "53950", "73055", "10225", "56881", "30440", "46150", "52384", "20374", "4256", "67881", "29786", "64090", "69570", "15640", "62856", "71573", "21321", "40857", "40962", "4276", "68190", "29265", "17103", "53977", "37858", "38360", "62851", "62311", "25836", "18724", "37280", "2454", "17012", "69878", "67664", "13166", "4871", "5615", "62363", "2927", "50655", "12470", "13034", "5403", "42580", "55032", "43039", "47775", "32656", "32580", "13989", "12576", "55869", "3441", "20195", "9412", "20010", "12744", "27160", "5453", "6352", "17424", "36608", "10759", "70554", "43842", "56065", "27123", "30423", "6786", "18139", "39875", "70476", "22240", "2683", "69661", "68910", "47727", "31632", "4502", "47459", "1611", "24484", "69814", "10815", "11825", "56795", "9820", "44609", "63052", "34688", "55834", "69701", "21099", "40493", "39977", "71450", "30638", "35434", "21455", "20996", "14106", "23061", "54662", "43485", "16009", "63137", "30744", "10336", "66535", "70929", "25739", "21290", "49000", "71893", "23696", "1295", "5069", "12811", "7292", "45968", "411", "64749", "69582", "18195", "45237", "72428", "63747", "54711", "1450", "1104", "57255", "64417", "61900", "27113", "23885", "12529", "29495", "58811", "17632", "54125", "56377", "45643", "9125", "65864", "44798", "35330", "34745", "19774", "33246", "2785", "71184", "8093", "7164", "60665", "58198", "5720", "48540", "11958", "47104", "65920", "50855", "12176", "31586", "20861", "25683", "54911", "38375", "3279", "42636", "71127", "55331", "24117", "52554", "59884", "41518", "11006", "13230", "66367", "30590", "59783", "5951", "39146", "25215", "45281", "40436", "53681", "15892", "49988", "42717", "10085", "8939", "37523", "41980", "6539", "58062", "19825", "25523", "12178", "61983", "3182", "16620", "48200", "49885", "38451", "15872", "14200", "38339", "58820", "35028", "27609", "10903", "39603", "14328", "6943", "6456", "64890", "35405", "2443", "22998", "69990", "31781", "63467", "65417", "2727", "56029", "69804", "24008", "68842", "51195", "753", "41318", "54049", "71414", "44096", "46670", "54616", "5734", "43799", "23713", "53511", "55598", "47154", "24534", "9291", "2741", "38633", "60131", "34753", "60683", "67274", "18426", "43209", "50559", "50530", "28610", "24839", "4765", "46236", "54569", "32945", "70871", "50637", "3694", "36815", "17871", "9448", "35218", "54250", "54721", "233", "48603", "71784", "53583", "65284", "73167", "18905", "4002", "55190", "2117", "12675", "57931", "68048", "27218", "15185", "55518", "4762", "67026", "10035", "21721", "11313", "32895", "52273", "59999", "57428", "57216", "57973", "31487", "4339", "71645", "26966", "43990", "39684", "47600", "65443", "72544", "33707", "66042", "31991", "70336", "37648", "63741", "48107", "26654", "68666", "3631", "54189", "33065", "52082", "27769", "20724", "40247", "35227", "24136", "41237", "40657", "31706", "5211", "7721", "6086", "44615", "14545", "30898", "59122", "23823", "42535", "38419", "63676", "6361", "44092", "61902", "20371", "25138", "34591", "59699", "21994", "17396", "12552", "30890", "43074", "7683", "56611", "62693", "4463", "37545", "43073", "38595", "64958", "24097", "38473", "1887", "57786", "59394", "70710", "59227", "32890", "22471", "10629", "22400", "43603", "51026", "6182", "50654", "340", "3710", "50664", "65100", "34942", "24962", "56431", "19880", "8813", "23278", "21736", "16858", "29608", "50232", "3676", "70461", "68695", "63374", "4248", "22035", "48332", "24487", "36062", "27146", "44537", "72085", "70545", "60539", "1327", "9588", "47779", "66762", "65054", "21418", "30559", "45271", "28821", "51738", "60910", "31066", "30256", "2191", "9628", "72988", "66571", "36573", "5551", "54318", "55243", "52527", "11146", "37939", "49054", "67217", "56071", "72505", "59101", "72913", "12272", "40676", "36103", "37628", "65521", "35840", "61520", "34213", "5568", "29419", "52654", "63914", "70918", "53456", "9889", "55944", "55429", "19408", "17248", "29844", "2413", "31891", "60961", "47704", "72039", "61291", "26429", "2757", "4561", "59891", "35881", "70594", "70696", "62819", "17051", "18805", "13068", "57462", "72295", "72361", "44330", "34203", "51955", "5073", "10360", "34863", "7875", "25078", "32752", "33817", "44413", "49691", "9422", "17191", "15766", "1715", "70287", "26495", "47891", "32414", "65333", "5550", "41571", "55413", "23003", "28450", "14933", "73174", "34081", "62631", "34202", "22843", "38872", "59846", "32193", "34254", "7085", "43529", "29880", "8733", "20277", "46269", "41186", "64928", "29148", "34824", "19309", "34104", "11207", "69454", "57189", "59049", "35636", "9022", "1612", "34028", "39713", "66775", "69005", "42812", "66826", "510", "9813", "17978", "28529", "38440", "63103", "45450", "3494", "10819", "72104", "25445", "22401", "16462", "7948", "45718", "9195", "10260", "68096", "48436", "50292", "44125", "7", "38875", "48916", "42599", "54573", "72153", "5436", "63812", "32783", "64818", "51716", "67523", "5045", "9845", "60071", "44392", "72031", "41765", "69182", "13452", "41658", "49376", "11181", "20817", "4216", "22185", "25935", "35775", "34294", "746", "62166", "57877", "32618", "35297", "34905", "17097", "55763", "29402", "71522", "2897", "56725", "35068", "36741", "52309", "13218", "46887", "47198", "69319", "38317", "67148", "29399", "50453", "8025", "25135", "35872", "57802", "68927", "46771", "65216", "42925", "63688", "58371", "42295", "41065", "33008", "49042", "67289", "33071", "71484", "31247", "50983", "50398", "51828", "25841", "10593", "18826", "9157", "23375", "69590", "33978", "35959", "21279", "34172", "22383", "29394", "21569", "57272", "53630", "66485", "69796", "55878", "14692", "56395", "49106", "19152", "24322", "22134", "56751", "30406", "30458", "1487", "36407", "15823", "24148", "16984", "50283", "49740", "12190", "68541", "27347", "32842", "46346", "19120", "64820", "26446", "39463", "53497", "57956", "42764", "25427", "59739", "6457", "35416", "57243", "32893", "68407", "5042", "859", "44774", "52080", "40397", "23787", "1790", "49870", "16101", "49600", "25592", "38888", "7697", "60474", "31469", "59365", "40302", "17052", "6976", "64980", "52956", "32566", "43004", "6428", "26451", "35792", "30552", "24496", "51879", "5358", "22120", "60522", "17519", "1583", "65506", "1895", "72853", "70297", "46326", "43227", "28425", "13427", "23260", "31157", "5963", "66633", "58798", "14254", "35874", "29553", "69378", "57178", "16291", "20401", "26302", "68152", "24185", "15453", "581", "53676", "56501", "19696", "33729", "47683", "53124", "52909", "72098", "32255", "64720", "2163", "35516", "52765", "42132", "5479", "67585", "59579", "10566", "53285", "26885", "10809", "61028", "15953", "43625", "17589", "1120", "46716", "18312", "28084", "10122", "51617", "29504", "41069", "63097", "55183", "60939", "731", "7571", "48965", "38157", "55841", "207", "6163", "66953", "23700", "7849", "18443", "39041", "3725", "59067", "27563", "7608", "52363", "50430", "47192", "36197", "14600", "20138", "63657", "7148", "9670", "4046", "23077", "58350", "49886", "23758", "51259", "2797", "23385", "65787", "48604", "58542", "54037", "6240", "43707", "29486", "23203", "37012", "36226", "16507", "68089", "32404", "49473", "66048", "34006", "17335", "58478", "24643", "56252", "31721", "45072", "68552", "32922", "7123", "12001", "18269", "10709", "36375", "16414", "6496", "49775", "12284", "7113", "56216", "64718", "62333", "42276", "54334", "41015", "1993", "52088", "59914", "38768", "50278", "73232", "12043", "5527", "28627", "7041", "6857", "31314", "35762", "22222", "36710", "5018", "52451", "48506", "45115", "69857", "44429", "30999", "41095", "49298", "70154", "34387", "63801", "38697", "20425", "59469", "14002", "12337", "69828", "21510", "12941", "30361", "71355", "26032", "56897", "21179", "68212", "12680", "5825", "58718", "73109", "44485", "43538", "62214", "40905", "48767", "24061", "5242", "23381", "38372", "32223", "27416", "46610", "65289", "49680", "50350", "59629", "59066", "54927", "42068", "60845", "37378", "8718", "71001", "61330", "34377", "2371", "35948", "64234", "50443", "7980", "44670", "3212", "54642", "33619", "54182", "64014", "9409", "67565", "13865", "72277", "40181", "35132", "62229", "61752", "70670", "28753", "27715", "1068", "23584", "73064", "22885", "71049", "28673", "7455", "69469", "24966", "62192", "55671", "979", "27578", "33932", "60888", "55008", "53186", "65459", "55774", "9015", "37716", "50190", "11442", "25129", "72712", "53133", "29468", "49562", "42073", "40879", "42976", "60380", "49455", "1143", "66942", "33649", "50597", "6693", "47500", "18731", "46251", "41500", "38737", "61565", "7389", "7938", "20225", "53464", "33681", "738", "35204", "7074", "59895", "14042", "36853", "950", "8320", "15421", "67578", "11512", "43368", "33432", "19362", "21710", "67951", "65706", "6214", "13176", "69339", "50258", "47609", "7476", "33591", "64728", "72182", "48060", "33349", "54259", "12991", "51686", "42813", "42696", "62750", "44366", "58176", "64982", "30653", "20280", "65649", "29960", "28892", "36729", "13842", "31928", "47351", "60673", "44686", "40952", "33231", "49255", "6544", "18540", "34449", "47613", "22752", "9098", "60379", "49045", "57996", "31564", "56067", "35234", "8612", "13926", "20882", "16314", "26993", "23010", "59192", "1282", "50022", "58097", "43547", "30002", "35251", "7003", "34964", "2333", "70925", "32113", "67324", "39107", "29987", "9934", "69406", "17772", "65168", "18440", "52333", "20063", "37026", "26670", "36158", "21743", "41544", "41488", "51136", "57857", "28048", "49378", "56264", "25722", "12892", "46822", "26310", "40438", "20667", "59280", "64359", "3167", "60728", "59311", "49635", "13853", "35707", "36099", "42564", "47118", "67448", "57704", "58607", "60716", "65454", "38087", "41340", "13152", "833", "25182", "67282", "46620", "70550", "18149", "34996", "23634", "18308", "16405", "53205", "16020", "66247", "27100", "9947", "47172", "9611", "63856", "44937", "71640", "55641", "30061", "20934", "29847", "8043", "14571", "8740", "20810", "44288", "3472", "39628", "2632", "11696", "69964", "12539", "25022", "58503", "35648", "47770", "62276", "22314", "7797", "4236", "46194", "13346", "64252", "49699", "18915", "62738", "62280", "11697", "39312", "7351", "62790", "31323", "22536", "36967", "66056", "1096", "53841", "4836", "66831", "54447", "21554", "3018", "22713", "65409", "43277", "60563", "3368", "26386", "66363", "58409", "21981", "45131", "39290", "30570", "14916", "47456", "67900", "66257", "16949", "32562", "40255", "24834", "6139", "26460", "73112", "17203", "44236", "49373", "28847", "67451", "51723", "19747", "6317", "57892", "48974", "61260", "69464", "10887", "44513", "4170", "56760", "4958", "16751", "15676", "29133", "50361", "58539", "37113", "68093", "57310", "57415", "58860", "59959", "61460", "13618", "38736", "25121", "64213", "28777", "31907", "35744", "23416", "16884", "52270", "59833", "38516", "26638", "57421", "49848", "72614", "72147", "40369", "19293", "34674", "14208", "65502", "1369", "60073", "774", "64635", "42010", "30320", "71005", "23317", "43510", "2177", "23004", "52356", "15876", "64682", "37857", "31425", "16675", "17408", "55241", "10954", "10143", "52972", "31746", "31260", "16046", "57385", "35207", "136", "44388", "27552", "53959", "67252", "5505", "45559", "2710", "64826", "51860", "22242", "69027", "54839", "3468", "10840", "12753", "44320", "34907", "27061", "64362", "48149", "51882", "59670", "7126", "39346", "49881", "21559", "25505", "63408", "11971", "65003", "66770", "60202", "39112", "30506", "32016", "58915", "59426", "21848", "20803", "64987", "64851", "64730", "25562", "63618", "17649", "52109", "26742", "67772", "37930", "22906", "35625", "23013", "47288", "48934", "44203", "24137", "72401", "16937", "62219", "55968", "64271", "12101", "9645", "3728", "67835", "28872", "57646", "37931", "22634", "26732", "32424", "46837", "33851", "17176", "9608", "22376", "20104", "12523", "1596", "51798", "61346", "6556", "39283", "71713", "72139", "69707", "41730", "54723", "62365", "41557", "48543", "13909", "57944", "71187", "60801", "21625", "37810", "1530", "26603", "1800", "42227", "18807", "62499", "4230", "64668", "14274", "25695", "31156", "19515", "66651", "53776", "68571", "48152", "66306", "62195", "42322", "68913", "31501", "6906", "24024", "50704", "69524", "63787", "51013", "21414", "7795", "33714", "8744", "15904", "57015", "56832", "22473", "7419", "9471", "60368", "10181", "35627", "6005", "11596", "21169", "21332", "63559", "48665", "10069", "59506", "43798", "54536", "70314", "42801", "12269", "27498", "57449", "38880", "43419", "10806", "42410", "27210", "45558", "88", "58395", "40032", "49227", "60497", "3287", "43475", "44016", "62502", "64158", "49515", "71725", "27813", "43709", "24921", "63323", "3605", "6272", "16900", "17612", "8308", "30029", "43703", "23936", "11490", "52668", "115", "27704", "15188", "39173", "14387", "10174", "58483", "32830", "49581", "2956", "17266", "20318", "13250", "22890", "69", "68827", "14126", "37898", "138", "39482", "36467", "17551", "38640", "55497", "30391", "14845", "28318", "70035", "28436", "42274", "42694", "18696", "32319", "11414", "43201", "71494", "16917", "47883", "44806", "16350", "10943", "15449", "68854", "67031", "2583", "62464", "47820", "45940", "61261", "38761", "69171", "50899", "59952", "60119", "21303", "4992", "24518", "41485", "15052", "68964", "68054", "21552", "57165", "41835", "43613", "40382", "44939", "46048", "24701", "71789", "65911", "8955", "26572", "19327", "72795", "20878", "71795", "19735", "7343", "15592", "44764", "63263", "29677", "56936", "24246", "34438", "30760", "52658", "70416", "20880", "10208", "49578", "25995", "50859", "30978", "35238", "18910", "36655", "15618", "17604", "34557", "53904", "39827", "64233", "15153", "31757", "35060", "66272", "28660", "13508", "57224", "46109", "27046", "64364", "10177", "67496", "38843", "12098", "54725", "25023", "52870", "68288", "69094", "63321", "8984", "49188", "68317", "60965", "70076", "47435", "34311", "19928", "41188", "26521", "25820", "33420", "45903", "64898", "36966", "50140", "32578", "27237", "33274", "5114", "45017", "15713", "46430", "46522", "14984", "39695", "60004", "70769", "68067", "30414", "23376", "45885", "16590", "2005", "25086", "33926", "41844", "39555", "63239", "29544", "40934", "29279", "3240", "24616", "66311", "48707", "46228", "5937", "50907", "73128", "1231", "17870", "14165", "41254", "11714", "19277", "2915", "53102", "43517", "49565", "26563", "38643", "55496", "42495", "65832", "67973", "2654", "39259", "7042", "52535", "46951", "54368", "35499", "58005", "67393", "5123", "25220", "24383", "17658", "42252", "18891", "50228", "54903", "38685", "11608", "58760", "28485", "37035", "15103", "1808", "13083", "42371", "58228", "65096", "73105", "43133", "32678", "38642", "59258", "11914", "63928", "51201", "62863", "36634", "24083", "63545", "31792", "27024", "19621", "58090", "25962", "40210", "7965", "37258", "7186", "26108", "61642", "66640", "71352", "31842", "31050", "13957", "37363", "71552", "36225", "57166", "9133", "61368", "46193", "39664", "8559", "2723", "11772", "10522", "13409", "3458", "53312", "62471", "51846", "5142", "5977", "23590", "11694", "63586", "38997", "482", "60278", "71990", "52884", "73152", "31454", "13441", "64693", "69392", "21523", "49002", "45094", "63721", "11548", "52648", "23178", "61360", "6161", "67681", "6484", "36038", "60901", "66029", "52042", "49182", "26347", "72733", "26745", "10751", "48721", "35164", "3191", "63197", "53617", "32988", "53153", "1862", "35281", "35025", "42978", "54008", "21187", "40474", "26952", "9540", "69856", "37998", "59254", "50787", "6955", "33320", "3995", "2626", "3759", "30767", "50810", "61419", "44915", "17048", "69042", "17356", "2977", "72706", "54866", "61068", "67891", "44576", "60771", "64478", "21364", "28037", "64683", "61187", "32202", "21450", "67336", "9192", "14194", "1658", "14339", "2231", "20526", "21097", "5589", "28614", "72841", "102", "67549", "41774", "20567", "69912", "26063", "49519", "17582", "23652", "67714", "12017", "9907", "12687", "3978", "17467", "44984", "9083", "45584", "2846", "59599", "70004", "19646", "39230", "72309", "53709", "4158", "66023", "54082", "47747", "70308", "10045", "33877", "60115", "55325", "9861", "20244", "26364", "60008", "17213", "17164", "15528", "16996", "25140", "42711", "11248", "17070", "37145", "7376", "68154", "37818", "11296", "248", "43372", "34568", "38446", "60636", "8248", "62746", "2594", "12901", "66113", "48544", "37606", "14723", "16774", "15569", "58666", "17092", "33294", "35107", "34593", "15621", "47501", "41085", "36406", "58611", "63300", "36935", "66305", "61416", "61845", "14229", "23284", "67902", "54502", "30034", "42115", "47112", "6968", "63874", "29875", "12352", "21611", "20889", "56815", "26237", "57588", "72116", "60787", "47938", "25259", "70006", "40575", "48073", "44465", "47913", "17382", "17110", "9246", "61919", "23715", "20618", "29241", "18623", "42342", "56574", "64216", "7298", "1614", "20546", "66343", "27476", "40647", "38525", "38476", "27624", "67081", "32126", "1264", "55353", "5698", "58463", "35036", "10303", "70698", "19041", "48162", "31520", "22399", "14976", "30469", "11152", "25158", "71978", "2351", "62020", "40423", "12162", "47520", "21225", "35057", "6010", "47668", "57912", "25658", "65240", "66927", "57706", "65953", "69784", "15983", "58735", "71831", "70823", "23429", "7232", "56285", "49018", "36059", "70832", "50395", "48585", "13321", "44030", "30527", "60899", "33170", "18645", "18846", "20834", "56403", "23724", "4755", "8159", "21385", "1785", "72775", "21948", "26008", "68320", "34429", "36901", "67705", "41675", "33362", "52047", "30205", "54955", "39172", "13530", "46803", "49308", "23116", "51540", "64829", "10374", "25390", "17760", "55310", "26294", "45028", "48216", "10939", "57414", "55036", "35968", "49534", "10649", "45753", "2526", "21688", "9624", "71687", "36902", "41092", "8962", "31253", "10105", "37271", "62651", "42106", "2894", "36870", "44664", "14340", "27012", "41586", "61491", "29622", "47359", "51434", "60232", "16501", "8721", "47352", "43390", "12198", "33750", "72597", "23434", "45165", "26212", "44745", "14880", "30415", "54727", "35267", "15388", "60764", "44221", "54473", "72905", "50817", "36402", "17806", "3034", "2318", "72968", "7915", "59559", "36711", "34166", "23083", "41406", "60254", "43921", "34879", "42710", "45628", "35604", "28148", "60619", "12509", "34413", "48255", "34813", "25900", "71376", "53288", "6905", "33235", "71886", "64681", "16513", "55226", "62966", "58487", "2173", "5567", "16103", "47102", "67035", "29516", "24576", "17237", "31850", "71541", "12840", "10128", "29872", "45124", "59082", "1741", "50355", "14496", "62520", "56654", "38384", "15887", "60650", "37023", "23934", "31112", "1245", "16473", "22343", "49824", "67099", "9916", "69240", "19292", "56610", "31439", "10962", "18814", "63455", "12163", "58659", "51949", "68974", "38296", "41666", "10049", "55649", "62955", "33641", "41203", "47284", "55746", "26702", "64580", "63091", "40150", "56980", "12308", "58986", "11025", "34537", "8389", "34249", "24140", "50038", "31534", "10114", "12412", "22681", "71697", "61119", "14462", "28135", "51950", "72461", "69520", "28511", "38982", "54489", "18273", "14365", "55822", "46415", "18784", "31854", "516", "53258", "58906", "31986", "63996", "6343", "3912", "6770", "35119", "19591", "27157", "17721", "53331", "69672", "63320", "22907", "31167", "40098", "28240", "50733", "56424", "52228", "38644", "48363", "4032", "22725", "39266", "59146", "48906", "61694", "36242", "3190", "27698", "34966", "11585", "47490", "70689", "58703", "27610", "5343", "65353", "26544", "38110", "38012", "46040", "31717", "73101", "65954", "9233", "11402", "52227", "14840", "31373", "45842", "64975", "41521", "19698", "42200", "67563", "54649", "37131", "46611", "4100", "42555", "4585", "29850", "33269", "68254", "31040", "57621", "2585", "58778", "12639", "30857", "41995", "57313", "2940", "9134", "21", "52795", "45077", "36330", "57154", "40923", "71252", "72398", "2042", "67139", "50783", "30076", "20582", "70514", "15902", "29009", "59118", "54550", "53679", "11323", "39177", "70671", "18017", "20786", "38293", "67846", "59714", "31826", "53935", "13952", "43169", "13912", "15671", "35661", "38492", "22232", "24311", "50116", "55289", "62535", "48011", "29334", "1678", "73126", "63028", "55077", "43307", "62858", "18450", "34656", "13401", "64088", "68265", "17153", "38993", "12232", "62225", "9082", "21875", "13814", "55842", "17079", "52191", "8919", "51304", "4000", "52105", "21050", "17481", "49887", "70791", "62628", "64613", "70319", "6823", "30442", "64125", "40199", "22048", "49067", "40076", "69753", "10274", "38239", "11300", "2848", "10487", "62699", "67924", "12433", "41519", "53915", "19576", "4490", "49724", "69130", "49156", "7182", "39321", "47296", "47818", "12197", "3229", "9081", "41619", "41727", "35331", "15539", "29609", "7616", "49856", "46761", "28369", "65109", "21983", "8218", "13786", "7378", "44182", "25409", "73039", "33643", "35515", "41045", "15094", "46469", "47274", "48690", "50847", "41899", "59580", "4225", "40244", "41419", "11050", "36069", "58755", "7218", "52230", "14596", "65218", "64177", "34362", "24820", "10739", "24485", "54403", "47624", "46866", "39339", "4406", "66720", "36369", "50070", "10749", "61417", "6435", "23588", "70041", "17310", "70273", "8280", "42864", "34838", "47309", "21091", "51453", "42549", "71242", "51292", "30085", "67386", "5041", "62705", "6863", "64546", "32079", "6920", "58278", "43010", "27267", "2820", "10407", "37321", "27273", "11046", "19135", "53240", "2787", "22531", "27425", "46413", "24910", "28032", "63395", "42018", "21613", "33225", "7766", "29052", "5540", "8071", "2552", "68098", "14856", "50085", "38070", "69873", "10826", "30454", "19741", "39667", "65141", "23957", "61358", "48952", "48053", "56160", "44462", "48717", "16676", "43691", "38457", "29882", "55609", "59325", "58741", "23846", "55430", "62122", "13408", "70439", "46343", "72194", "60107", "11010", "56383", "22795", "3415", "41253", "34161", "25500", "22312", "39332", "22566", "52810", "27394", "669", "7338", "9479", "39279", "13700", "49735", "56497", "70628", "72146", "23777", "17803", "45672", "62294", "2667", "35333", "56055", "52936", "30583", "26509", "3050", "3210", "64657", "29649", "46204", "29569", "31007", "69056", "36333", "19511", "69415", "62115", "53028", "40230", "24433", "40136", "56951", "18042", "7143", "54590", "56094", "22106", "13505", "56855", "23379", "23498", "49973", "18274", "36557", "72066", "28360", "22329", "38158", "16968", "62198", "47523", "13788", "57237", "19185", "57508", "6487", "26324", "57024", "68182", "26875", "37342", "34018", "4592", "4282", "3403", "57848", "16775", "23530", "48701", "34934", "48550", "47857", "43183", "35332", "56265", "30498", "37643", "39089", "51699", "33399", "61860", "33061", "12053", "25331", "39513", "34548", "5607", "27693", "60404", "17246", "54441", "49808", "50571", "54621", "54193", "61548", "72638", "19514", "36725", "35726", "3070", "49385", "32429", "54032", "32826", "23996", "13119", "61637", "24765", "36100", "21458", "27375", "52360", "68558", "27263", "40396", "3219", "63195", "49048", "18935", "27742", "22370", "6686", "23215", "32933", "14312", "57200", "43380", "11316", "37920", "57353", "51913", "2701", "1041", "1959", "63253", "12826", "12318", "54672", "16682", "19094", "62282", "57260", "236", "57676", "25288", "50782", "37236", "7282", "48886", "40640", "27941", "27406", "41462", "21808", "41461", "13045", "16179", "295", "65796", "54033", "52919", "39740", "60310", "46818", "21683", "8674", "5892", "30906", "12486", "19470", "50567", "55622", "16186", "55355", "1520", "680", "24089", "60226", "30037", "58486", "20314", "11771", "14089", "29065", "33699", "34807", "65036", "72107", "31095", "25957", "19795", "49428", "23372", "20232", "544", "55603", "2314", "7079", "31999", "10641", "24991", "64719", "68838", "17345", "59558", "68300", "55851", "65678", "72065", "64908", "25619", "15593", "53448", "68123", "72960", "8725", "54104", "21819", "12855", "72888", "5290", "36434", "32597", "8564", "67703", "29266", "14878", "21478", "11273", "70200", "65933", "24272", "22194", "64604", "31724", "9601", "45873", "56147", "12398", "9754", "25460", "67828", "57972", "54835", "62328", "59624", "31494", "56712", "23339", "46157", "63439", "1226", "56892", "71055", "54396", "70745", "28371", "799", "6081", "70271", "6688", "61246", "54010", "19499", "60321", "46851", "36452", "12646", "54272", "33149", "42063", "34192", "18836", "69836", "39579", "61023", "70343", "11032", "16025", "50734", "11514", "12764", "15987", "62796", "53614", "57464", "40482", "59243", "53903", "46997", "10498", "68183", "67447", "58036", "18678", "51936", "50693", "1057", "5691", "54625", "7598", "55838", "59839", "71476", "27636", "16217", "33741", "33546", "46391", "45004", "57624", "847", "42384", "71118", "42981", "2829", "64138", "33124", "59164", "21622", "50318", "39855", "5223", "31679", "4662", "56965", "41567", "36423", "53748", "48822", "54264", "35717", "28155", "7440", "12325", "45875", "56801", "1151", "11579", "4722", "44478", "47505", "12844", "51500", "5324", "48838", "70082", "10988", "72389", "22176", "37063", "17547", "70169", "38404", "5291", "9753", "72283", "32604", "56730", "49326", "2599", "35423", "26013", "25164", "27977", "49189", "5464", "25512", "51089", "37887", "29788", "36424", "66193", "48864", "71351", "287", "72980", "41564", "72136", "36681", "11344", "36078", "28362", "67354", "23106", "60692", "13593", "21190", "49892", "57655", "29136", "15335", "61639", "25652", "1548", "55515", "42529", "63153", "49587", "12049", "31988", "36450", "9160", "46684", "46680", "45566", "37908", "11027", "60441", "5533", "40428", "17962", "16703", "20563", "54841", "44992", "12551", "18868", "4626", "20337", "42855", "70433", "10231", "8141", "46390", "19786", "58771", "51404", "21972", "1946", "11426", "41743", "63773", "69099", "70005", "24150", "69631", "13115", "2612", "56277", "63864", "72412", "68361", "62748", "22555", "14811", "5823", "55078", "24560", "42118", "59962", "61410", "11385", "8444", "25805", "19036", "70891", "20223", "13921", "56374", "10803", "60230", "12491", "27754", "60096", "21153", "20572", "61340", "43115", "20064", "69792", "44953", "2618", "31783", "1879", "37752", "66677", "31809", "20217", "5415", "33318", "69607", "50412", "62623", "3919", "36419", "45082", "44703", "11312", "2309", "70896", "17131", "68385", "66509", "42480", "35407", "21069", "8451", "32388", "50876", "13278", "2761", "66161", "23449", "3884", "36836", "30684", "27041", "16420", "37117", "71265", "71250", "21268", "66666", "43384", "12246", "57149", "62031", "16810", "47329", "15221", "64118", "52775", "62184", "37165", "16258", "1315", "42539", "6046", "808", "8534", "46554", "53246", "21835", "33820", "28389", "65352", "18508", "12658", "1162", "79", "20649", "35187", "49569", "34134", "70159", "47953", "55732", "70820", "15387", "60322", "65460", "47507", "59464", "53352", "70808", "65729", "37819", "16158", "50202", "40544", "41540", "50303", "28520", "11928", "19910", "32699", "56763", "17707", "44364", "66515", "34342", "24585", "58641", "5481", "20720", "15631", "64819", "42490", "17781", "68124", "11636", "8483", "41731", "35656", "31200", "21132", "7813", "57504", "44385", "12700", "11337", "6256", "20394", "22443", "41055", "9987", "32242", "19238", "17778", "12757", "13841", "19642", "48388", "49447", "42623", "51144", "29498", "5222", "40974", "6581", "45206", "33602", "72013", "57720", "26400", "41162", "38166", "61274", "681", "14117", "61009", "46829", "45744", "50908", "21746", "43345", "5808", "9609", "13646", "32929", "14565", "8186", "35446", "50428", "5461", "7160", "1641", "50674", "40617", "7029", "40832", "31135", "37514", "66414", "50628", "6879", "1618", "62928", "10216", "58237", "15556", "46270", "42818", "1746", "30121", "22031", "56695", "43728", "35626", "42525", "27911", "61484", "24288", "21967", "48566", "849", "21786", "53674", "108", "31565", "56791", "37632", "49397", "26834", "68828", "46742", "20627", "19911", "63757", "1997", "5141", "48734", "18643", "8246", "70065", "14898", "70435", "54813", "32513", "31528", "56532", "7579", "62940", "22578", "60882", "16048", "49332", "71855", "69168", "36750", "655", "52600", "44702", "12790", "13896", "59134", "9903", "59203", "65782", "23517", "2681", "39842", "636", "1677", "1077", "66479", "65671", "20025", "26243", "25401", "56262", "19999", "30250", "54830", "18800", "55117", "15249", "14132", "47225", "30987", "43757", "66949", "9480", "46058", "11845", "56454", "71776", "3410", "41966", "36542", "23697", "26340", "56792", "50649", "54232", "2928", "50658", "41104", "20297", "53763", "21865", "55986", "8103", "58271", "52386", "48384", "19180", "34099", "53795", "59346", "29500", "5860", "13963", "49019", "70157", "49503", "11124", "9286", "54025", "25403", "54053", "18181", "59387", "44820", "29548", "56365", "33243", "58783", "16511", "13324", "3055", "10414", "59377", "63064", "60352", "72174", "33485", "41431", "15074", "37638", "64867", "11814", "39925", "43100", "15455", "30247", "58744", "12157", "35784", "51894", "26765", "44951", "48164", "2871", "70807", "20367", "29057", "46428", "71986", "54678", "71223", "35439", "25541", "44923", "62558", "57840", "31139", "26273", "6197", "35505", "25498", "39991", "33205", "20100", "491", "54504", "23089", "27323", "19681", "22023", "69314", "13273", "45465", "22505", "20751", "64470", "62081", "39066", "18644", "30921", "60520", "72580", "50378", "6859", "10432", "41566", "19849", "29823", "61374", "41009", "55158", "18939", "34648", "10528", "55305", "67652", "43643", "50755", "45919", "23899", "24905", "43279", "39753", "57740", "39319", "19618", "47774", "32735", "13754", "45790", "2021", "38663", "64251", "56096", "69730", "53225", "46381", "8259", "59375", "63909", "10380", "39750", "26091", "62449", "66500", "13457", "12059", "4980", "28390", "59974", "66439", "67133", "12465", "69868", "3424", "31226", "22935", "34024", "65106", "5960", "69957", "24387", "43915", "44722", "61682", "29410", "17526", "32100", "3289", "57725", "21343", "58045", "45869", "19872", "42279", "38887", "67426", "30483", "11573", "18852", "37440", "19084", "31127", "68951", "6266", "32924", "34708", "42838", "59236", "52877", "59682", "20502", "66164", "26005", "26524", "11803", "3855", "60679", "61365", "40485", "39599", "33539", "67686", "52904", "8013", "1218", "51715", "40078", "8234", "58699", "24511", "44020", "919", "35365", "47264", "65411", "22345", "58393", "30264", "13140", "33250", "67383", "29353", "44786", "64884", "8600", "58535", "39583", "21145", "25359", "51284", "4636", "44353", "15890", "59503", "33439", "3944", "67803", "9567", "14844", "15486", "22655", "40301", "58843", "14107", "42547", "40422", "28648", "1577", "28968", "209", "30846", "31177", "41887", "22580", "64033", "39805", "43840", "70773", "13882", "36397", "36216", "40003", "44603", "43900", "59435", "33577", "50608", "825", "32077", "7711", "43433", "65279", "21078", "46690", "53132", "62895", "4333", "66632", "31921", "40878", "4653", "31465", "27149", "68994", "38701", "29583", "25412", "23458", "9179", "52108", "63201", "41074", "62212", "25826", "8374", "28855", "56062", "68644", "8912", "60003", "16932", "71433", "40129", "67206", "21874", "37557", "47216", "42293", "43624", "5874", "52942", "69190", "47961", "33466", "1366", "14607", "29306", "28027", "8065", "53467", "50131", "2988", "8169", "10227", "14306", "5664", "853", "36005", "63221", "9570", "27557", "50386", "62942", "58864", "2100", "2364", "71852", "57105", "37922", "40663", "45238", "55689", "63534", "53351", "67603", "54541", "59310", "45950", "9850", "29620", "32037", "66922", "7815", "35690", "31831", "69084", "69773", "53468", "57179", "21151", "8552", "16813", "53944", "31307", "39697", "11139", "58107", "56345", "3513", "37660", "30375", "23487", "61867", "35329", "904", "1254", "32208", "11559", "8335", "22897", "39652", "50749", "38567", "27421", "5938", "15261", "16325", "44741", "13129", "30086", "26277", "56057", "27250", "56646", "71420", "30652", "39124", "3244", "48919", "22836", "11589", "30562", "9337", "10501", "22666", "44136", "44142", "69923", "24408", "25218", "42021", "5038", "28763", "67554", "3156", "13536", "48431", "30431", "30474", "72502", "11755", "30408", "49003", "70186", "20457", "5140", "18901", "57068", "36602", "28464", "72325", "64978", "20945", "757", "18086", "4617", "24373", "49805", "30208", "18029", "47926", "16786", "9802", "34101", "3428", "20008", "32793", "1676", "31075", "32218", "38233", "34284", "45336", "42800", "57362", "50750", "70563", "13376", "46261", "18683", "46833", "66903", "3957", "57270", "64511", "10867", "70855", "60913", "59585", "46088", "33908", "6168", "62064", "63152", "46334", "5576", "27249", "21056", "39558", "10010", "40502", "72668", "6675", "71672", "64674", "65741", "52724", "39145", "23596", "20729", "48856", "41649", "11515", "49105", "54795", "26496", "42111", "29422", "49094", "44735", "1044", "28632", "45242", "34759", "48895", "8274", "13797", "17609", "5186", "43579", "37852", "58512", "9781", "62937", "69711", "41062", "10688", "61980", "22715", "1343", "9402", "38251", "33844", "1486", "53033", "52906", "21860", "48573", "23937", "38666", "65657", "29332", "52826", "30917", "34301", "8644", "2939", "18993", "68391", "17711", "61217", "27652", "15302", "51883", "51583", "15707", "13668", "611", "68839", "71234", "11382", "25208", "10645", "62423", "32048", "55394", "56536", "7177", "16137", "49924", "10967", "60857", "37412", "59878", "23836", "21620", "47848", "67723", "27222", "35097", "23862", "36147", "61899", "21282", "15845", "23217", "8672", "41029", "23930", "18847", "17003", "17769", "7478", "203", "60585", "69938", "25344", "39739", "36476", "25454", "43204", "50809", "29325", "44170", "40886", "66049", "26547", "7238", "7280", "20156", "24961", "18287", "64019", "32943", "48844", "13360", "6245", "7600", "59109", "73217", "34385", "33037", "72473", "7648", "12093", "22627", "42353", "52661", "71752", "13270", "3352", "15523", "63683", "56734", "63068", "63798", "39744", "48846", "35193", "1197", "27000", "46532", "11463", "47963", "13393", "30218", "57376", "32998", "48412", "44974", "44074", "41612", "50647", "42315", "24539", "72360", "45273", "16837", "19933", "64270", "61539", "27337", "9350", "56110", "36587", "73032", "58969", "45564", "52339", "60726", "36488", "50920", "14409", "67642", "23974", "64363", "47113", "30672", "22135", "52293", "46796", "72356", "37126", "68634", "64124", "66503", "28754", "26478", "37813", "70322", "50892", "12184", "29607", "53131", "51915", "6126", "42310", "35924", "22979", "34023", "6019", "72934", "41515", "69655", "9266", "30099", "61326", "5122", "65952", "40994", "32045", "37451", "72041", "36837", "25267", "47153", "63783", "32537", "19538", "51148", "52820", "67056", "29600", "45515", "9339", "56064", "2085", "30276", "23915", "31363", "3609", "67879", "62339", "29615", "32908", "25059", "40507", "48118", "71000", "5361", "31450", "17128", "48522", "66723", "11195", "53424", "68512", "37202", "58976", "73073", "6881", "22556", "33708", "6193", "50742", "6368", "17034", "58658", "1482", "28617", "8177", "34719", "3161", "14870", "4095", "41548", "42341", "70445", "5053", "38668", "71116", "51764", "15600", "1965", "3705", "19822", "6294", "38144", "16670", "19677", "8842", "28931", "19518", "66502", "58118", "24919", "52057", "22286", "49901", "31570", "49022", "35927", "6058", "45685", "64063", "36877", "40804", "30513", "21205", "55793", "18128", "18969", "23035", "61377", "28011", "67029", "54606", "43376", "43225", "11702", "6776", "11260", "25779", "31777", "53072", "32533", "9075", "26666", "56978", "19506", "861", "63937", "61751", "21058", "2327", "44204", "67993", "48652", "3726", "31142", "2771", "57321", "58593", "65190", "35941", "42953", "19680", "41427", "69519", "19984", "69532", "23118", "67510", "14917", "18907", "13404", "47291", "5157", "11793", "18585", "40323", "5341", "50280", "63220", "38817", "68891", "6599", "51545", "48289", "55966", "49163", "8507", "65675", "19850", "65066", "38946", "3823", "7393", "60446", "16008", "63037", "23602", "46664", "33012", "12569", "42579", "58709", "2576", "72786", "4317", "58651", "49353", "22199", "53020", "24718", "6559", "25653", "25932", "9384", "10994", "57463", "6772", "55821", "53831", "29085", "45113", "45857", "51499", "55804", "54933", "38478", "46121", "50484", "23981", "606", "60955", "56817", "26508", "43997", "34552", "11501", "40204", "15376", "35643", "65655", "57551", "22678", "39745", "22322", "38785", "43048", "42810", "34002", "2737", "31873", "4694", "16792", "26461", "40224", "17301", "55293", "70920", "18730", "4904", "23832", "67158", "66277", "47975", "416", "60767", "22052", "30696", "39106", "20806", "17080", "26604", "39528", "51260", "43193", "52583", "13691", "42958", "15949", "33915", "1542", "29586", "27877", "45678", "52789", "63105", "69600", "56624", "34436", "70191", "12729", "17964", "69565", "67498", "35109", "59764", "35472", "71043", "18506", "34386", "36623", "67175", "51509", "38228", "45760", "42674", "2421", "40057", "58074", "26505", "19026", "56257", "28791", "47165", "46609", "42234", "61130", "50117", "13561", "56518", "15952", "23858", "3862", "70022", "48201", "65538", "40701", "45637", "28448", "29901", "9516", "10935", "3266", "7609", "71354", "61878", "67918", "2428", "70468", "22560", "11649", "19230", "55938", "25490", "62470", "60134", "35754", "35957", "43958", "41035", "41864", "17209", "29832", "2260", "28717", "17162", "29488", "17095", "39238", "13878", "24872", "21627", "4510", "57719", "52578", "63984", "17644", "18500", "26382", "27184", "35146", "47237", "47364", "67690", "40116", "7908", "32781", "58304", "25310", "46458", "34353", "32993", "2261", "13559", "2154", "53891", "65636", "64447", "45666", "49253", "59931", "970", "65415", "50672", "29738", "60492", "48964", "65619", "65600", "61255", "23604", "40024", "67113", "31623", "72563", "31609", "28548", "66466", "50174", "3555", "41530", "42298", "6098", "29511", "10878", "63770", "30737", "27893", "44907", "40885", "63473", "58737", "49243", "55474", "33898", "37543", "11799", "53603", "25304", "70291", "11247", "73193", "18903", "2109", "29434", "14372", "12121", "45976", "41103", "22848", "54353", "3462", "8971", "10468", "73014", "63339", "47889", "12535", "3128", "4999", "72479", "50987", "67387", "35561", "45381", "17899", "58974", "59127", "35826", "2144", "12604", "36105", "70829", "25867", "39681", "54065", "39678", "13611", "18233", "52760", "11203", "15833", "11934", "52372", "27048", "31917", "52378", "29195", "60923", "55777", "48853", "48609", "51526", "71841", "8804", "39802", "35645", "7559", "48875", "4438", "41388", "53199", "51101", "24413", "62484", "26170", "7583", "27529", "23995", "48841", "15717", "54683", "58502", "64807", "58925", "67304", "18972", "20818", "48736", "5180", "34327", "3488", "44155", "32082", "34257", "22029", "14393", "30453", "29408", "55592", "64684", "11101", "65748", "6438", "69385", "9912", "16600", "21488", "45712", "61956", "57537", "46028", "38267", "54679", "17950", "11493", "48322", "49218", "40999", "63291", "899", "16939", "69968", "19046", "65817", "716", "21441", "7153", "27255", "56477", "72966", "52045", "53777", "14209", "8260", "21106", "46417", "40774", "35069", "43627", "18663", "8685", "24669", "61312", "41842", "64947", "64671", "18348", "50692", "33159", "21840", "26996", "50106", "35385", "37230", "12170", "61289", "62012", "31648", "58243", "50358", "4509", "43395", "18892", "72445", "1488", "46475", "46509", "18680", "52017", "67654", "8352", "60027", "17605", "22797", "64737", "1233", "29379", "13732", "43310", "6980", "16827", "5208", "36580", "52513", "58934", "64714", "10955", "39763", "15967", "20245", "14708", "55521", "44268", "31883", "58491", "68607", "22327", "30435", "13749", "54490", "62943", "21525", "53305", "40964", "1835", "38635", "57403", "3346", "56842", "36357", "14527", "53269", "37597", "47245", "53735", "42258", "69494", "64936", "51389", "7590", "1070", "12458", "38106", "40531", "42834", "31661", "70989", "24263", "20187", "28946", "13627", "49244", "59218", "20282", "54060", "18706", "39327", "31642", "39632", "36304", "14321", "58127", "1719", "5071", "2280", "23507", "44108", "37953", "34699", "15609", "40221", "2589", "61986", "83", "14920", "21030", "18636", "36165", "61506", "61104", "15062", "26466", "9605", "10901", "5846", "17643", "52955", "71633", "2781", "41012", "64675", "34349", "71339", "57364", "13207", "40522", "22865", "45895", "1866", "20664", "16638", "4831", "21890", "50003", "8547", "18708", "47059", "52589", "23646", "12276", "43551", "168", "9442", "53906", "52387", "68032", "24975", "62977", "1285", "46474", "13482", "66472", "7513", "2380", "21371", "51424", "28598", "13953", "4211", "65763", "40442", "22301", "20685", "48160", "59250", "33073", "72180", "46493", "71444", "31281", "58154", "36934", "21919", "2277", "36276", "37987", "23587", "52851", "56816", "28930", "48052", "8828", "15337", "66997", "35706", "29908", "30203", "64578", "52538", "66650", "68645", "70362", "14716", "22726", "2777", "42556", "66812", "41982", "69353", "10326", "27592", "59492", "12463", "46263", "4991", "60642", "14025", "23055", "36950", "34758", "19405", "14741", "24994", "55150", "28566", "21111", "5629", "48954", "70419", "4861", "41888", "42781", "30212", "24054", "70706", "18596", "38023", "61446", "43655", "44643", "15245", "18429", "30383", "67107", "50760", "17451", "58320", "7105", "69690", "72241", "14950", "22000", "22430", "1445", "57833", "65702", "52559", "70856", "43470", "1457", "1269", "38855", "42569", "46626", "49040", "35168", "29074", "47966", "68610", "28348", "42706", "21600", "33909", "66586", "66797", "10798", "66730", "13359", "50020", "59361", "13160", "26883", "52478", "32049", "68966", "834", "40601", "64079", "50870", "25302", "55154", "5557", "22499", "21668", "18497", "17312", "34271", "27678", "65807", "28897", "1237", "16485", "49792", "33249", "55114", "26314", "46035", "24235", "71284", "72810", "71417", "12871", "65112", "41465", "59074", "48578", "38121", "48884", "8921", "48083", "36905", "26559", "32273", "29059", "12175", "25473", "38884", "6130", "63802", "1180", "1595", "30477", "67855", "4028", "53605", "43924", "10692", "13075", "39845", "49625", "62590", "5782", "14103", "48793", "58011", "51328", "71403", "10141", "26355", "48473", "66497", "53530", "53778", "51105", "51553", "60678", "10557", "71900", "4535", "57526", "38159", "33155", "45387", "50612", "53476", "26662", "49120", "39874", "68010", "66955", "5065", "21658", "63636", "16519", "24988", "67329", "28911", "7662", "34468", "33913", "18399", "46710", "24893", "34422", "18860", "48393", "30407", "2059", "4711", "20259", "67346", "1215", "18931", "17353", "68538", "12724", "252", "15659", "4241", "58616", "48518", "16545", "8913", "67948", "5827", "49254", "14390", "50509", "41704", "32904", "24076", "52962", "21322", "21484", "67861", "35587", "3124", "11254", "46489", "7226", "64450", "66789", "69867", "25106", "55313", "16540", "21422", "27345", "64663", "50471", "71830", "23662", "5195", "29707", "51619", "48552", "15018", "56773", "34446", "17142", "22682", "29361", "55853", "28867", "6852", "48024", "9207", "956", "65464", "43976", "24902", "36694", "50261", "3087", "40131", "32952", "13917", "41305", "39607", "32348", "67403", "10653", "54228", "49092", "56558", "34280", "36822", "52959", "28838", "17363", "28703", "46200", "62336", "51121", "50178", "24063", "67104", "13298", "61569", "53566", "7658", "30810", "63897", "15769", "3825", "35749", "9324", "68163", "6659", "49012", "32958", "58051", "59778", "30443", "1795", "24273", "2125", "36254", "37438", "61842", "59634", "35048", "55216", "14860", "62474", "38383", "66887", "58180", "22780", "48905", "52128", "46329", "38310", "42235", "64190", "7428", "40763", "61494", "6289", "63108", "12925", "49788", "10497", "26683", "39850", "39035", "70415", "62621", "70982", "57143", "43697", "2852", "35899", "47233", "72990", "63106", "19384", "37370", "4520", "71179", "67007", "63570", "65429", "47107", "35324", "25817", "63818", "13575", "24319", "15866", "44921", "44597", "66618", "26087", "18368", "37174", "55813", "28963", "454", "48493", "14257", "24657", "22938", "12993", "14391", "15237", "40421", "48045", "7141", "60662", "69196", "28606", "27527", "16520", "66180", "39097", "56418", "18464", "7459", "27612", "29944", "48500", "59461", "64701", "18557", "66694", "20712", "20472", "1360", "66448", "57693", "64821", "13906", "4640", "25702", "46910", "17397", "53241", "34868", "4864", "24914", "31548", "337", "24635", "64538", "15324", "53226", "31496", "47393", "49434", "22007", "20893", "3486", "54701", "27003", "33805", "58175", "43202", "12222", "51921", "52714", "53038", "63402", "32531", "14040", "69439", "28351", "50764", "43673", "50821", "54381", "33367", "44282", "53912", "71843", "21463", "20308", "54103", "72903", "60784", "11478", "8145", "39943", "62870", "56579", "68566", "69507", "65711", "58451", "52450", "17880", "26447", "64868", "33552", "24103", "19222", "42786", "62872", "21711", "26132", "8728", "12667", "46233", "64420", "26306", "13511", "60582", "1117", "72406", "39550", "3627", "27863", "14925", "57819", "61732", "5333", "43502", "24444", "39448", "10800", "30537", "71962", "37455", "22625", "43536", "14542", "40193", "4627", "61200", "63738", "63243", "70478", "38354", "4883", "9373", "36343", "23344", "26527", "35455", "34875", "41138", "7090", "50243", "29961", "16201", "39786", "31349", "50158", "72315", "42014", "38243", "55890", "68830", "49531", "8028", "50181", "64409", "64617", "34954", "54363", "15557", "20638", "32522", "38341", "8487", "34743", "40677", "71807", "38180", "43118", "15220", "3659", "25601", "33853", "25133", "25292", "64030", "30015", "67877", "66024", "35004", "45804", "17390", "51814", "69260", "45379", "36129", "65775", "53833", "66349", "69949", "62830", "70649", "26663", "48538", "63013", "71448", "57247", "26289", "38861", "5597", "70283", "54263", "72520", "43641", "68377", "24591", "72692", "56302", "20274", "48508", "5746", "40703", "21638", "19273", "48915", "14037", "25524", "51111", "71089", "26598", "7457", "46067", "27229", "7585", "67675", "18630", "43904", "50167", "69951", "68733", "20159", "55141", "24084", "27922", "10682", "17422", "49355", "3527", "33426", "9981", "23484", "12460", "10545", "5737", "19134", "26682", "31173", "61987", "72351", "70875", "5138", "29496", "47910", "45129", "56874", "10243", "60125", "69578", "527", "1976", "40021", "8356", "3040", "39091", "56081", "37933", "12028", "53", "50695", "18947", "6855", "41596", "40669", "9953", "4380", "37885", "5351", "69065", "57232", "32798", "58887", "42938", "34290", "19269", "46445", "51976", "35461", "56284", "29111", "42338", "28953", "44730", "10100", "16386", "39430", "31563", "11351", "29704", "44529", "64611", "64212", "47762", "15919", "59961", "44790", "22224", "33407", "14656", "46448", "6320", "1594", "55723", "9838", "46907", "52995", "72649", "5162", "3693", "16249", "24699", "6992", "5402", "44010", "68741", "5331", "2292", "29555", "66080", "15042", "22086", "48324", "30786", "44151", "33391", "71698", "15834", "5994", "72626", "24367", "27033", "6894", "40615", "70924", "48714", "1223", "11076", "47211", "49513", "2970", "18300", "39428", "34760", "53873", "16275", "71519", "4292", "72877", "67871", "54984", "55564", "72340", "35716", "14279", "22380", "71158", "3083", "66996", "38589", "29172", "68358", "57035", "62426", "28347", "64394", "27167", "28772", "21264", "58341", "5095", "71572", "32630", "3463", "70690", "53976", "1561", "33190", "35632", "23794", "57901", "59335", "27479", "60323", "15752", "4187", "49088", "33196", "70948", "1229", "58979", "69807", "29340", "71747", "18791", "19431", "45289", "12691", "13897", "7696", "65085", "71626", "2944", "14444", "25492", "19543", "45434", "24920", "33447", "42114", "1273", "39081", "6269", "40410", "36390", "50532", "25666", "3025", "40511", "29602", "8818", "19476", "3275", "66108", "31353", "51387", "52187", "70927", "43404", "62567", "18277", "51655", "36545", "22440", "70853", "21174", "67944", "41906", "61632", "71827", "29056", "31078", "51972", "36958", "27762", "40168", "26335", "64505", "32108", "61567", "14812", "26432", "13130", "67509", "6562", "44303", "36306", "59008", "34697", "43443", "42952", "43555", "19082", "17857", "42725", "70090", "11975", "16091", "60112", "10278", "57393", "13471", "71649", "65596", "40487", "13729", "60768", "71663", "13295", "66749", "44165", "24579", "54792", "29940", "15576", "14230", "26373", "813", "33342", "34383", "53860", "34091", "20517", "30539", "37368", "19679", "16503", "5571", "56357", "11781", "23679", "30861", "11264", "39292", "7124", "814", "5078", "25758", "48735", "42140", "24073", "2239", "53588", "42963", "6667", "34886", "1097", "392", "60051", "6676", "68026", "30519", "55887", "13791", "31441", "43843", "64333", "15927", "27827", "20384", "49295", "48071", "2694", "18824", "36416", "20387", "70570", "14836", "10352", "59", "47229", "969", "698", "67740", "44466", "30380", "72355", "1567", "9950", "71302", "35538", "45386", "68496", "2003", "41362", "34131", "69010", "12174", "54882", "5773", "1586", "3900", "44729", "3453", "278", "55660", "38351", "16760", "6970", "61790", "67194", "19201", "51294", "14053", "64695", "54358", "37177", "10485", "9483", "1621", "66038", "26150", "6114", "54906", "24228", "21945", "14748", "24155", "27642", "59981", "35894", "22157", "25429", "34012", "53880", "28346", "47760", "55883", "38411", "25150", "30020", "23302", "9858", "7617", "39581", "57391", "11377", "24787", "71981", "1620", "44683", "40746", "9708", "66423", "27373", "22669", "18412", "39498", "45560", "70370", "61449", "70217", "49797", "70205", "43917", "19039", "22855", "55217", "23785", "33488", "65077", "40432", "35486", "68853", "62920", "51435", "54355", "35304", "17040", "49101", "68021", "13666", "8358", "56682", "56661", "62541", "16768", "1135", "1532", "62038", "15085", "14778", "19438", "16955", "16425", "13930", "27307", "27808", "22218", "11217", "32327", "64708", "31577", "38588", "18677", "27253", "21896", "36849", "63432", "16381", "67672", "25370", "2368", "18280", "26658", "59907", "59743", "28047", "51436", "54859", "24630", "20193", "21883", "57633", "31879", "9006", "14497", "12708", "67886", "50381", "24656", "53628", "18262", "6886", "38968", "23143", "66511", "55266", "16188", "24503", "69901", "4860", "51238", "64000", "41090", "61036", "67185", "23462", "62453", "32923", "56362", "51128", "66250", "39795", "47917", "4234", "72679", "68070", "1093", "28021", "70137", "11306", "72918", "20483", "43045", "69894", "57009", "67366", "50553", "49714", "48903", "22257", "29107", "60045", "6178", "46659", "68078", "40292", "11168", "18453", "7133", "68582", "68989", "47905", "3606", "64382", "17306", "64760", "28835", "18701", "13055", "67424", "22172", "32549", "30426", "53492", "38985", "2019", "3012", "41513", "32228", "42458", "40692", "18772", "58158", "67367", "43071", "7259", "62644", "55902", "42893", "60993", "32481", "39437", "9298", "47790", "21457", "12310", "46786", "42459", "22264", "18297", "23863", "60556", "19531", "41926", "11661", "1158", "51043", "29931", "48773", "18613", "1910", "55575", "58238", "2086", "7611", "12179", "40121", "65802", "43348", "56955", "64647", "32999", "31575", "16616", "11262", "3612", "23800", "70383", "44607", "9591", "26573", "66981", "25328", "21297", "59183", "20171", "65192", "42330", "470", "39127", "12187", "67151", "70530", "3674", "20918", "28557", "7627", "928", "9592", "40984", "12839", "70094", "14763", "5640", "39034", "54243", "40950", "56382", "10773", "64173", "18631", "10961", "11151", "16283", "56489", "27538", "52953", "55979", "61817", "22927", "14602", "44562", "30553", "3785", "32897", "4965", "25026", "30298", "65389", "25959", "62612", "34916", "72959", "69095", "29853", "17383", "45871", "30935", "48551", "68866", "64704", "47180", "9922", "30646", "59606", "50078", "24293", "42866", "450", "70905", "7031", "66256", "8925", "16802", "44570", "72493", "63968", "6173", "42505", "14197", "60089", "28279", "32795", "57586", "14834", "43610", "63353", "24003", "46180", "26752", "1263", "20959", "10590", "35387", "44009", "33980", "18912", "5694", "69777", "6941", "10981", "15722", "4451", "18107", "59986", "66601", "63307", "633", "31044", "52373", "1278", "63944", "4479", "65395", "63085", "45158", "14544", "37845", "52413", "18350", "68235", "1038", "68553", "6798", "16413", "33796", "47333", "37081", "7306", "65172", "9549", "32393", "55767", "32962", "66453", "52331", "63823", "15533", "54413", "45969", "33348", "31788", "28732", "14155", "67704", "58266", "2493", "50309", "45438", "26398", "13544", "60837", "16322", "43238", "12268", "37216", "5931", "15218", "45435", "61931", "9929", "67112", "35803", "11111", "42302", "52763", "28891", "28944", "21048", "10366", "61881", "29270", "53001", "23565", "22621", "8125", "11280", "44990", "18472", "43005", "67416", "58907", "21244", "63962", "2300", "27918", "46398", "20227", "15058", "54773", "60387", "44195", "62332", "34153", "27025", "23676", "65151", "68448", "13645", "32358", "27909", "60123", "14689", "65683", "13876", "46794", "37793", "49226", "18589", "21144", "61420", "5806", "55040", "3396", "3968", "65404", "6404", "22420", "6832", "71684", "9035", "14695", "20768", "22837", "33323", "13437", "45490", "49518", "54285", "61697", "3885", "54277", "57795", "3192", "23950", "51675", "63405", "32694", "51706", "61536", "67553", "24584", "42255", "48101", "30711", "58392", "16432", "14897", "73173", "46268", "68813", "14193", "53002", "10465", "50276", "20051", "68321", "41813", "5194", "23085", "51756", "71423", "41411", "2765", "18673", "9846", "42209", "21186", "13532", "15422", "50586", "1920", "43427", "17700", "43324", "64400", "31474", "41579", "7733", "50486", "18689", "8415", "2211", "54194", "47824", "7636", "59630", "16474", "68014", "11175", "63420", "62360", "40547", "12291", "46556", "38740", "12", "56886", "34681", "56447", "63366", "10681", "68894", "11462", "64981", "28015", "4403", "32331", "28479", "58113", "19261", "67614", "35254", "70389", "40596", "27657", "23431", "62354", "65044", "17289", "41527", "20923", "40477", "63724", "5731", "13946", "54408", "67146", "20885", "29515", "70864", "50569", "17732", "62377", "48839", "28994", "31103", "28591", "69438", "22637", "26144", "821", "11430", "62304", "7287", "36219", "40087", "40327", "10475", "37689", "1165", "25788", "53362", "21116", "39363", "40357", "68231", "64955", "66858", "24206", "30381", "34808", "72593", "8455", "19629", "15488", "49827", "3993", "64144", "56502", "56450", "65045", "58171", "46555", "42534", "71658", "68746", "54675", "12917", "42608", "51941", "69015", "10453", "36297", "61453", "8940", "37090", "21540", "34485", "37847", "9264", "65465", "51003", "10973", "73231", "17756", "45997", "43504", "56442", "17165", "37542", "32239", "29031", "29549", "72172", "38821", "43218", "69959", "68498", "68503", "54004", "26718", "28813", "27669", "6606", "21644", "9335", "57312", "40673", "25314", "69318", "45933", "15227", "69515", "30732", "49500", "71390", "28056", "38208", "42163", "65931", "51538", "56233", "51636", "66143", "54789", "52556", "58533", "53587", "13146", "54214", "55754", "27625", "35748", "11638", "67204", "29388", "6760", "62221", "32674", "72515", "12559", "6459", "3481", "5716", "62659", "69398", "53345", "28592", "23309", "58470", "30367", "55927", "47091", "20093", "11423", "6885", "2752", "42631", "1556", "54603", "43178", "26254", "58456", "54356", "62517", "56630", "19785", "18392", "1141", "60343", "71474", "8798", "72961", "54574", "42513", "31381", "9132", "16725", "10655", "43824", "57238", "57026", "34088", "9084", "61816", "67658", "22013", "17432", "55380", "59277", "3621", "27086", "50754", "71647", "39543", "29537", "60820", "44483", "42984", "62129", "63384", "60357", "24299", "30930", "34592", "49727", "28603", "20930", "3568", "39718", "65718", "30736", "46315", "33701", "43328", "24389", "58932", "24085", "7009", "66783", "23308", "39656", "62097", "23909", "62741", "30016", "13775", "72736", "37511", "19668", "56720", "32418", "9229", "66513", "44634", "58526", "37166", "63891", "36411", "3049", "33964", "59100", "33265", "60719", "21337", "51172", "30982", "5280", "44181", "19256", "51994", "29999", "15999", "10762", "50527", "39646", "2142", "52430", "70638", "67737", "36668", "29808", "2258", "1759", "70691", "31452", "9677", "10925", "60421", "17915", "53380", "9515", "51020", "38958", "20103", "52847", "53887", "24385", "40876", "40105", "8042", "64250", "19247", "27797", "67748", "59497", "19469", "52141", "658", "50948", "59465", "30337", "13628", "38794", "57075", "57629", "57234", "72532", "32309", "51606", "23840", "25282", "20398", "29343", "36119", "3895", "26743", "60567", "39429", "48005", "46513", "38629", "66506", "7380", "52706", "55372", "72928", "40815", "53526", "41941", "46323", "8869", "18173", "63820", "70597", "39070", "58770", "3591", "37540", "19199", "52740", "18924", "11675", "3763", "48867", "44814", "20686", "28261", "25309", "11513", "39378", "46079", "70016", "14855", "56366", "8298", "24182", "46806", "7838", "49440", "38105", "4610", "68883", "29321", "20002", "63836", "21317", "69509", "13795", "11098", "6684", "3305", "7985", "55081", "55094", "62178", "29528", "24133", "44531", "11924", "14015", "62939", "17795", "53559", "9257", "65545", "49970", "66835", "1228", "4222", "71577", "7255", "70502", "23818", "13769", "13204", "47203", "54733", "9698", "10922", "35401", "40201", "9675", "29763", "55965", "16910", "25250", "33359", "3741", "33411", "26584", "41327", "31008", "958", "28272", "8226", "37788", "60785", "27175", "39932", "39601", "44673", "34968", "66005", "49041", "37860", "35470", "69154", "53064", "62588", "10624", "14940", "43476", "66492", "6158", "54313", "41990", "66584", "17920", "33731", "28761", "66610", "22310", "31313", "59063", "72796", "23515", "917", "14060", "68354", "7601", "24611", "37298", "25360", "57125", "6187", "41376", "18501", "34415", "65955", "10538", "35247", "9805", "39649", "30884", "15131", "16641", "52317", "66909", "27706", "25827", "49742", "5723", "46895", "44242", "43614", "43898", "50256", "48013", "31165", "10924", "44824", "44622", "7538", "15763", "60914", "55802", "13885", "49321", "69533", "15552", "63172", "22547", "27654", "62495", "69403", "56098", "302", "54085", "32874", "52392", "69349", "26567", "10665", "33068", "35431", "55317", "30106", "73011", "8434", "16693", "67195", "3757", "18301", "14963", "60179", "40414", "33330", "47249", "11440", "478", "64075", "47659", "49761", "9489", "13090", "47269", "26268", "36696", "45965", "5904", "38677", "8437", "49946", "42436", "4036", "69373", "2981", "49717", "63114", "44760", "35839", "60074", "12196", "4742", "22624", "19010", "12714", "56338", "2332", "69081", "19091", "29064", "49893", "55122", "38524", "59033", "32641", "10738", "49545", "27780", "54503", "13949", "23509", "58740", "14674", "71061", "24190", "54188", "9935", "19338", "65412", "39553", "34602", "39776", "30963", "26375", "26978", "36229", "16469", "40290", "63742", "7816", "53011", "52834", "29103", "14828", "1123", "44539", "39399", "21431", "27757", "64873", "5948", "5817", "4734", "9236", "42933", "29716", "69825", "51964", "44608", "43314", "48293", "56003", "39272", "66299", "62523", "3391", "65858", "48706", "25544", "69527", "72585", "3515", "58405", "47258", "56535", "22388", "38413", "9112", "31581", "62843", "14203", "36997", "5135", "10193", "65794", "27456", "9106", "27962", "30689", "66862", "34524", "46529", "38272", "45545", "45352", "4219", "57887", "37609", "24716", "71526", "37580", "20087", "193", "35862", "50225", "33123", "59062", "67964", "10657", "62394", "27682", "63488", "21377", "20759", "59674", "26983", "39207", "12961", "67726", "7540", "10224", "63202", "68087", "5816", "65624", "23052", "56653", "60393", "67046", "22802", "33139", "22834", "50077", "38035", "62194", "39475", "34395", "64915", "45311", "10013", "27197", "27866", "49965", "51863", "52237", "5375", "53422", "71706", "35310", "9928", "12964", "58977", "56342", "5370", "39456", "21567", "67866", "66449", "43446", "49216", "31591", "31587", "9868", "11681", "29227", "11329", "10664", "61768", "57643", "61949", "32465", "9613", "49875", "2080", "11637", "42552", "15580", "1122", "33298", "60205", "69926", "21469", "39835", "20489", "72189", "22874", "52608", "4011", "24325", "68135", "9404", "71589", "40321", "62413", "58412", "1819", "48090", "67485", "29813", "33271", "71372", "49701", "37079", "4244", "54093", "2167", "53821", "27143", "31108", "6803", "71047", "32914", "11610", "10856", "329", "37709", "24244", "33303", "23533", "16720", "1830", "3983", "63340", "12173", "37980", "50011", "63789", "41219", "4387", "69890", "49024", "41402", "30644", "49201", "67340", "43853", "21515", "43992", "20958", "25474", "52076", "15727", "40169", "58403", "14432", "60209", "71864", "68388", "41869", "64048", "6021", "62418", "53916", "6946", "13271", "52640", "13301", "36016", "33934", "41814", "53318", "63639", "18530", "48249", "71214", "11437", "33663", "6755", "29563", "52808", "24848", "25528", "57708", "66959", "58235", "3648", "25572", "23171", "13214", "51786", "63627", "39878", "63277", "28333", "61603", "9812", "42362", "35676", "23692", "66941", "45752", "21527", "34237", "13706", "47615", "1010", "49704", "46657", "13839", "25825", "8080", "61054", "40902", "13372", "36993", "50433", "31324", "51641", "19909", "47639", "44605", "27293", "67923", "53451", "59849", "25014", "54300", "70897", "22916", "18966", "52118", "63206", "2240", "55458", "25985", "39659", "68833", "9244", "39059", "37651", "28699", "37237", "57941", "1513", "55800", "28370", "52262", "31019", "4082", "17661", "58851", "26556", "11540", "13569", "33526", "21532", "70537", "4295", "21328", "69660", "43013", "35320", "63950", "16516", "17684", "20246", "66397", "64292", "29310", "965", "9071", "44024", "29079", "16446", "668", "2851", "42363", "55148", "20875", "50895", "21975", "49799", "18386", "17174", "49103", "34639", "4205", "66114", "24145", "9116", "67598", "5023", "7597", "58108", "62239", "52617", "16408", "9620", "24954", "38167", "70902", "16340", "57342", "25972", "21628", "25167", "558", "51676", "65239", "36279", "58219", "4309", "41027", "47522", "31876", "11421", "5777", "55419", "65262", "18353", "56675", "27712", "29658", "67967", "7192", "64407", "42550", "60406", "31294", "11495", "41007", "52", "56976", "6388", "29747", "36983", "70360", "32492", "68240", "56968", "35307", "21021", "20363", "34605", "26055", "15204", "31505", "44097", "3911", "49588", "29162", "29811", "36055", "33797", "66274", "7979", "58206", "18324", "12695", "23225", "66598", "9174", "55930", "35702", "49638", "23504", "56150", "20770", "16286", "48210", "72578", "22002", "24797", "27368", "42482", "41375", "25940", "70863", "18998", "32073", "18023", "20863", "44647", "25889", "70650", "21210", "1731", "176", "54891", "61823", "52084", "15894", "100", "72054", "11062", "48804", "34770", "45153", "23053", "5583", "11002", "14942", "14999", "14187", "53301", "46337", "69208", "48653", "3569", "55364", "45148", "58184", "39136", "66137", "7474", "39907", "5379", "36205", "32821", "64484", "69694", "11311", "21218", "69323", "36533", "26431", "54889", "2674", "2122", "11219", "8796", "9970", "47429", "1064", "10055", "30918", "6993", "28779", "60776", "15381", "51071", "52264", "12815", "9871", "45755", "28545", "13047", "46125", "63284", "60626", "70972", "7081", "10478", "7326", "23893", "66665", "43637", "60475", "62603", "53964", "25472", "63014", "12878", "6354", "13291", "27386", "26938", "36614", "54147", "69953", "7989", "55252", "10241", "8732", "20061", "18163", "66061", "63100", "36179", "10608", "33636", "20287", "57053", "63161", "7294", "7519", "35886", "49928", "32352", "2162", "65705", "70851", "14009", "18878", "4247", "70969", "8092", "26733", "26793", "44661", "63256", "10697", "72716", "67543", "63418", "65824", "2637", "68824", "12410", "36104", "9177", "72694", "15186", "32386", "45630", "9472", "42903", "7821", "6358", "7129", "47253", "16434", "43999", "34816", "44928", "36723", "52503", "43787", "64790", "6426", "35429", "15466", "49950", "885", "11209", "25098", "72652", "26290", "9357", "38948", "12414", "8476", "20657", "3166", "25554", "62472", "73075", "16345", "3345", "45039", "54671", "56217", "62047", "27070", "14290", "69484", "57188", "21435", "51239", "20481", "37787", "50266", "32584", "37065", "63611", "16194", "26974", "45009", "26215", "6795", "21424", "62818", "69309", "34107", "33793", "45912", "53549", "32281", "2492", "46225", "63070", "48727", "53780", "23550", "68132", "12720", "58244", "23227", "63869", "23938", "38086", "44137", "63852", "8360", "15321", "35268", "52931", "5198", "39502", "59010", "45320", "10165", "46404", "50762", "42434", "4097", "26560", "50160", "26766", "3685", "42930", "41642", "15242", "12420", "1787", "16427", "43770", "17074", "50925", "42203", "12942", "22716", "9251", "13776", "47602", "61755", "573", "32759", "39800", "11409", "63141", "69517", "47079", "63946", "46320", "61774", "67969", "27264", "8322", "3132", "9857", "72765", "59006", "22368", "50474", "30991", "48548", "42502", "38081", "5935", "71080", "45416", "24300", "54165", "3799", "63862", "19983", "68419", "64607", "40347", "29041", "29863", "25576", "69563", "55237", "55006", "57683", "56232", "63659", "50851", "60984", "42749", "24331", "58065", "56031", "7814", "26326", "25449", "47647", "13105", "7447", "48075", "53111", "60936", "20426", "43043", "14677", "34102", "31305", "51147", "60250", "63651", "32432", "55787", "51823", "16346", "61244", "25048", "7410", "33835", "60082", "26737", "47137", "13594", "27991", "53589", "64971", "70480", "13333", "46552", "6806", "64169", "63722", "60697", "5463", "38857", "14857", "57349", "38623", "41218", "37809", "32959", "3821", "48031", "22014", "28565", "21884", "62965", "47822", "72726", "41006", "55899", "45454", "70633", "61894", "12397", "3258", "3661", "27154", "60199", "56764", "57457", "12497", "54715", "61784", "17064", "32736", "707", "46133", "24295", "17018", "9332", "33287", "62625", "51729", "16154", "9225", "21493", "71794", "6840", "72233", "17465", "64265", "39355", "51077", "31435", "67764", "64809", "55155", "26223", "47677", "38225", "11776", "65715", "24135", "7355", "32303", "34921", "36705", "25188", "26052", "36302", "47828", "27835", "72771", "46492", "71375", "63758", "70944", "72892", "70738", "30326", "16399", "43337", "58056", "31429", "55817", "63469", "19296", "5113", "48901", "53054", "66520", "42405", "21288", "18079", "61815", "17260", "26729", "31549", "41114", "55294", "13891", "47441", "66336", "12496", "59957", "9573", "59835", "7220", "31711", "34560", "28090", "63396", "7812", "45612", "62525", "54513", "54148", "28898", "57117", "29878", "52537", "28192", "9693", "11617", "17618", "64005", "58870", "37475", "62967", "47940", "1283", "28991", "58445", "42500", "4106", "73015", "44896", "66825", "1276", "33336", "7240", "57665", "50440", "15920", "34624", "71149", "47581", "65206", "59730", "41537", "61400", "48227", "22083", "2581", "8538", "66840", "19163", "64056", "56503", "28208", "32550", "52403", "35280", "11531", "51766", "40062", "46177", "44681", "69251", "69390", "65994", "69790", "34409", "9743", "12786", "9434", "31756", "52727", "48342", "19400", "38965", "48753", "26337", "61323", "42524", "9027", "20319", "6054", "66682", "1095", "14402", "13006", "27354", "18601", "51735", "9364", "39243", "43444", "52793", "38369", "25039", "42090", "34141", "33030", "21363", "62976", "10423", "7204", "62078", "20841", "46653", "22479", "47763", "71671", "13540", "60335", "60014", "48392", "50662", "51608", "44358", "47015", "46396", "52480", "50711", "6466", "17091", "49171", "59766", "22570", "56818", "66872", "5552", "10686", "17098", "8005", "61651", "26535", "4952", "9960", "36456", "16184", "24287", "14486", "17615", "61101", "40858", "57947", "59481", "60215", "24354", "69001", "25574", "43192", "37834", "8646", "68444", "18369", "913", "48478", "42289", "54187", "38998", "20585", "12052", "69057", "50102", "23315", "16659", "12016", "38292", "6529", "15332", "62456", "21487", "51496", "1086", "12018", "56921", "42752", "26876", "5755", "48286", "14267", "43943", "46085", "28025", "22426", "3361", "63854", "42061", "48161", "9602", "31658", "10535", "54034", "31672", "67517", "15797", "47814", "31728", "69671", "43101", "54090", "56197", "39884", "47442", "5555", "7425", "33618", "40162", "8490", "22494", "13898", "52178", "35110", "51717", "11164", "63587", "13051", "19567", "45156", "12648", "68329", "3647", "45717", "20990", "58208", "35379", "33599", "46184", "47829", "33479", "58360", "36622", "60811", "16555", "19098", "2828", "52304", "42723", "45622", "56186", "11946", "1083", "68677", "24210", "18503", "49319", "24685", "30393", "5975", "52557", "48584", "4347", "63080", "36148", "14176", "22417", "53507", "37109", "22100", "603", "68646", "65029", "19218", "62507", "60763", "43164", "36687", "45299", "47073", "36217", "22374", "13005", "43919", "4379", "42372", "42947", "41028", "52188", "61590", "1763", "29973", "12568", "51379", "70766", "65060", "60664", "17219", "1130", "71298", "60922", "47041", "64732", "39405", "21806", "24390", "5468", "16988", "58466", "28613", "37745", "10618", "72709", "35946", "56355", "2153", "32344", "44573", "47597", "16116", "53607", "4815", "31519", "1082", "60040", "12650", "34923", "12860", "68532", "31171", "10678", "46050", "51242", "51278", "36532", "1351", "63580", "61840", "66499", "14637", "31399", "62382", "65571", "51638", "39765", "60854", "24937", "71993", "63513", "4138", "54014", "21673", "16781", "66474", "39665", "3932", "40945", "42685", "63463", "40571", "44173", "8523", "36492", "33516", "8519", "38232", "33872", "65014", "52928", "31853", "45498", "14928", "22216", "9692", "72145", "42672", "13586", "5718", "32153", "46654", "56993", "55063", "46597", "13347", "65488", "19263", "49049", "33609", "4645", "6111", "35973", "7342", "16150", "39044", "15614", "33044", "67196", "54229", "30387", "68410", "43237", "68820", "17167", "31511", "46015", "21068", "24811", "19047", "72126", "55366", "32370", "24625", "2041", "40241", "53423", "33425", "36266", "55270", "4021", "30382", "23637", "22056", "33077", "42783", "18742", "62637", "56932", "25179", "50132", "10171", "34960", "2530", "65968", "14024", "25728", "10418", "26085", "4398", "3671", "40284", "55647", "71053", "32384", "68803", "46981", "1293", "44650", "28233", "71293", "51425", "33743", "53634", "28321", "37510", "32700", "9492", "18921", "35919", "58876", "22602", "21880", "66582", "71134", "3607", "68103", "58647", "50772", "17743", "57821", "4949", "4547", "17693", "59256", "2893", "7925", "69280", "66854", "59274", "10416", "8875", "3843", "42489", "72024", "13474", "20910", "44957", "49815", "200", "46502", "31030", "48386", "60668", "64282", "771", "45534", "23342", "9152", "36125", "54771", "39700", "55726", "18305", "69077", "1518", "67983", "61839", "49707", "62254", "17114", "27630", "9604", "58161", "62873", "36562", "32775", "24143", "18661", "35923", "18124", "46501", "41458", "72827", "54809", "57942", "21067", "66273", "46403", "16097", "62098", "45726", "25231", "18144", "36630", "1726", "60348", "53667", "43082", "63575", "59984", "54347", "15470", "35262", "43775", "51476", "42394", "35991", "47468", "20295", "22442", "31292", "15515", "59491", "12776", "14371", "52462", "47756", "51611", "2486", "70214", "45083", "55049", "29335", "45525", "69100", "30716", "21796", "32617", "20158", "15738", "47873", "72870", "60060", "65256", "22299", "54904", "37854", "15032", "37107", "26780", "6231", "26463", "62468", "62480", "13138", "59913", "65727", "55727", "45343", "27528", "32808", "20113", "56189", "55584", "57746", "22004", "57925", "71454", "18988", "27050", "3640", "30418", "55263", "23179", "16594", "6265", "69029", "72399", "39838", "70590", "55675", "26242", "27840", "13228", "72183", "8651", "8518", "71621", "60460", "39836", "53347", "5315", "33916", "21739", "64646", "22840", "28149", "50927", "15537", "70132", "45949", "57402", "46017", "9100", "15835", "6324", "17619", "8270", "20842", "37896", "58160", "44019", "69311", "39483", "9392", "33581", "6061", "2393", "45378", "3311", "44864", "51247", "50670", "28649", "7999", "29854", "33082", "34204", "16245", "66760", "69230", "31884", "43884", "53774", "10811", "27399", "71086", "71412", "43618", "60847", "72109", "21305", "67597", "27774", "70815", "61796", "47043", "22075", "25903", "56185", "54985", "67209", "14292", "12938", "53732", "46406", "8405", "51711", "3507", "15415", "39854", "69170", "30894", "58388", "26577", "69502", "29870", "24963", "20324", "51845", "57416", "40753", "27608", "54310", "26914", "16556", "26384", "68296", "54070", "45889", "29010", "13501", "36301", "6929", "7739", "13545", "65819", "10153", "6329", "9764", "66773", "3727", "52193", "59259", "4402", "42929", "8777", "68991", "23272", "12734", "13622", "5124", "47577", "58602", "5778", "56445", "37521", "13135", "22631", "12339", "46137", "69865", "10869", "5521", "40340", "40597", "3800", "72616", "6555", "548", "61982", "31805", "63262", "39294", "52954", "14584", "13247", "68477", "63654", "28948", "58434", "28343", "25241", "11993", "35176", "70993", "30587", "7117", "27496", "46921", "34787", "71622", "7463", "52300", "27691", "41954", "66613", "37265", "4111", "63169", "14210", "14718", "57514", "36582", "72527", "68720", "71822", "70589", "57705", "62080", "27401", "21571", "7834", "9080", "16731", "1208", "65527", "3638", "11206", "64636", "17395", "47564", "68299", "38499", "65542", "57156", "44952", "17324", "45244", "35570", "66958", "47422", "36072", "7691", "15047", "12946", "21009", "60765", "18209", "62348", "65177", "32245", "28574", "20310", "36786", "1695", "20630", "4289", "34561", "50049", "11524", "5305", "9172", "55554", "12080", "64946", "48891", "30876", "7483", "55920", "9327", "38866", "19659", "37218", "70356", "72193", "55056", "21184", "9557", "27547", "35606", "6312", "22862", "55769", "62890", "65230", "3851", "20414", "34354", "6120", "62090", "19389", "56896", "58927", "48587", "48105", "11099", "22702", "18438", "24130", "60500", "41661", "70549", "54865", "27285", "44162", "62196", "59216", "64661", "23766", "67163", "18733", "35459", "19444", "39464", "44778", "9444", "22127", "59871", "42522", "47090", "37069", "45841", "51876", "64351", "48461", "3770", "46213", "61363", "27379", "13019", "20364", "5368", "5764", "72549", "17031", "71624", "50195", "67440", "19344", "39993", "7194", "12242", "64135", "47210", "67860", "46862", "71521", "32397", "3792", "64527", "65398", "29194", "58406", "22733", "4620", "61579", "19652", "21821", "40862", "69930", "5879", "69729", "51736", "47481", "28368", "16923", "56207", "59943", "58205", "58032", "34607", "30817", "1562", "26049", "34475", "44916", "62193", "67443", "57702", "44793", "38253", "49322", "44120", "41620", "61155", "72338", "29651", "49135", "28602", "63595", "77", "61630", "20609", "52177", "70777", "197", "3779", "20300", "39360", "13033", "43689", "13161", "4900", "10126", "59356", "48782", "25783", "72886", "48911", "19035", "38571", "14786", "22768", "33100", "17696", "20170", "40137", "70403", "3203", "13782", "41858", "64998", "45693", "72994", "11029", "65102", "16627", "3665", "61909", "43050", "6963", "62359", "17420", "34791", "58446", "7761", "9408", "314", "64673", "47751", "48098", "8412", "70301", "70012", "17630", "4735", "2407", "41365", "4554", "67811", "23300", "51384", "33431", "35462", "68400", "49118", "71347", "67649", "38692", "10616", "4096", "52703", "63574", "63394", "69305", "1526", "26750", "52752", "4238", "42926", "66933", "8402", "49838", "59591", "3544", "16542", "44325", "53303", "63530", "15251", "13304", "33745", "13577", "61827", "39967", "72062", "49966", "72943", "3521", "39533", "9277", "61004", "25114", "23088", "48269", "70153", "7904", "41247", "6870", "38018", "9108", "60353", "9841", "41030", "3330", "48262", "51973", "20083", "25609", "31017", "35015", "66082", "23755", "24050", "42461", "34565", "28658", "4948", "4584", "62704", "67230", "71094", "70031", "22928", "43664", "32415", "45412", "30949", "7518", "18789", "20040", "37584", "36918", "65208", "30052", "15228", "26901", "45096", "4786", "67650", "53997", "52933", "34339", "51109", "46127", "12091", "32078", "46408", "64170", "63735", "59078", "60671", "62497", "45001", "9966", "10905", "23310", "37690", "49288", "414", "32645", "22154", "42222", "43811", "22407", "68606", "69150", "7382", "67360", "28834", "11318", "8365", "38681", "13226", "7921", "68076", "70988", "61157", "16352", "67826", "34197", "9511", "70225", "44426", "31366", "48925", "55579", "47717", "68131", "20106", "30192", "71785", "16767", "64157", "31696", "12335", "34799", "12547", "59963", "45561", "73037", "41385", "52326", "43730", "38044", "45362", "36889", "17621", "53067", "37092", "63949", "36718", "29033", "66177", "28263", "523", "16157", "14282", "12545", "17371", "54485", "49325", "36446", "65959", "57496", "50941", "11806", "50540", "61789", "368", "46926", "39773", "10835", "17728", "16861", "16073", "49032", "47095", "3532", "3216", "26327", "54913", "32535", "26012", "31625", "28901", "48305", "12507", "42538", "44845", "4228", "67534", "46730", "15323", "35034", "31009", "36920", "53532", "62710", "53720", "36175", "44416", "20935", "46338", "65915", "52448", "12343", "56613", "5719", "15425", "6896", "10574", "27509", "62551", "14890", "9425", "11707", "4934", "45854", "9721", "20665", "18203", "10990", "38749", "21141", "29311", "61711", "72848", "55135", "16392", "15283", "1538", "30243", "3418", "52232", "8908", "65169", "49662", "15397", "8862", "17797", "43006", "32227", "59874", "47893", "32619", "32659", "36164", "19626", "62200", "56806", "18553", "6268", "72077", "69215", "46591", "19709", "4950", "40793", "37730", "8572", "11232", "43208", "23689", "45239", "21495", "21407", "13020", "24813", "66068", "4778", "25243", "15674", "24489", "61369", "19751", "14275", "67863", "17072", "35836", "29693", "44563", "7953", "55673", "30990", "49126", "40310", "38255", "1275", "25536", "39610", "58210", "30974", "55363", "9607", "26065", "69560", "51768", "40063", "44417", "18962", "9188", "8036", "37287", "37011", "20962", "68247", "39743", "52093", "60055", "70747", "50846", "70497", "51694", "41452", "40917", "66293", "41093", "57826", "67219", "36393", "31159", "55174", "21664", "24110", "48523", "21347", "60815", "1060", "42366", "21971", "72304", "46516", "39661", "28040", "57292", "11864", "61190", "3961", "66236", "8364", "2645", "26746", "54781", "56814", "65414", "10726", "2987", "9925", "54738", "56448", "5612", "19753", "14140", "37912", "72411", "44600", "58965", "67285", "72444", "2755", "21949", "23335", "68792", "63537", "36971", "11718", "57937", "53908", "54505", "66211", "46237", "33567", "36859", "73093", "41476", "25051", "66404", "3371", "51151", "29226", "63380", "40", "27513", "40770", "6966", "50521", "56635", "37472", "22164", "19685", "5047", "41801", "30850", "70486", "17840", "7923", "10093", "53960", "36653", "68166", "62217", "72803", "60206", "25379", "60980", "27818", "52596", "63360", "30170", "59357", "36619", "53453", "6668", "18327", "72337", "1146", "6657", "56526", "38853", "47890", "70244", "67548", "47190", "10054", "27694", "50001", "14814", "41149", "3789", "58473", "63642", "44578", "37277", "65111", "13184", "53442", "69794", "937", "37222", "7866", "60574", "47308", "29232", "63865", "57806", "41703", "39072", "8019", "34215", "17727", "54118", "65608", "71805", "69837", "1546", "26679", "23666", "14014", "40068", "15906", "36311", "5356", "51280", "3144", "14217", "2469", "64345", "32760", "26133", "38803", "7723", "52116", "63539", "7006", "62421", "5643", "13112", "32703", "15203", "32784", "67057", "1053", "62265", "20987", "19396", "35917", "17192", "28066", "52179", "25196", "2895", "68893", "57190", "55318", "57454", "25272", "30069", "46731", "63631", "49252", "17775", "71435", "71175", "53013", "46284", "62091", "23482", "37098", "27132", "19271", "25224", "6162", "20932", "60407", "4116", "48950", "10385", "45828", "6188", "30742", "57922", "51301", "39817", "25736", "72750", "62322", "65363", "30881", "59130", "41043", "65712", "62113", "37837", "52132", "36728", "43220", "70197", "31543", "34718", "36385", "42074", "14551", "30706", "61702", "61337", "50916", "51834", "49150", "46834", "51049", "13864", "30288", "63042", "56702", "73106", "59350", "53166", "12954", "37769", "58302", "20281", "42829", "8923", "61633", "57361", "24684", "26503", "17680", "59262", "9526", "18617", "7832", "13770", "22296", "32405", "14234", "8193", "33005", "38671", "57177", "52728", "10906", "36353", "24951", "53027", "61794", "54036", "39167", "23717", "15566", "38798", "44830", "55131", "8472", "48343", "72027", "42460", "64350", "53175", "65193", "47305", "35763", "36797", "53861", "48418", "50423", "7230", "6691", "56612", "34355", "29302", "18606", "3498", "52138", "28999", "44726", "56798", "62247", "35979", "25207", "58939", "3067", "34495", "61583", "56433", "60046", "25289", "61038", "45150", "29355", "66919", "15688", "551", "60137", "66218", "32250", "14511", "21143", "62718", "4846", "31320", "17211", "28541", "58582", "45677", "49273", "63655", "9772", "37480", "38344", "54482", "44718", "43143", "43869", "45116", "56140", "53853", "64485", "35904", "46328", "28128", "29661", "63824", "24352", "56219", "70685", "59230", "12324", "38333", "12914", "37993", "32860", "18", "69049", "14355", "33309", "3920", "70503", "72556", "51544", "9202", "48957", "65918", "20118", "72207", "37038", "26126", "10152", "37753", "23303", "56783", "40112", "50682", "43834", "2093", "10321", "15055", "50999", "29336", "60455", "54048", "42408", "53942", "38907", "17146", "65418", "32124", "72562", "19930", "63478", "69647", "50869", "36472", "47283", "10347", "8198", "322", "40075", "70366", "41353", "30485", "41700", "9370", "34672", "13332", "33460", "69123", "22743", "15511", "69770", "14609", "64584", "6752", "48568", "40307", "51347", "71033", "1557", "11965", "12518", "68449", "46491", "71428", "19893", "72703", "70467", "2855", "21605", "6677", "61304", "6502", "37103", "60550", "67932", "21238", "18406", "23923", "44052", "38708", "64525", "24868", "17061", "66549", "72455", "41836", "22877", "1385", "56681", "66246", "24687", "67655", "30175", "40566", "35596", "14591", "35410", "26860", "21227", "36677", "60057", "23442", "54303", "72621", "50526", "8991", "38639", "61016", "34486", "62787", "16707", "19529", "30989", "17182", "63971", "57512", "38380", "63480", "58202", "52753", "71695", "11056", "57630", "45314", "42212", "70002", "72321", "61203", "36693", "45931", "68725", "1929", "23815", "11137", "40634", "17433", "25700", "54815", "53970", "12139", "17254", "60382", "55958", "61522", "24551", "16077", "7789", "46942", "51584", "20413", "10972", "25265", "747", "72904", "58923", "27290", "36591", "12900", "8569", "7758", "27601", "28746", "70067", "21811", "15290", "9984", "61163", "54483", "9105", "7859", "40743", "58994", "9904", "57326", "14799", "67012", "23946", "20586", "42476", "46250", "41070", "33312", "47859", "10406", "23304", "12099", "25680", "47835", "15650", "47697", "31641", "38850", "1190", "23673", "9727", "13287", "50237", "38090", "6625", "24556", "26111", "20360", "41273", "30531", "14577", "37039", "18783", "54112", "53570", "27284", "18813", "34536", "15403", "25242", "31036", "58042", "20622", "47232", "4368", "49689", "62670", "50271", "30398", "2864", "65043", "59690", "70834", "29472", "8563", "33450", "54766", "43486", "61734", "70044", "32911", "43338", "3059", "18929", "12095", "13804", "19116", "59718", "13672", "69347", "65154", "5964", "1145", "44548", "65694", "1700", "45915", "33173", "51535", "33722", "42102", "61056", "52236", "64665", "10279", "63486", "69682", "7676", "16727", "18340", "32532", "36957", "12264", "46707", "66876", "55906", "32829", "27382", "28902", "67685", "14546", "67837", "12542", "69576", "67673", "39183", "11564", "17724", "17817", "60135", "14901", "50729", "8172", "17719", "4663", "62040", "69146", "51398", "56105", "41923", "67153", "67856", "39079", "46074", "50937", "34981", "28445", "64429", "20675", "57902", "42098", "22722", "27586", "8233", "19575", "52767", "57288", "43507", "16266", "38963", "27080", "6733", "58454", "65042", "36180", "31166", "71015", "52077", "20626", "61394", "61602", "34015", "22790", "25582", "30211", "23414", "71968", "54872", "55836", "6094", "10588", "59658", "41955", "13890", "67999", "37337", "41734", "9917", "54181", "37841", "51856", "4899", "12294", "25305", "70556", "46896", "5488", "2614", "57002", "30605", "5215", "10291", "34694", "9383", "33706", "11554", "2802", "71960", "39578", "63769", "49361", "27417", "36293", "34545", "11049", "37399", "72435", "33711", "1420", "40118", "10781", "39984", "49409", "28476", "13884", "31765", "58627", "61653", "58102", "65351", "30682", "16564", "46202", "53871", "68664", "5263", "12659", "70068", "26229", "7487", "17811", "26204", "52280", "44241", "17007", "46766", "70830", "2921", "41611", "44033", "22646", "49348", "36531", "6250", "63679", "72535", "9760", "41881", "29127", "39369", "47864", "50290", "56158", "69609", "19594", "53984", "50438", "65231", "59700", "35336", "29749", "19096", "39406", "68186", "52846", "46595", "4274", "41313", "2356", "21822", "8105", "980", "21587", "54742", "1402", "36262", "68797", "58330", "59781", "72594", "14967", "51420", "71580", "58378", "45536", "28270", "48635", "36929", "3019", "15546", "46367", "53981", "3828", "33788", "59659", "35850", "40549", "26138", "16181", "54772", "32877", "65247", "20188", "60164", "62094", "30915", "17016", "52379", "49311", "15642", "64802", "60515", "2621", "6060", "62487", "52532", "10043", "15857", "41201", "13193", "33484", "51246", "1306", "38600", "54890", "33794", "11213", "51183", "3078", "52521", "58146", "67102", "15117", "64567", "39934", "34374", "13144", "65295", "56022", "29278", "73110", "27925", "69169", "44298", "63701", "65281", "37832", "7546", "70404", "56414", "72874", "7341", "63694", "41659", "23827", "3723", "39730", "20867", "2739", "35884", "55414", "17914", "19603", "11308", "36923", "64319", "33944", "71025", "11177", "5396", "26664", "67466", "26391", "40127", "71871", "11713", "16673", "36335", "3682", "65022", "40341", "72191", "9587", "49464", "58047", "51719", "11796", "11287", "61742", "45508", "33284", "52681", "4498", "985", "25076", "44520", "65396", "29130", "65413", "11543", "26703", "63203", "65399", "64266", "32299", "63817", "17587", "47169", "44859", "56537", "67017", "57591", "54532", "71019", "27155", "1815", "49358", "51592", "68458", "51180", "17262", "65234", "8213", "63528", "57176", "65631", "42889", "28748", "51758", "34628", "28118", "64529", "69087", "67341", "10505", "16559", "30636", "39156", "50483", "4791", "14251", "41650", "59196", "34653", "34594", "16806", "23168", "22024", "56940", "6216", "22575", "40390", "22521", "767", "24881", "49377", "66778", "28294", "63775", "23142", "8555", "61316", "54992", "29513", "10039", "35282", "44614", "37341", "48063", "60871", "15522", "54451", "51858", "73143", "29512", "49283", "55885", "65387", "10592", "53120", "45043", "25598", "43093", "48355", "66874", "46207", "61476", "31742", "24749", "59669", "60950", "23907", "28086", "38379", "38249", "5027", "16207", "18396", "13693", "14418", "17394", "43089", "69341", "55140", "45575", "45910", "56739", "25192", "29159", "42729", "30078", "14359", "29276", "13535", "40666", "61679", "11606", "5569", "62528", "64916", "71950", "46459", "7509", "49362", "46019", "58186", "22941", "70386", "49719", "72931", "16147", "2764", "66179", "21366", "51205", "9668", "29365", "47590", "68985", "38060", "65677", "63086", "18527", "65662", "34078", "25255", "21914", "54453", "17124", "30045", "44528", "34458", "37593", "19381", "19606", "54528", "62479", "35390", "43281", "38212", "6600", "5611", "27332", "45529", "41245", "9145", "2574", "10268", "38335", "32458", "71534", "69002", "10793", "37456", "5349", "32240", "42340", "53416", "56620", "54901", "35465", "20473", "64540", "10449", "44324", "33293", "28708", "38282", "71994", "68263", "37559", "37950", "38373", "15015", "8816", "30419", "69320", "41071", "2906", "23399", "53438", "24075", "25168", "50621", "51874", "9198", "26919", "26045", "38700", "32622", "19979", "39986", "34599", "34343", "41632", "31863", "1490", "65810", "32365", "706", "33680", "62784", "23476", "16062", "48642", "57678", "6205", "58861", "1589", "72919", "51593", "34882", "12806", "70116", "15898", "65402", "42857", "8820", "73022", "69512", "67766", "48471", "66329", "62503", "64349", "17121", "42296", "69700", "48331", "39298", "22067", "11508", "15404", "67927", "44276", "54567", "57765", "2556", "28166", "57539", "60989", "25645", "58055", "67884", "53190", "45219", "4660", "69988", "49637", "61504", "66508", "37952", "38713", "5317", "57447", "2721", "8052", "20283", "35191", "1019", "63219", "48745", "46351", "43968", "1916", "68242", "53378", "59246", "28731", "6049", "60555", "7030", "69061", "19144", "67078", "54312", "51229", "51613", "70621", "3485", "6573", "19695", "4751", "12856", "19706", "4168", "48946", "41621", "1925", "64326", "25497", "15124", "67302", "33588", "48174", "6230", "71930", "57448", "65334", "50950", "24318", "5809", "16124", "22149", "35311", "64643", "15497", "12688", "12263", "18727", "3412", "4637", "41962", "12328", "68099", "43847", "36869", "43527", "52700", "21311", "22723", "31812", "62871", "58452", "40822", "72511", "32377", "63018", "37665", "62609", "27252", "51667", "49756", "852", "9763", "13383", "50719", "72414", "1007", "7345", "42541", "64199", "70641", "57370", "2762", "49502", "10727", "72926", "19490", "27674", "70265", "14638", "67482", "59940", "53062", "18011", "72517", "7488", "8673", "12429", "71495", "57835", "38532", "23756", "18532", "18834", "23757", "35497", "27748", "39691", "49136", "59897", "19235", "5591", "57269", "64960", "40653", "67059", "350", "9790", "6566", "27216", "22920", "11619", "15501", "46756", "26381", "871", "24636", "39020", "28725", "48939", "32254", "10651", "12789", "12125", "48723", "58317", "58119", "71064", "30278", "52073", "55663", "58013", "17279", "2853", "14300", "5197", "22340", "13764", "70107", "51440", "3586", "60561", "17764", "28006", "61396", "67278", "19411", "69756", "43590", "12061", "31202", "20491", "20135", "46259", "52552", "65972", "70299", "47214", "44005", "1489", "46401", "36525", "34447", "16510", "19336", "54735", "54244", "42956", "33047", "32701", "56348", "56142", "24708", "50659", "41094", "3053", "32464", "45034", "50185", "15956", "10828", "64633", "46830", "50929", "22760", "71922", "42998", "44296", "11347", "36981", "69546", "20594", "59318", "71446", "3348", "47646", "6408", "3722", "54669", "10544", "60972", "25326", "28170", "20215", "13880", "21643", "47726", "47290", "29001", "17313", "35026", "31867", "68330", "6323", "4146", "59439", "58389", "30307", "54575", "1880", "17199", "4802", "15007", "12446", "71738", "58961", "11973", "30108", "12136", "71938", "10859", "7962", "25721", "15896", "66707", "43577", "71115", "39956", "33983", "48787", "29435", "71059", "17198", "4618", "52948", "40026", "45657", "1661", "59822", "6721", "19390", "26677", "25102", "50570", "21473", "60534", "23108", "31914", "18955", "33110", "1074", "57673", "18036", "65410", "14480", "71655", "65501", "39873", "6356", "30957", "23076", "10024", "12119", "69601", "68376", "33461", "47496", "10358", "12579", "56179", "37990", "18175", "15275", "55707", "23773", "52813", "5397", "27039", "26491", "40742", "13529", "51488", "48699", "69368", "35131", "58691", "30039", "43392", "54172", "17006", "26022", "8278", "11191", "36418", "10156", "34615", "36169", "34690", "11068", "71537", "15342", "20529", "28749", "11165", "9478", "53761", "26545", "38055", "38261", "69110", "30230", "66591", "21492", "1930", "70007", "26365", "50162", "2264", "65155", "70139", "34761", "2776", "5371", "17282", "5663", "34177", "72855", "28611", "51665", "63406", "49979", "38970", "48626", "33823", "68353", "34772", "6736", "163", "44675", "44577", "35818", "56704", "71136", "56156", "52032", "6093", "58199", "24608", "8408", "26413", "41924", "57812", "50212", "48858", "32020", "49505", "49204", "20865", "51821", "51359", "55846", "44210", "39407", "65833", "14044", "44034", "20601", "7014", "49124", "47628", "50129", "19985", "28723", "15469", "39965", "353", "19778", "18628", "25005", "25901", "14123", "70499", "56490", "70377", "60682", "1002", "6979", "61126", "23051", "24245", "42582", "27007", "72591", "28397", "58226", "4057", "23298", "14441", "47658", "55344", "61555", "33667", "1905", "62297", "28130", "54267", "19427", "34974", "14865", "2447", "17375", "37362", "52303", "63732", "63518", "34132", "4399", "54639", "67058", "65916", "69334", "55828", "60213", "12721", "20823", "51776", "72633", "67701", "38138", "16445", "7885", "10133", "25775", "41636", "42520", "12767", "20661", "36650", "40308", "26895", "53525", "65922", "51047", "15428", "16044", "22073", "23783", "61272", "55525", "7317", "64432", "50118", "7525", "41021", "49465", "52085", "38196", "44152", "25631", "26459", "37504", "42178", "37226", "23181", "17495", "32627", "21531", "47871", "36617", "48285", "69330", "62557", "50327", "4913", "29997", "54763", "32160", "42638", "60169", "51386", "47052", "43908", "9036", "64817", "14364", "70064", "54091", "51745", "51769", "35824", "14246", "40237", "19482", "39968", "30566", "9427", "41392", "20794", "49836", "21251", "66337", "34110", "37476", "29416", "35421", "24348", "54052", "62278", "21857", "14598", "44399", "44025", "54940", "61259", "22804", "18851", "69225", "11651", "19899", "16369", "47405", "43033", "26863", "28356", "51112", "7826", "29525", "71185", "229", "6424", "64857", "21459", "306", "72540", "55940", "65945", "41526", "71906", "24170", "20424", "32606", "61748", "51731", "25397", "30355", "23976", "4623", "68126", "41371", "71198", "57815", "15142", "27510", "59767", "16643", "35823", "22159", "37849", "32410", "66691", "34912", "3427", "35414", "65783", "36621", "60356", "63519", "18338", "61109", "23210", "57527", "29715", "53665", "2803", "54459", "57209", "25430", "52629", "54469", "44104", "52467", "66866", "40744", "27383", "58717", "19051", "5713", "34461", "17681", "62004", "1144", "64839", "52759", "62493", "29367", "24548", "17231", "66198", "50722", "64032", "53195", "57585", "2891", "44374", "14080", "57501", "28157", "37301", "4816", "67676", "62292", "68941", "50683", "64523", "60850", "27805", "6598", "51515", "27415", "10595", "60757", "72278", "36170", "29347", "14810", "17744", "66355", "59568", "65187", "46332", "476", "48294", "27294", "72423", "28869", "4141", "25027", "4204", "49406", "27812", "912", "16422", "69306", "39337", "51512", "42029", "66802", "28022", "63270", "19320", "26074", "17318", "8089", "45119", "57394", "51931", "5472", "12424", "16718", "22581", "21985", "6957", "24847", "46777", "3329", "58258", "18231", "55886", "11650", "40641", "4570", "19303", "30322", "34999", "20579", "70559", "6300", "55425", "17484", "13454", "67178", "1514", "54791", "44870", "15502", "2779", "67623", "25615", "68139", "30204", "44239", "8387", "38778", "11732", "71459", "55997", "69254", "64472", "22996", "67997", "69594", "46131", "6734", "23994", "56298", "30791", "46317", "36222", "33248", "7898", "43601", "39653", "56082", "31364", "28325", "33175", "29905", "28124", "12709", "69893", "15538", "42798", "27631", "58041", "6363", "22619", "8815", "44086", "72987", "47885", "27262", "53401", "54309", "61262", "6991", "57642", "26858", "63698", "62086", "54674", "38455", "1056", "8511", "53749", "58015", "55928", "49158", "13801", "60944", "29735", "66544", "64716", "13258", "11683", "11479", "39132", "58204", "60920", "56592", "38356", "42787", "54823", "51978", "8265", "71369", "72550", "37219", "48326", "60606", "19571", "5811", "59936", "60931", "29625", "21547", "20986", "12362", "71125", "24401", "35838", "64461", "16524", "36117", "21188", "24690", "63589", "59626", "49746", "40543", "8150", "16700", "7912", "30959", "65345", "48914", "66362", "34462", "11432", "46350", "8822", "53399", "31477", "1", "15989", "27824", "39798", "30188", "12922", "24291", "11753", "56885", "24980", "14774", "51518", "29772", "68795", "8195", "38526", "38830", "42876", "15808", "71074", "7802", "3523", "46576", "8430", "69365", "47032", "15802", "37360", "28255", "73035", "47840", "32593", "27765", "45234", "15410", "70713", "19391", "14824", "60658", "22496", "36010", "52613", "55189", "9948", "44107", "49504", "41469", "11937", "73", "71915", "16588", "27530", "61025", "24303", "18765", "39372", "60481", "4806", "25177", "22448", "12021", "20551", "58754", "1853", "940", "41829", "39655", "43496", "15405", "30475", "62188", "34881", "21041", "59500", "58303", "47817", "70700", "11315", "47369", "39215", "44357", "768", "22098", "10600", "40088", "28139", "59825", "72662", "36712", "61638", "39343", "6251", "52929", "58366", "48664", "54414", "65385", "59713", "10132", "64231", "57610", "63214", "29860", "46453", "49835", "68870", "43016", "66631", "27784", "5779", "24817", "56266", "68595", "41742", "71468", "32472", "2845", "8332", "15756", "69184", "17483", "70323", "69821", "56595", "4753", "48888", "27186", "13177", "45152", "21795", "13718", "33022", "43860", "12238", "66971", "70054", "46888", "72327", "1362", "12627", "21831", "57989", "3168", "20849", "6405", "35950", "48419", "35328", "57699", "46500", "63594", "29318", "12169", "11297", "34083", "39937", "12114", "50959", "27488", "5625", "66468", "40932", "5050", "57779", "5279", "49554", "67788", "9717", "56761", "22353", "52359", "5652", "58955", "9241", "10145", "68784", "63207", "62698", "5174", "56428", "70183", "954", "42665", "15854", "34685", "52508", "54942", "51308", "62903", "49090", "40776", "57603", "13757", "15669", "62335", "52893", "64076", "42138", "18884", "15027", "55812", "54401", "7957", "9288", "37932", "59527", "54682", "23826", "1884", "24147", "70130", "23896", "61409", "64040", "9463", "40941", "54654", "62368", "65005", "5502", "16240", "30028", "5483", "70778", "2546", "4796", "36982", "33573", "34031", "52786", "46993", "742", "2116", "69408", "45533", "62973", "4675", "51910", "63803", "7466", "21887", "54651", "6897", "45455", "71107", "5574", "62440", "12252", "30574", "48288", "63908", "56520", "58598", "27453", "66873", "43764", "62101", "36854", "4334", "46282", "20871", "15190", "30059", "45984", "1960", "51534", "72038", "53359", "71799", "22663", "73221", "39239", "39268", "53280", "42112", "4357", "38899", "21104", "47604", "63461", "71814", "3926", "4273", "46420", "52838", "51067", "46741", "69262", "23706", "39957", "73077", "32044", "43405", "2772", "31901", "40019", "10957", "13398", "33193", "36202", "56161", "66536", "70772", "62501", "30089", "70701", "71983", "6006", "30705", "68834", "18323", "70626", "34601", "56394", "51009", "8007", "42285", "58071", "11669", "57287", "25471", "23060", "69213", "3902", "48256", "7408", "45137", "66641", "72192", "46903", "300", "71710", "8911", "31676", "33299", "26359", "13377", "34390", "2131", "66279", "62176", "60350", "68009", "46773", "12915", "16591", "28375", "797", "44579", "66577", "5989", "3979", "46594", "72190", "45183", "27032", "19304", "42512", "66663", "23678", "8084", "24198", "15285", "53070", "24382", "27942", "29542", "46360", "10611", "44705", "72789", "45437", "64496", "72055", "63049", "10437", "8808", "61910", "61199", "11424", "56584", "24648", "23953", "37", "43413", "11041", "32466", "37695", "6259", "62367", "22548", "4842", "48130", "42357", "72037", "376", "15990", "71988", "56121", "21062", "23246", "19074", "6552", "5267", "41235", "68257", "49039", "58568", "72371", "65751", "55807", "48190", "2355", "58515", "22878", "27392", "14813", "53155", "10680", "39519", "36584", "17600", "5022", "46084", "8852", "11966", "65497", "70347", "34640", "60507", "56090", "34222", "60275", "36682", "60844", "49943", "69030", "72170", "45735", "11863", "7205", "71249", "49598", "37619", "69064", "29876", "22372", "42269", "9943", "69124", "10519", "59668", "17298", "4047", "45248", "40215", "11001", "65926", "19286", "50866", "11378", "58440", "53061", "12538", "37290", "44885", "70434", "60312", "29345", "32033", "36594", "26069", "41947", "52971", "44486", "35996", "70340", "16148", "24402", "2742", "4567", "14237", "32711", "10992", "15219", "45813", "64871", "51703", "62511", "4932", "23867", "57339", "24505", "14232", "14244", "69966", "45350", "26657", "51854", "71148", "30844", "65263", "46274", "50", "28075", "5701", "52515", "5700", "28987", "23212", "42777", "60048", "58597", "25693", "9300", "28249", "15879", "55314", "56076", "72284", "11961", "46446", "37929", "5470", "6417", "44454", "22466", "8456", "40523", "65258", "63690", "72488", "53180", "30210", "57147", "33890", "36863", "24443", "7831", "49674", "12477", "7005", "1268", "3831", "51243", "46938", "72851", "14451", "13461", "15134", "17708", "41288", "68872", "23965", "44708", "34375", "42793", "6244", "68877", "42533", "22980", "71540", "46839", "4190", "27675", "4039", "45557", "59706", "40389", "23910", "19664", "44064", "45840", "13812", "72718", "54450", "70165", "15125", "66823", "70243", "32539", "71835", "16129", "17239", "26208", "35195", "47432", "18787", "15764", "459", "28859", "33846", "51951", "13739", "10233", "53334", "72793", "30005", "69779", "50684", "63543", "2108", "34765", "58592", "72007", "6432", "46625", "5253", "26879", "13895", "25013", "59338", "58528", "5456", "38628", "61869", "17125", "2123", "15276", "28688", "71729", "70484", "54915", "66652", "61224", "64660", "49380", "49573", "29618", "5272", "10695", "10091", "7225", "50596", "41850", "15418", "9050", "27535", "14069", "12326", "66234", "25434", "1841", "64845", "48980", "54370", "47723", "23331", "72443", "71392", "43103", "38837", "34484", "962", "54386", "63664", "70510", "61948", "10167", "17028", "53535", "38388", "25080", "45641", "33365", "21360", "30417", "4068", "56925", "72823", "72434", "32189", "14484", "59004", "30332", "14156", "14848", "67359", "37201", "51810", "9801", "36887", "27083", "39808", "41830", "69452", "58294", "73220", "7974", "40788", "44346", "53364", "26931", "52567", "29043", "418", "15807", "15071", "67587", "23819", "25172", "32602", "30430", "63346", "68960", "46293", "4818", "27622", "60280", "3643", "44985", "28481", "46947", "23336", "39754", "58632", "24447", "61467", "48871", "61944", "39909", "51416", "14347", "35127", "69948", "13397", "9648", "54467", "70128", "30910", "33772", "9990", "58280", "59628", "64712", "21621", "55820", "22957", "40239", "52720", "13072", "12006", "23020", "18422", "6175", "6959", "41171", "39671", "50455", "4103", "58665", "59406", "36458", "56249", "7634", "308", "9109", "10255", "10549", "71857", "27839", "68268", "24194", "59141", "68896", "44194", "59097", "562", "50407", "42375", "69580", "717", "45850", "47310", "7152", "60751", "53392", "58883", "68598", "67583", "2865", "38723", "16535", "55509", "11113", "25922", "46572", "3389", "19759", "26185", "72801", "7744", "34272", "33474", "45414", "12608", "33555", "39606", "18649", "26979", "51322", "49125", "10342", "12027", "28007", "49186", "20924", "67161", "47098", "62744", "69214", "44511", "49491", "23648", "71310", "39738", "54605", "20643", "26757", "36676", "72986", "24179", "27537", "15484", "67406", "55033", "12583", "30233", "51001", "52547", "68267", "61645", "70949", "12755", "26479", "15530", "48109", "36002", "51780", "31596", "31800", "59240", "26612", "52599", "65040", "64787", "44900", "54486", "39590", "35199", "71031", "5872", "17340", "67170", "54256", "60268", "34187", "10197", "26448", "34773", "69843", "11093", "12100", "46254", "37737", "33807", "66865", "58874", "15312", "12710", "19675", "22337", "30260", "61098", "4134", "12550", "14064", "49028", "30596", "37466", "38495", "30875", "72671", "1537", "43746", "37554", "56145", "11792", "51070", "71765", "27141", "66235", "26311", "62835", "44230", "28656", "52248", "56411", "8999", "26785", "43680", "4622", "35108", "35559", "72754", "69709", "69071", "1775", "18322", "41145", "54718", "24056", "14378", "17440", "58402", "10541", "68456", "1393", "47800", "13663", "24490", "64962", "4598", "14516", "59133", "8597", "4332", "3281", "1372", "23657", "19124", "8635", "68739", "13100", "20994", "35951", "66456", "10495", "13960", "5304", "13934", "47403", "3337", "56343", "15472", "40046", "69956", "16221", "37512", "25111", "469", "28862", "61037", "51815", "6669", "64062", "69573", "26293", "47911", "55392", "30715", "25890", "1458", "61433", "56494", "45257", "34798", "9464", "29250", "4006", "61134", "68588", "51017", "26370", "6930", "15662", "56100", "10511", "51438", "49342", "29466", "14048", "47810", "42817", "36405", "67243", "11274", "48017", "14923", "41910", "6558", "26649", "8546", "42615", "67775", "66721", "5889", "21088", "46656", "34137", "29097", "26741", "35667", "45565", "65792", "23293", "23838", "14628", "16307", "32881", "24845", "18179", "67437", "4191", "60058", "12928", "44588", "49684", "50546", "41826", "837", "49811", "40044", "24044", "58484", "70357", "61308", "8440", "17818", "10058", "10987", "53279", "58129", "64970", "71554", "3226", "30932", "39096", "50874", "9459", "26948", "17244", "24123", "26847", "7276", "38605", "72276", "65382", "58953", "41264", "68118", "25745", "70844", "66203", "49369", "41439", "53152", "7624", "67318", "42155", "7660", "35294", "21855", "69472", "53068", "55086", "24234", "48078", "66344", "8515", "18625", "38547", "15865", "69861", "71673", "48852", "10441", "46911", "4514", "48292", "16848", "31516", "5188", "70198", "17752", "53332", "64733", "11355", "52869", "32220", "48224", "23844", "39251", "11800", "55402", "47138", "49025", "22447", "6153", "6067", "20263", "71722", "64966", "47131", "11081", "17501", "5972", "54938", "58499", "18633", "28260", "12736", "32446", "28887", "62431", "26", "52675", "72774", "56666", "32631", "46224", "10261", "58730", "67391", "66860", "4481", "2157", "40665", "43652", "1323", "331", "37272", "7822", "53552", "64811", "26058", "28435", "1848", "5628", "14658", "22706", "58431", "35970", "16272", "69245", "11196", "1205", "59530", "32168", "28714", "57100", "3514", "13061", "54017", "15986", "36067", "67817", "61395", "69996", "33098", "34746", "61250", "40796", "54646", "56434", "66518", "32965", "4307", "48373", "67069", "18381", "24742", "22522", "72951", "52049", "18710", "17464", "51958", "8394", "52673", "73061", "50646", "39139", "25054", "37470", "58177", "68304", "49316", "20415", "53794", "18223", "30046", "23242", "40693", "57080", "60272", "6441", "59315", "61188", "69552", "44736", "70174", "24464", "64593", "27185", "29312", "447", "70666", "60707", "18865", "38770", "40446", "35492", "4565", "67021", "36620", "68468", "2822", "61139", "18650", "53478", "32032", "38694", "37353", "30240", "54668", "46138", "53484", "62980", "8345", "15749", "1078", "66001", "31220", "48401", "18640", "37743", "72954", "65160", "36989", "28691", "63994", "29093", "61080", "61795", "44547", "72695", "51192", "32902", "41755", "59518", "28044", "30620", "10125", "1851", "9855", "26905", "64184", "27389", "21204", "36554", "34783", "44662", "8920", "39714", "29313", "41539", "67249", "46288", "60521", "5486", "9803", "8066", "55926", "67335", "24561", "25996", "36249", "3864", "5528", "34400", "20389", "22289", "5284", "7581", "26062", "56634", "62309", "63790", "44527", "19455", "51080", "24570", "61652", "35102", "29842", "8823", "73190", "72270", "22971", "45737", "39693", "61965", "30098", "8064", "51501", "9748", "40394", "14594", "16166", "13630", "23241", "35243", "9940", "46361", "10883", "9094", "31890", "50669", "26838", "32354", "32705", "39879", "65150", "211", "36172", "18556", "12146", "13954", "69229", "54714", "20697", "14280", "15918", "18771", "37927", "8658", "52741", "53801", "42267", "66345", "12301", "47517", "55809", "73163", "10789", "45045", "55277", "38739", "51945", "67508", "6116", "25376", "70799", "65356", "27471", "68957", "6557", "13040", "51691", "1680", "66411", "47778", "51401", "35148", "31751", "23539", "71371", "29636", "15899", "52857", "59903", "66233", "68796", "45832", "36938", "63140", "47492", "6964", "42609", "60154", "9932", "15868", "21693", "21101", "40556", "3587", "25166", "44442", "59272", "41243", "22168", "62478", "60083", "9731", "44509", "24115", "31031", "67003", "37024", "72240", "59534", "14248", "39751", "23363", "15814", "42882", "16306", "39772", "25382", "61237", "57729", "26799", "56640", "23161", "4027", "59725", "736", "27381", "21962", "44721", "51909", "23598", "41357", "41309", "44458", "67659", "52334", "56979", "24065", "45774", "33704", "37350", "69141", "13221", "55171", "8948", "26409", "72234", "21563", "70420", "42411", "69413", "21325", "67462", "47557", "64469", "56629", "32214", "35196", "51266", "33687", "18250", "51314", "19937", "3223", "66053", "30041", "15950", "21697", "69450", "14323", "15041", "19581", "7365", "8583", "17132", "29904", "29705", "55000", "64380", "60564", "27136", "38819", "11153", "67226", "24889", "17220", "22542", "23044", "39627", "8634", "28669", "58560", "22816", "21378", "48171", "38955", "47989", "31233", "2795", "52401", "71850", "24411", "67129", "4667", "30378", "64637", "48979", "49297", "70907", "6527", "54133", "27275", "8076", "57082", "64846", "41377", "50362", "67128", "8445", "72839", "29899", "58772", "9703", "31013", "68018", "45955", "22027", "49352", "64089", "17105", "30572", "3134", "52533", "55400", "11989", "10849", "12989", "41054", "71844", "18964", "53182", "2390", "29113", "19809", "19838", "50274", "27773", "43473", "16255", "64725", "11184", "71787", "8832", "19589", "26818", "13925", "70798", "16851", "63715", "57850", "59538", "49179", "15792", "39401", "6901", "31380", "7064", "24261", "53487", "31337", "57712", "7861", "44106", "61729", "59924", "49357", "28689", "36250", "51468", "44441", "32740", "37725", "35663", "2603", "71811", "3987", "70506", "40411", "63560", "49099", "6794", "54787", "28039", "4632", "19086", "71590", "68341", "41729", "39517", "19651", "34729", "66365", "4789", "71316", "5939", "38168", "69849", "52319", "52650", "553", "15411", "10204", "60308", "17662", "33016", "23341", "10081", "12951", "58275", "39141", "40205", "942", "28530", "11249", "22606", "4579", "60291", "22793", "55275", "8903", "66369", "19095", "57058", "40008", "54029", "46205", "38250", "13504", "40939", "61937", "25091", "20684", "54780", "3474", "40045", "54986", "34164", "60527", "55354", "21742", "67320", "31342", "66327", "58938", "13223", "19322", "54595", "67567", "14231", "7026", "39025", "40185", "59456", "46858", "71997", "36931", "71020", "71612", "61607", "35876", "58722", "16601", "27843", "17341", "69491", "43902", "38522", "3553", "11052", "59459", "6276", "51309", "22393", "59780", "31735", "1858", "29638", "43599", "20435", "14180", "50960", "34496", "31140", "25099", "21928", "68283", "69088", "55850", "38953", "29393", "43895", "51963", "49659", "23721", "2014", "3510", "17510", "66936", "21833", "52030", "66460", "48266", "14863", "46319", "66627", "16065", "44946", "67568", "45071", "5159", "29958", "40497", "62398", "26506", "591", "60015", "13014", "70789", "47936", "34547", "35556", "49716", "36878", "29424", "1834", "53212", "62658", "60832", "29793", "60968", "8994", "45793", "27014", "44323", "1411", "23320", "34957", "62430", "17651", "14522", "19554", "30936", "67752", "32197", "741", "42784", "15817", "53204", "2562", "51029", "62519", "60653", "19394", "61614", "50985", "51211", "69508", "12338", "57553", "46691", "34319", "53163", "18608", "25435", "51455", "34289", "51085", "21774", "56245", "47417", "30433", "43571", "55452", "15700", "44819", "11876", "67926", "2702", "72808", "17466", "23247", "59014", "20216", "45108", "30852", "58897", "72863", "25212", "40094", "60106", "52881", "10270", "28979", "17655", "17650", "71699", "22539", "35081", "59214", "22999", "51374", "62230", "20565", "68583", "62866", "8502", "18410", "1718", "47691", "2942", "62383", "33532", "17425", "56786", "63677", "30160", "8756", "71202", "56728", "52429", "17603", "55234", "17990", "57900", "18257", "7021", "2718", "50833", "61616", "28935", "28788", "72865", "24240", "36688", "34068", "38933", "49407", "36792", "16428", "69840", "44310", "70038", "45725", "19856", "15246", "40726", "56808", "23404", "26043", "45225", "54069", "59445", "23769", "69758", "4558", "35368", "66548", "16815", "71018", "10953", "19154", "51551", "4542", "69364", "71837", "33232", "28074", "71567", "71483", "48501", "15636", "63714", "46622", "63223", "27761", "3577", "25477", "51657", "27702", "53721", "52308", "12663", "44482", "65525", "46535", "35540", "1119", "71853", "69763", "26607", "23947", "32264", "57170", "46861", "12936", "43265", "34472", "53910", "72504", "10980", "356", "26048", "43822", "3760", "63943", "25799", "19990", "43466", "71561", "46644", "2639", "68472", "697", "5293", "49845", "67584", "58992", "1999", "36412", "64143", "37067", "18148", "47478", "32213", "65829", "47980", "64883", "5524", "32564", "71051", "417", "36372", "54383", "903", "2926", "2206", "71486", "56835", "2147", "38776", "62451", "51186", "12669", "54593", "15445", "21314", "54039", "41503", "70814", "13424", "34226", "72341", "33128", "66162", "13945", "61892", "24577", "36007", "33786", "38504", "1446", "70392", "28092", "35551", "42831", "42715", "34930", "4260", "8208", "67147", "69680", "20016", "36909", "56212", "17104", "45662", "24471", "55382", "12444", "26944", "40736", "56687", "26158", "12220", "20430", "52464", "4202", "21287", "64595", "43654", "46821", "17024", "60739", "30874", "48988", "58132", "41410", "36895", "8316", "11538", "7267", "55002", "26405", "31968", "26913", "72560", "61121", "45394", "68800", "31464", "8255", "15758", "39444", "21551", "44232", "19107", "38200", "36516", "48662", "29994", "13560", "21715", "19176", "22189", "32435", "69147", "17970", "19509", "51896", "39086", "622", "53854", "47584", "63278", "51257", "59899", "60447", "10415", "15693", "57468", "32666", "68898", "46648", "9787", "2881", "61207", "40479", "7749", "31786", "49520", "61311", "29215", "65610", "58957", "16924", "4748", "66590", "8633", "29959", "70028", "11278", "56852", "27967", "66465", "21748", "29219", "69019", "34261", "45372", "18270", "18658", "52713", "53556", "37981", "28667", "2857", "40282", "9085", "7251", "65693", "57452", "48994", "63965", "33664", "68413", "44930", "41354", "37278", "47319", "4817", "20477", "71934", "57230", "36296", "71101", "48897", "69852", "25694", "36946", "5932", "8632", "65861", "29744", "9319", "21908", "41492", "23306", "49666", "56986", "53052", "20523", "19487", "42056", "48335", "28607", "62960", "60248", "33691", "18634", "5590", "49789", "48426", "16082", "62465", "54969", "38057", "36924", "46399", "69824", "62988", "10516", "49829", "5414", "31909", "8891", "23797", "11502", "33435", "36475", "2791", "44672", "5649", "41857", "23631", "59866", "25256", "42442", "14704", "71739", "15844", "37607", "63421", "3141", "36039", "62657", "41949", "2028", "2567", "37057", "18291", "66673", "5910", "39115", "5753", "38659", "7450", "61192", "43300", "62164", "47223", "40050", "35549", "54828", "62127", "2186", "23987", "42816", "13163", "71742", "71084", "17405", "657", "25644", "5241", "29946", "25897", "35151", "64126", "55951", "5070", "49739", "8639", "15054", "30155", "7435", "67981", "35052", "20175", "11407", "60211", "55393", "9031", "52781", "57827", "65094", "26534", "4506", "52390", "52833", "30389", "48442", "49111", "69691", "48041", "11562", "941", "66146", "69059", "52597", "21413", "4534", "10570", "2512", "42167", "36857", "72840", "12553", "41979", "23574", "15414", "19724", "13154", "4018", "46189", "8235", "24898", "49490", "60634", "5780", "15234", "24996", "9522", "13723", "30103", "53839", "22887", "9406", "62713", "63315", "12431", "40300", "55360", "37697", "25539", "2295", "23669", "16401", "49098", "59362", "15437", "21331", "72087", "28248", "68753", "8904", "61164", "46196", "50313", "27508", "33928", "50164", "53139", "20595", "56340", "71544", "62604", "55949", "37310", "24770", "39514", "27838", "66851", "38711", "55377", "20433", "16003", "12823", "7951", "23195", "41837", "52233", "60661", "42308", "51618", "47039", "69739", "67760", "56474", "19828", "13904", "18001", "53168", "22265", "12075", "6989", "68177", "46379", "65950", "21474", "51452", "63877", "33120", "29956", "71383", "2033", "69750", "35743", "16498", "26686", "10712", "11302", "19601", "61759", "33728", "2850", "23571", "18539", "38624", "57639", "24146", "36646", "20544", "3962", "53326", "43431", "21154", "2397", "25928", "78", "5307", "67725", "57152", "31839", "35011", "72132", "46785", "72842", "71353", "48764", "65885", "28852", "59043", "6072", "60646", "33341", "12094", "6771", "29762", "37153", "66107", "62486", "64900", "53649", "27938", "15477", "2715", "17207", "29674", "71203", "61595", "44314", "54192", "48984", "13977", "50627", "66245", "56468", "68379", "69741", "14039", "66185", "32704", "20427", "59251", "52184", "5803", "31635", "48204", "47746", "64353", "56598", "44498", "36436", "19419", "59053", "63255", "54537", "65551", "53402", "58037", "59861", "51407", "16285", "37077", "34109", "38443", "40367", "50061", "62865", "9158", "19099", "4823", "2350", "23524", "53320", "73119", "19038", "4130", "2985", "22472", "46616", "42587", "59390", "59623", "36637", "20284", "7511", "61704", "34692", "49251", "6303", "68586", "54116", "14746", "17779", "71305", "58972", "32483", "54248", "50188", "26214", "5348", "69082", "25043", "6467", "33081", "51209", "62800", "26844", "60218", "68350", "18986", "2784", "36791", "38204", "64746", "39554", "48800", "31704", "943", "4638", "25271", "27668", "67130", "43971", "31885", "14243", "40585", "34404", "14733", "15922", "13003", "7659", "60358", "52878", "26464", "5261", "25664", "31154", "32109", "9279", "27219", "56900", "41936", "27494", "37308", "48648", "64539", "44965", "40960", "34149", "52742", "57031", "10371", "53827", "67100", "56250", "18897", "30632", "52522", "44336", "2255", "57184", "56865", "25160", "17771", "38615", "8272", "42808", "1878", "29337", "46252", "30819", "17398", "44524", "9130", "63892", "2025", "32182", "3130", "72512", "31385", "67096", "27616", "29234", "69875", "72939", "4128", "3292", "42619", "43753", "24755", "55043", "9039", "12495", "17614", "10578", "57097", "12253", "52541", "69501", "65435", "46388", "56647", "59828", "43347", "17321", "48784", "59782", "14510", "21176", "44176", "14908", "15406", "59179", "68766", "52895", "16928", "41959", "71142", "52242", "25931", "61762", "33869", "15045", "54507", "2195", "31866", "21970", "71798", "33413", "39418", "23370", "39362", "23691", "2232", "39706", "32306", "66282", "44180", "46357", "50171", "35171", "50391", "65687", "24195", "7046", "65344", "71357", "43808", "69179", "4315", "61942", "63004", "56461", "54372", "21619", "25593", "35317", "54088", "5130", "72667", "53460", "63006", "59484", "51072", "4933", "47841", "63999", "16722", "35564", "9470", "2809", "28005", "70262", "26166", "22826", "31703", "9079", "37633", "33497", "12527", "42724", "67809", "29567", "20978", "57384", "54456", "50004", "269", "65008", "39324", "24779", "46373", "5749", "39373", "56193", "25162", "42233", "22409", "39515", "28546", "2943", "26025", "4994", "53047", "2642", "68935", "38569", "37358", "33054", "50000", "61727", "2453", "64741", "1332", "61988", "55831", "64483", "64397", "725", "68943", "4151", "26278", "67047", "32668", "40010", "14334", "43907", "1107", "10936", "42273", "30189", "25480", "48458", "8029", "32607", "62682", "432", "49112", "61014", "64180", "55918", "18357", "3360", "44640", "31770", "69021", "61474", "24214", "37443", "37168", "52457", "13641", "68215", "23153", "4769", "72249", "52206", "12417", "44617", "32764", "69553", "22703", "59300", "3354", "32317", "56947", "70018", "63120", "70033", "52975", "48569", "23245", "42396", "51714", "12090", "36788", "54970", "35043", "67516", "59260", "34984", "60345", "46370", "57882", "45724", "47876", "48708", "29214", "57771", "71274", "63319", "57936", "28989", "18970", "11768", "48103", "3531", "60940", "32884", "60258", "63702", "30321", "16372", "71654", "70162", "42454", "64381", "59026", "18245", "715", "16586", "23960", "32187", "42033", "41802", "35856", "35586", "65825", "58911", "72215", "5487", "65401", "35773", "33043", "27431", "46867", "8670", "46983", "63593", "23671", "24205", "15121", "31633", "45973", "68298", "58457", "36011", "4847", "47423", "46038", "34757", "46146", "63397", "43937", "61526", "13583", "35064", "36600", "58780", "37824", "27714", "68383", "9711", "46607", "19975", "24100", "15675", "61580", "4069", "30602", "22449", "8124", "33982", "1126", "26701", "30738", "55743", "25534", "3181", "58008", "68609", "17608", "21663", "66081", "70978", "17916", "27732", "61667", "4614", "4350", "53653", "2732", "42859", "59369", "25426", "12517", "9087", "71939", "38752", "60245", "62684", "57856", "45596", "1939", "65288", "60043", "50121", "40451", "54836", "70660", "63556", "385", "37320", "72524", "68908", "628", "49402", "20129", "55482", "46757", "7603", "10612", "54452", "15210", "10076", "45192", "1299", "11911", "54115", "67245", "56400", "57750", "45005", "34939", "57470", "11192", "21555", "37268", "18251", "59910", "66019", "60612", "8344", "30314", "4306", "40350", "16748", "5416", "59145", "24277", "42553", "53210", "73241", "68110", "12899", "16210", "14885", "26181", "34739", "48754", "51437", "33520", "19052", "9490", "73047", "50165", "47952", "15917", "30804", "31841", "58962", "52496", "31048", "15568", "47178", "38996", "72018", "67205", "24830", "65579", "6164", "46111", "71711", "15946", "51819", "68491", "70069", "50145", "9974", "68346", "40333", "47443", "71434", "68369", "40739", "18143", "16551", "39380", "57793", "45823", "1638", "62457", "3020", "7412", "71761", "18428", "12128", "50563", "62662", "315", "24243", "55103", "36930", "44894", "52506", "57525", "68601", "70578", "17780", "37198", "66438", "47063", "60489", "66688", "46341", "33216", "28329", "32209", "40909", "69011", "10246", "72522", "52880", "60476", "13929", "43272", "49533", "57715", "5235", "72879", "67836", "11813", "50887", "4253", "52819", "35395", "68620", "25606", "42358", "69870", "488", "10978", "5853", "3613", "5511", "18215", "53650", "8810", "54896", "7535", "40126", "36141", "53157", "64713", "50822", "4512", "33977", "54975", "18647", "32159", "57365", "1740", "55818", "19340", "54458", "68205", "49725", "5103", "21881", "42176", "65441", "12524", "55338", "31760", "48248", "68034", "7250", "19208", "41793", "60116", "1355", "21812", "50856", "28768", "70608", "49987", "14731", "1363", "16972", "53292", "4608", "64085", "3706", "43769", "54028", "18580", "26569", "44887", "57536", "10523", "53044", "56051", "2225", "52988", "8171", "59872", "24403", "53048", "72781", "61357", "59731", "10092", "66346", "13174", "44273", "31422", "51529", "23625", "47903", "18616", "3442", "8802", "34186", "28050", "57885", "37275", "64254", "72437", "29209", "1809", "40959", "66384", "30611", "34823", "7644", "58421", "24816", "72144", "48703", "48837", "27196", "63381", "28681", "19406", "10845", "33429", "69231", "28915", "10271", "66447", "71211", "37650", "59964", "50402", "29183", "55803", "53791", "31908", "52849", "5595", "20418", "49784", "69067", "65241", "53874", "8254", "11183", "20760", "9046", "4197", "51548", "15383", "69942", "67220", "63514", "40144", "35801", "25332", "70142", "12846", "54394", "64425", "4820", "37140", "46675", "53700", "73137", "9902", "62162", "60262", "50905", "71942", "22277", "42262", "51137", "60471", "550", "64012", "57135", "37508", "13849", "42323", "52311", "60093", "39417", "24460", "26525", "32169", "1302", "26694", "51988", "21562", "57211", "7820", "6742", "53808", "58715", "33130", "38832", "64346", "8200", "58426", "25828", "55540", "14227", "57411", "61066", "23541", "44412", "14211", "70526", "39635", "52657", "47025", "26784", "67761", "30", "23025", "6473", "54978", "58128", "34993", "12124", "62694", "63164", "35776", "29768", "8856", "69205", "51623", "32796", "2027", "46783", "46483", "12867", "21749", "27904", "30010", "16853", "55750", "43448", "32553", "43819", "17552", "35354", "34720", "59586", "16633", "18284", "70784", "72439", "5653", "53206", "14308", "38430", "72785", "55308", "27822", "31509", "41756", "43559", "29650", "61498", "42544", "44133", "66678", "10534", "24578", "56408", "40303", "11842", "68590", "35560", "48443", "28431", "28957", "62544", "64365", "72467", "14891", "56281", "2339", "34871", "41232", "4947", "39370", "52215", "50872", "41637", "20143", "1224", "72308", "63607", "5601", "3465", "31348", "66169", "63670", "24212", "68687", "20155", "16766", "7103", "56913", "25533", "13574", "11227", "11242", "6780", "71023", "43722", "60806", "62978", "29642", "57625", "38198", "37721", "65041", "57398", "57028", "4650", "64211", "41693", "41182", "29262", "7918", "13978", "31038", "42208", "68128", "5365", "50040", "66547", "11995", "53742", "21921", "44114", "68701", "9344", "15563", "16960", "50636", "29826", "72092", "54751", "44054", "14271", "21014", "8347", "9060", "38647", "9092", "39764", "7209", "55316", "42594", "4826", "50882", "58712", "59953", "15104", "42365", "67710", "72579", "65840", "18797", "3065", "71103", "26296", "6038", "47937", "9401", "19802", "112", "43889", "52647", "4298", "9510", "20039", "52744", "67036", "20971", "21315", "73017", "66740", "53930", "24827", "70015", "66685", "19070", "25115", "27898", "49445", "16448", "72873", "28609", "38931", "29029", "13914", "66983", "11103", "17860", "3791", "10952", "33217", "18248", "58711", "47246", "46801", "31246", "13699", "66597", "21206", "15778", "11226", "36581", "31806", "66014", "41838", "71566", "55790", "28943", "61514", "19512", "62213", "50365", "41707", "38584", "21183", "5314", "1473", "3840", "16096", "18595", "26579", "11621", "2518", "4308", "67179", "23551", "50241", "47766", "16558", "29627", "31218", "5143", "43137", "23435", "37806", "35697", "66320", "14590", "13985", "22854", "69018", "71790", "62736", "71275", "31852", "23080", "53633", "21173", "71309", "5165", "46148", "26687", "9876", "5086", "19905", "60099", "64840", "38586", "3265", "55251", "32833", "46141", "27524", "24913", "18593", "16973", "43477", "54107", "35760", "14041", "50333", "18332", "17367", "9701", "12333", "18564", "39012", "22599", "56224", "17741", "57347", "63451", "59607", "56974", "4530", "19572", "15871", "65812", "41524", "56196", "40843", "15409", "62074", "46565", "8811", "23520", "30328", "58034", "54099", "64662", "71342", "51181", "52015", "5506", "69805", "29599", "9292", "5447", "15849", "72345", "10033", "25656", "72711", "42842", "35501", "39988", "43718", "9475", "54218", "63711", "49944", "67802", "2299", "55595", "45401", "65992", "47999", "33256", "14222", "20474", "19060", "18354", "61290", "34833", "34924", "70646", "71165", "65074", "59932", "7977", "54001", "59705", "26092", "54516", "31857", "38353", "71995", "71760", "45526", "48001", "48252", "51194", "25532", "59643", "56240", "8424", "23816", "53121", "6636", "37615", "69785", "14488", "43946", "40990", "57866", "17230", "60819", "59265", "24184", "2808", "66121", "53974", "1477", "24441", "430", "61940", "20530", "52067", "2007", "42684", "13440", "31490", "59395", "14776", "32636", "28839", "27641", "23427", "16789", "13338", "16546", "7252", "56392", "2325", "20767", "36928", "53428", "23991", "13650", "72945", "43712", "53170", "19730", "31179", "3475", "14513", "32519", "59978", "47754", "31406", "35230", "23591", "31407", "28516", "28401", "51749", "41868", "61181", "10097", "13637", "52692", "31189", "8060", "23674", "45674", "4362", "63417", "5480", "2089", "34979", "21868", "35535", "41426", "39223", "64964", "51451", "68531", "9488", "51038", "13016", "43829", "62781", "41051", "59412", "65056", "44191", "63311", "28522", "37474", "915", "29919", "54329", "27881", "4494", "23945", "14670", "25991", "1405", "20465", "11815", "26186", "19528", "34792", "10168", "26143", "58195", "9843", "38622", "55485", "50582", "38638", "33923", "46549", "22363", "49977", "48209", "29204", "71419", "48241", "40627", "39891", "18552", "55501", "46908", "15958", "48679", "16573", "70538", "28635", "66612", "20238", "46397", "44164", "36707", "5781", "17800", "65688", "13017", "30374", "17053", "19952", "70088", "1580", "37534", "63652", "17447", "66287", "59990", "19853", "11787", "8317", "45676", "34627", "28424", "59288", "8261", "38777", "66753", "28335", "70156", "15512", "16915", "5638", "16769", "16699", "25064", "52965", "12525", "12955", "14361", "20556", "40841", "26881", "1320", "69940", "58878", "64944", "43692", "4688", "2966", "72111", "42450", "51839", "2055", "45957", "43126", "56108", "33720", "11722", "60604", "46930", "18498", "49214", "28745", "9052", "71041", "41302", "23362", "65852", "43688", "48835", "51290", "15585", "43762", "12731", "18809", "39567", "48270", "53338", "31174", "62179", "65026", "1166", "23313", "47868", "55212", "16318", "72996", "5613", "24337", "45962", "16459", "67669", "1016", "33710", "583", "71631", "17151", "67907", "19712", "61106", "13980", "42583", "40656", "48973", "5353", "9446", "71821", "32454", "40034", "33480", "69591", "27040", "42195", "11742", "66074", "5708", "35088", "28088", "61744", "13229", "51789", "31426", "40982", "54041", "17874", "72210", "59474", "36075", "36644", "9794", "9774", "12320", "49558", "3720", "8262", "14319", "49603", "44934", "34737", "21007", "56519", "13009", "30567", "29115", "50606", "16631", "51708", "18082", "29616", "31932", "60222", "30863", "50978", "1267", "63851", "67001", "22019", "4364", "35059", "38898", "15322", "70113", "7944", "27224", "72377", "62025", "39001", "66572", "53868", "8745", "42456", "48933", "22309", "51645", "10752", "7966", "15356", "70776", "47143", "12893", "17546", "71319", "43450", "46890", "5214", "30343", "42758", "11818", "27283", "21032", "50589", "26017", "3115", "35374", "54641", "29168", "41387", "254", "3899", "22205", "20886", "11612", "19227", "18235", "15682", "37876", "67101", "7873", "40633", "27206", "60246", "17115", "39441", "53437", "24683", "65837", "71449", "53678", "57298", "1948", "57296", "60020", "50434", "24775", "20486", "15279", "65336", "24222", "39138", "4404", "40079", "10443", "419", "15912", "26119", "57289", "33758", "68475", "7512", "38922", "36101", "70379", "3817", "36346", "28680", "31256", "51302", "54491", "49938", "40670", "9285", "40812", "29890", "27665", "38597", "54019", "15160", "63660", "65670", "11419", "41347", "65934", "22136", "1544", "36388", "43781", "32692", "72674", "64965", "46130", "36194", "70188", "57660", "63088", "50234", "20915", "38022", "40338", "67166", "10839", "28857", "35319", "18015", "40535", "62645", "11415", "59195", "53162", "27733", "52169", "5684", "25433", "33998", "69195", "65105", "68940", "49245", "31646", "36603", "38273", "59199", "16155", "29789", "16804", "47745", "2265", "28340", "7865", "2861", "29791", "27709", "49457", "69441", "42986", "48240", "25816", "28060", "24077", "29447", "43982", "51346", "12359", "40279", "23494", "30891", "11481", "26835", "44802", "54136", "8615", "19501", "51327", "3008", "49153", "14291", "13783", "33957", "24793", "31412", "38506", "18965", "68408", "6996", "40379", "18747", "14368", "73001", "14245", "1647", "44516", "26213", "57153", "9615", "21733", "21203", "65991", "34487", "41723", "47430", "24887", "69558", "17337", "7245", "65341", "3526", "57487", "17043", "56833", "44949", "11504", "37464", "26001", "37223", "24767", "37538", "1243", "40036", "73182", "37626", "48265", "22137", "13164", "64619", "56667", "28622", "18694", "27615", "28403", "54432", "70345", "61233", "5437", "3801", "1906", "43817", "4739", "36252", "20236", "69748", "59180", "58578", "22796", "21228", "60676", "32849", "53082", "36360", "9336", "66712", "66409", "38885", "53219", "22012", "63246", "64312", "33956", "72894", "70085", "71779", "53844", "20304", "56659", "49404", "20234", "10134", "47696", "71264", "816", "43991", "42354", "53396", "27164", "62700", "55640", "59384", "16471", "39309", "49552", "57966", "2800", "12133", "71598", "65375", "5007", "17472", "8128", "35621", "24562", "69806", "12728", "53432", "5758", "63512", "28933", "47799", "9824", "39447", "43899", "49975", "32007", "58858", "42317", "47887", "69521", "17558", "2439", "6183", "58232", "33549", "45420", "67518", "47761", "71167", "20905", "4766", "27591", "25485", "31213", "72875", "27314", "30413", "20471", "45577", "3061", "4059", "24138", "3335", "39099", "21987", "8366", "55062", "40228", "45458", "71907", "62285", "67258", "56385", "15684", "71027", "5775", "23271", "12379", "45763", "25463", "60184", "17047", "9030", "18458", "10609", "61144", "69788", "16623", "41758", "70312", "15394", "55725", "66757", "7180", "30669", "26708", "22612", "59427", "28098", "43438", "7285", "51880", "24159", "59226", "63759", "6594", "44781", "35201", "38706", "6455", "41271", "63632", "29048", "58331", "42291", "55837", "60384", "33491", "20504", "44128", "36735", "6925", "32685", "27728", "43037", "8110", "41781", "29942", "57352", "7241", "12498", "64667", "36139", "50028", "50901", "65716", "55614", "44055", "52338", "686", "53198", "28018", "52609", "51644", "7296", "10311", "64104", "42701", "22937", "69755", "17574", "28962", "37708", "36028", "12809", "40304", "60647", "49664", "28079", "23875", "25229", "14272", "10521", "58432", "1101", "30723", "20634", "66350", "36027", "44193", "31468", "9529", "13740", "4604", "50206", "50794", "27722", "40231", "23635", "46987", "41553", "69467", "56293", "15014", "40092", "36759", "30455", "35740", "64632", "6584", "71453", "68150", "41833", "71169", "41863", "53363", "60675", "53004", "32500", "23795", "66814", "8500", "38095", "63881", "1860", "11899", "46763", "18180", "32729", "2334", "5751", "30673", "25248", "71292", "43024", "49865", "30970", "21558", "72669", "4441", "13990", "61938", "36108", "46695", "800", "47280", "40274", "24836", "63259", "9564", "70368", "63148", "66051", "17490", "2171", "17415", "9956", "13829", "54425", "6391", "57622", "43604", "59194", "46518", "52697", "27932", "25481", "20748", "61600", "22427", "19048", "58007", "14052", "56268", "45801", "40399", "52398", "6370", "53966", "59636", "30535", "8002", "65515", "19209", "11156", "25616", "67730", "3295", "47346", "34941", "33091", "68210", "47927", "16239", "38470", "23151", "21645", "71135", "16690", "54350", "51820", "64869", "14131", "37285", "21836", "34848", "40614", "15639", "16254", "11457", "43358", "51712", "22110", "34121", "30019", "22945", "68661", "41395", "17448", "14270", "55613", "46905", "65666", "69424", "68069", "14703", "62218", "18671", "48525", "57614", "69575", "40711", "4048", "44119", "7409", "39565", "54464", "29597", "64051", "878", "59174", "36427", "50581", "33947", "20049", "10018", "3650", "23483", "65020", "54863", "71832", "55907", "36207", "36760", "66172", "21266", "32010", "61866", "40846", "31267", "4173", "20925", "14859", "50708", "68585", "48655", "33281", "35361", "54046", "45862", "4073", "43255", "12073", "27590", "47510", "61868", "7055", "59675", "57593", "42851", "63475", "70795", "45977", "60945", "21946", "27438", "37693", "18409", "27180", "57226", "4293", "33570", "11659", "29867", "43407", "68618", "42499", "18427", "184", "16890", "25355", "65925", "65257", "61245", "1528", "12262", "61475", "68309", "41413", "55245", "10194", "62848", "55710", "28244", "52277", "10509", "4573", "27756", "37636", "35750", "51594", "55936", "40800", "42943", "65903", "14889", "54287", "58728", "51827", "3316", "48361", "26234", "35123", "71204", "25713", "71661", "51558", "23955", "61185", "57513", "41832", "51525", "54948", "62610", "24093", "37741", "21470", "33478", "72656", "8774", "47515", "30751", "14118", "17195", "49851", "23465", "57293", "25764", "17562", "69461", "34408", "6100", "22770", "66268", "50408", "1115", "58893", "20699", "71777", "18968", "45617", "44111", "67254", "22111", "8636", "14133", "34763", "62008", "45105", "28437", "41671", "62299", "43402", "69159", "46871", "41407", "7580", "11065", "10390", "28997", "4780", "27444", "49207", "42946", "8840", "5148", "8063", "72745", "2661", "50850", "4172", "39026", "41696", "58818", "39707", "5236", "68395", "36341", "65908", "32084", "6509", "51426", "19002", "19608", "4606", "56282", "44639", "35532", "12911", "6637", "43617", "29934", "49915", "71692", "20718", "65494", "50126", "68086", "57079", "28670", "51145", "47973", "20558", "45095", "12346", "16560", "34232", "16544", "29474", "70143", "10690", "45860", "62922", "70865", "6500", "7928", "18158", "29930", "23292", "3269", "19980", "68176", "57694", "40928", "18729", "11246", "60194", "16191", "56868", "64856", "25868", "26655", "51005", "2166", "51930", "28849", "40790", "8896", "6740", "68544", "53842", "24162", "42741", "34581", "64055", "36776", "67783", "52110", "23984", "25187", "2624", "41688", "67435", "14332", "42658", "64923", "13168", "28373", "8696", "69176", "32827", "71966", "67117", "19284", "68167", "61219", "33296", "43425", "12636", "49645", "21689", "27234", "30300", "48291", "26676", "16852", "58437", "45084", "33764", "6648", "60360", "53682", "21686", "41018", "70579", "57987", "53328", "4784", "15192", "42591", "28934", "14447", "10791", "3313", "6891", "55303", "22668", "3225", "60100", "54731", "39386", "41350", "11497", "66970", "71174", "37394", "32004", "57846", "49164", "47574", "19109", "18201", "38264", "39837", "47900", "24738", "56731", "56423", "18651", "5102", "40264", "66642", "55960", "18317", "61787", "57074", "39829", "24164", "42446", "69321", "22080", "8148", "38929", "17249", "31492", "7961", "23049", "43546", "69017", "31767", "13087", "65282", "56023", "26196", "50811", "28692", "18141", "28900", "30013", "22742", "9379", "34744", "3445", "47385", "25468", "70474", "55179", "59182", "66125", "8196", "56290", "36462", "6167", "62729", "38423", "37214", "38275", "59540", "16011", "12251", "13349", "61521", "33344", "29612", "1236", "17308", "30253", "13220", "16200", "6149", "35759", "30867", "17202", "33078", "17536", "10949", "8974", "24660", "68728", "42987", "61985", "9216", "3708", "66869", "25649", "62979", "28113", "53933", "35130", "20992", "67813", "12132", "32469", "18754", "32322", "32623", "72484", "27848", "62057", "29554", "4601", "7505", "20359", "58623", "33087", "51245", "4677", "43095", "29626", "11758", "30137", "46897", "21284", "47610", "22674", "45920", "35142", "9643", "39181", "50525", "62026", "28700", "13826", "32594", "1134", "10341", "7387", "34087", "15926", "34251", "31097", "34851", "1685", "71243", "47833", "23312", "28829", "55903", "4420", "36187", "24174", "60712", "70079", "584", "42250", "53775", "11477", "23804", "31085", "34403", "37792", "55280", "40759", "45918", "7893", "60120", "8276", "60369", "24885", "47630", "40777", "63797", "45425", "38234", "15282", "64325", "28767", "43711", "26668", "66695", "38784", "12436", "2960", "13961", "1046", "69238", "6671", "47839", "20648", "49239", "2938", "17903", "28316", "41937", "58776", "23461", "34333", "28459", "39860", "830", "43632", "22148", "47629", "72188", "53544", "10275", "63227", "39908", "19103", "63833", "56538", "15992", "14926", "6851", "43743", "34170", "30242", "65009", "63899", "50858", "58027", "27778", "18347", "51244", "8446", "12655", "58233", "20914", "17449", "60253", "56173", "5727", "46271", "71569", "54587", "66931", "10637", "34118", "56379", "23371", "64767", "29613", "24732", "70489", "50800", "13042", "51098", "73054", "10289", "49394", "20388", "13412", "33716", "27424", "35160", "18812", "13358", "12642", "39023", "764", "23437", "12015", "30462", "68294", "36285", "44947", "28793", "28354", "13329", "1846", "58842", "42199", "49432", "18918", "6403", "31354", "47721", "1376", "11819", "50050", "13011", "63450", "6475", "61699", "28122", "59494", "64490", "67460", "20977", "16985", "58501", "34444", "35012", "65499", "54717", "17925", "55972", "12225", "52064", "31551", "16127", "16086", "1689", "17538", "38469", "1585", "672", "16883", "62690", "23864", "26077", "41580", "44866", "11197", "41363", "46813", "57049", "25275", "65063", "18014", "50387", "54659", "46724", "4546", "7039", "59856", "61496", "18976", "47368", "14139", "18037", "32217", "44950", "16765", "2717", "57381", "62901", "34369", "39999", "18544", "22059", "71632", "28480", "20816", "47415", "58775", "70659", "26898", "60835", "66131", "44991", "39689", "27532", "68999", "8459", "15278", "19818", "65643", "59332", "28586", "53024", "31759", "31330", "46874", "28972", "9020", "11119", "67838", "46223", "24153", "52155", "72267", "11231", "34072", "71952", "15093", "38112", "38688", "56775", "56440", "64025", "5226", "556", "9411", "71285", "48845", "47051", "46059", "65660", "32613", "63969", "51025", "49471", "51432", "39672", "4670", "43044", "5820", "25112", "72311", "22933", "10974", "55121", "5185", "35209", "31507", "28503", "28273", "19014", "67638", "16737", "7746", "69460", "13001", "68621", "6295", "16764", "31513", "35985", "40365", "40516", "40660", "10263", "47065", "7452", "14228", "46661", "27393", "55395", "39615", "26423", "32732", "6254", "59519", "39660", "29089", "48855", "12566", "22267", "44159", "29835", "58187", "53928", "59345", "54811", "33776", "45588", "8584", "9920", "31361", "58298", "9312", "17359", "52261", "53711", "56581", "11005", "27493", "5818", "58018", "42597", "6565", "1927", "48851", "4723", "14494", "23042", "61676", "40492", "69602", "14162", "50249", "33292", "11644", "16604", "61953", "59902", "71216", "49401", "3276", "44335", "63772", "16039", "8016", "62459", "17082", "67417", "37083", "28600", "21866", "53851", "62882", "50797", "59889", "30141", "23297", "16028", "23380", "14181", "23779", "67061", "22344", "4763", "10796", "8383", "60745", "13407", "32792", "18774", "61798", "52229", "29720", "66348", "2458", "3743", "32142", "39017", "46498", "62063", "52265", "52340", "5974", "60508", "67197", "8713", "60445", "30965", "59848", "17001", "27862", "66476", "29430", "57653", "1340", "28926", "73060", "68033", "49476", "67307", "72089", "25270", "13210", "57853", "69871", "28154", "23638", "22608", "68043", "16782", "46646", "17117", "36775", "7931", "65219", "41075", "43280", "27295", "49962", "8342", "13382", "27814", "28655", "40125", "8537", "16035", "70155", "32966", "6112", "39819", "9309", "26195", "784", "71007", "72333", "51092", "68530", "61084", "63308", "34127", "982", "54705", "7593", "41545", "72119", "59845", "69223", "51868", "63096", "29157", "65673", "12349", "55718", "18388", "41751", "30490", "72521", "1609", "3680", "68116", "19288", "60059", "67683", "40645", "2347", "53471", "40889", "49823", "62756", "62740", "3249", "32538", "4983", "16426", "60851", "49702", "50777", "54140", "22385", "65816", "68592", "66156", "11200", "38237", "38216", "42603", "23829", "38566", "17628", "18425", "27043", "15830", "40464", "61142", "66870", "1507", "4087", "23009", "66201", "56851", "5136", "52913", "2092", "21052", "3525", "30127", "52394", "44724", "33055", "8938", "14573", "42221", "4930", "30358", "70967", "19279", "61086", "45397", "3037", "46156", "4721", "18373", "67570", "60238", "31442", "40334", "36321", "31463", "47088", "59677", "7236", "12371", "11676", "11212", "71728", "37045", "11684", "45570", "62145", "13526", "19963", "60866", "13986", "38165", "16361", "64766", "9657", "39731", "13114", "22208", "41554", "48995", "57466", "59046", "13246", "25761", "58192", "46693", "50027", "64215", "45423", "63388", "56869", "7099", "67094", "50343", "24711", "20654", "49359", "27209", "40001", "37328", "26336", "34893", "24903", "46810", "6977", "29883", "59642", "61376", "54047", "48547", "63477", "1873", "12988", "48403", "66243", "7320", "43659", "46240", "60825", "11506", "4892", "9626", "3421", "12740", "62560", "12213", "28184", "32490", "28674", "45058", "26653", "15128", "50688", "41403", "69939", "22911", "14785", "15317", "57976", "49468", "64244", "6638", "14548", "72348", "32667", "58489", "17688", "60102", "53092", "24733", "54305", "49418", "27243", "14993", "16187", "7325", "7288", "57017", "43576", "48229", "2628", "61059", "26097", "18829", "58621", "53075", "24513", "53718", "43825", "51397", "7539", "56046", "72627", "45410", "18584", "32417", "16332", "70951", "30110", "11277", "62733", "64753", "28743", "66697", "34143", "61218", "33989", "7558", "13306", "49236", "55278", "42382", "32528", "16095", "50635", "16226", "71217", "38016", "11224", "18457", "70351", "12853", "25180", "40859", "71040", "37388", "13443", "35650", "10537", "46418", "3431", "748", "43826", "68936", "52886", "56733", "36974", "24605", "45986", "17738", "31102", "43731", "40755", "9317", "43175", "56565", "17271", "20081", "23963", "2398", "23977", "12019", "33049", "19332", "38699", "6842", "71989", "68274", "45430", "46762", "37903", "56122", "33629", "47262", "17961", "5800", "14129", "35981", "53725", "23701", "27892", "58857", "12999", "22043", "38741", "61172", "25552", "18873", "7993", "26411", "22904", "49532", "19283", "63111", "13234", "71424", "53773", "71723", "69586", "22444", "50139", "62233", "6570", "48779", "6218", "39192", "49030", "35641", "46665", "18514", "56303", "72281", "39643", "54066", "66267", "64198", "8481", "19873", "26264", "70811", "38511", "50244", "24217", "53616", "43782", "56531", "5766", "24330", "61813", "30444", "37339", "11980", "31219", "12464", "26292", "22667", "40138", "22304", "3208", "49093", "46000", "18588", "41647", "67702", "1156", "66930", "7356", "914", "37352", "60473", "70493", "28366", "56894", "33307", "2326", "62509", "43029", "42161", "33997", "26887", "69173", "71326", "20540", "49492", "60778", "32919", "55271", "15241", "769", "51371", "8471", "19976", "70019", "43383", "30219", "72079", "63977", "15025", "59135", "57906", "52252", "70129", "70757", "57968", "54038", "12906", "27964", "21691", "1610", "16090", "22983", "15262", "57332", "48799", "71006", "57061", "15126", "56959", "29040", "64517", "12548", "27405", "55075", "33888", "10769", "22969", "49349", "26631", "20512", "6521", "67624", "679", "44966", "25924", "2898", "54391", "47777", "60876", "53565", "51087", "39075", "37988", "32038", "57962", "66341", "55526", "15050", "2416", "11527", "72169", "32110", "20936", "19791", "23262", "30713", "63626", "43057", "50298", "4434", "19167", "14562", "48643", "25290", "12570", "22483", "16288", "91", "40061", "9829", "5247", "12135", "20993", "67646", "16019", "68047", "48509", "5934", "16906", "21646", "5919", "51866", "48187", "104", "34525", "71399", "28186", "48631", "12644", "29426", "18108", "18325", "10708", "65895", "65223", "9073", "64154", "14433", "70398", "38952", "34356", "58991", "54376", "25773", "9168", "8690", "25977", "52257", "3561", "61423", "64037", "17368", "24296", "27841", "47342", "1674", "3781", "63177", "33726", "15972", "5282", "8461", "60960", "14764", "69160", "30692", "11789", "44032", "61973", "34965", "11133", "54233", "70642", "48057", "25874", "70385", "70426", "18793", "30923", "4686", "43568", "54632", "70452", "59348", "17314", "19425", "70242", "71385", "45159", "56014", "11510", "68846", "9128", "61256", "32058", "6644", "59244", "18587", "5182", "57718", "16742", "39436", "14995", "2811", "7612", "55917", "9137", "69518", "12205", "65470", "50345", "69983", "26652", "7760", "67470", "27214", "20625", "23113", "6003", "36173", "32669", "1748", "22364", "182", "70353", "2726", "27170", "2606", "48321", "4033", "62898", "69604", "43465", "40037", "31731", "42383", "26189", "59979", "6342", "8381", "6397", "4866", "37149", "4898", "13342", "60337", "22034", "71646", "32288", "33438", "6597", "3172", "65455", "7791", "33151", "73179", "37515", "55067", "44826", "57813", "42677", "34515", "3761", "23357", "24119", "48654", "73046", "66822", "37002", "23614", "25428", "26536", "11057", "54776", "45830", "70540", "49754", "47398", "19970", "71669", "71350", "18220", "15859", "25916", "59217", "60743", "64715", "16412", "3221", "11737", "72381", "25186", "50111", "4943", "9330", "72541", "20559", "68667", "33109", "68153", "6731", "67939", "24911", "31396", "55612", "44889", "1527", "4024", "62981", "17150", "39597", "70334", "42648", "44895", "35914", "32768", "32394", "24745", "5068", "71724", "52365", "22138", "48177", "14684", "26820", "35364", "30878", "6074", "58408", "12949", "1637", "56015", "3139", "8057", "35605", "4156", "28498", "51579", "67862", "29066", "51844", "39769", "13491", "41744", "69297", "35278", "42734", "59438", "72076", "36029", "13422", "5922", "64752", "51947", "19951", "6878", "62154", "8873", "36919", "20160", "33669", "14467", "41997", "72555", "35713", "69595", "18008", "10554", "51904", "11055", "25181", "37432", "57048", "6099", "71903", "24957", "58785", "51812", "56113", "8807", "28641", "58022", "16396", "64729", "33647", "42675", "57479", "46452", "50576", "31766", "2011", "18857", "14991", "50326", "29124", "58078", "71600", "11403", "3493", "2415", "60225", "38759", "7140", "386", "29524", "41047", "40515", "13356", "37291", "7404", "32925", "72042", "62391", "39222", "33746", "43642", "69422", "31298", "50845", "15151", "43000", "49312", "51566", "68398", "23552", "45338", "3430", "51283", "16278", "26857", "38377", "54546", "15981", "69139", "69186", "9227", "27172", "3667", "58644", "42080", "50275", "29146", "57756", "34420", "6944", "50962", "2539", "17008", "3805", "35443", "30445", "50648", "12466", "63077", "41156", "59596", "55717", "20315", "63917", "27533", "9123", "14377", "59555", "57231", "34426", "27692", "12807", "59211", "21296", "35493", "62005", "7115", "11709", "10080", "18833", "11741", "61384", "2716", "836", "39668", "4600", "18153", "44860", "43761", "61684", "38810", "1453", "33768", "57168", "67974", "65621", "4939", "46186", "47809", "56486", "27660", "62337", "18919", "14019", "11720", "54799", "19493", "12202", "53031", "64320", "69428", "18874", "54242", "3487", "61472", "804", "63251", "8533", "65367", "37970", "61135", "29282", "47881", "24004", "72260", "65471", "51803", "62751", "17573", "21280", "43980", "52147", "20929", "36855", "4563", "19637", "16632", "46678", "58891", "4455", "60877", "21180", "50519", "4562", "27139", "57121", "31955", "12007", "19530", "48759", "9410", "3639", "59722", "33589", "7635", "9321", "67922", "57050", "62919", "63621", "72854", "69308", "52174", "30158", "3833", "47904", "34488", "29632", "37833", "41123", "36998", "6036", "62792", "31402", "8219", "68965", "48044", "33350", "48731", "51761", "6339", "2971", "43936", "28415", "57726", "9043", "53996", "50924", "43145", "22435", "18096", "52865", "24773", "2997", "18669", "62925", "16713", "66925", "14745", "65469", "46548", "38120", "71746", "30467", "10983", "42474", "4770", "63555", "49586", "2040", "56990", "61969", "23986", "32361", "36431", "43967", "72409", "3733", "7247", "40734", "11846", "27238", "50046", "8161", "4194", "6495", "8812", "62481", "15072", "65320", "41080", "64827", "72501", "64770", "29126", "69754", "56306", "23743", "62958", "57396", "15553", "18155", "50134", "28252", "52033", "19943", "19805", "21471", "28563", "35599", "36528", "68617", "15005", "63279", "4978", "48918", "40694", "41142", "48195", "45049", "40866", "29689", "30834", "37588", "54499", "23478", "52107", "54977", "9765", "71017", "48214", "69827", "18819", "56688", "15438", "50109", "33228", "27325", "52231", "39788", "14541", "36040", "67749", "61230", "70380", "31861", "8522", "56867", "45348", "30109", "61382", "47406", "54750", "19810", "9819", "63075", "32941", "20021", "49057", "65365", "66929", "5542", "67409", "35998", "69914", "42348", "55137", "11865", "63146", "11324", "60304", "28285", "36716", "43403", "25763", "38123", "13469", "48453", "64453", "49540", "2880", "21657", "32654", "55034", "23117", "69342", "35197", "71077", "1961", "22288", "6274", "53533", "36310", "6538", "15870", "63123", "15801", "63194", "43638", "50803", "4941", "13870", "9484", "37323", "69947", "69718", "65139", "66493", "20828", "45357", "6208", "20774", "26023", "9352", "45354", "45011", "57960", "7756", "10370", "11371", "68181", "66989", "3079", "20094", "55162", "35065", "1652", "35852", "37535", "44123", "51621", "51427", "64597", "24070", "20984", "16527", "7308", "52428", "49287", "50500", "36224", "12613", "2335", "63661", "21794", "2113", "28512", "11121", "21434", "56054", "1359", "69052", "36666", "64155", "7420", "59202", "45026", "23802", "25508", "2272", "71120", "44954", "37426", "45556", "71666", "27362", "57601", "9807", "4148", "67582", "58687", "68727", "57716", "26037", "63352", "53573", "2442", "26726", "53733", "18242", "13608", "20574", "43906", "67806", "23281", "35140", "51494", "56182", "33282", "72202", "3923", "51892", "4749", "68542", "44101", "56498", "1792", "46881", "71288", "45456", "38755", "8116", "68127", "20309", "1758", "16348", "29703", "51146", "69405", "22371", "5733", "47625", "58384", "32224", "11695", "47884", "16652", "31192", "49011", "56694", "21039", "65577", "60128", "30164", "54643", "2148", "817", "41384", "69829", "23340", "22981", "28203", "40243", "2047", "52565", "34707", "93", "1672", "67067", "66104", "72984", "3438", "52058", "21483", "51208", "10931", "59901", "40993", "48287", "18843", "45698", "13203", "17389", "68285", "12607", "41603", "63552", "72211", "62647", "1523", "69024", "35489", "33368", "66702", "53443", "56334", "52726", "14569", "69850", "40850", "3700", "42762", "63041", "30758", "19489", "44058", "69989", "29314", "52622", "50624", "20102", "4555", "4919", "47933", "72625", "36866", "1670", "24783", "35989", "41328", "53074", "46514", "11520", "29655", "52385", "63286", "20827", "13453", "40117", "65384", "62107", "6179", "71410", "12245", "11521", "19339", "43552", "63819", "13975", "9144", "12920", "50161", "28816", "70839", "14838", "49858", "35435", "12288", "25263", "49967", "31736", "25209", "30752", "68291", "50801", "16492", "51800", "45571", "35042", "38729", "22880", "4587", "12606", "18578", "14815", "38797", "33467", "64452", "42320", "47277", "14816", "28555", "31644", "44591", "23723", "41750", "72923", "22109", "1937", "32046", "41613", "35922", "64430", "9531", "53465", "54788", "25191", "35289", "59430", "7606", "54183", "21053", "34788", "70810", "11890", "46243", "58700", "57505", "37205", "48605", "3000", "56446", "14629", "54342", "407", "14092", "19251", "60952", "19066", "64976", "16749", "43631", "53699", "7557", "55909", "26014", "24426", "71130", "832", "27553", "17494", "65156", "25950", "57207", "17770", "7200", "36441", "63616", "66820", "35617", "9393", "13463", "36291", "15889", "47313", "12519", "9372", "46210", "46809", "10625", "20392", "44595", "7844", "12143", "45695", "29794", "37996", "69543", "24326", "21751", "68672", "16934", "27480", "16779", "3299", "40810", "45035", "71750", "37916", "11253", "16799", "61064", "52043", "72730", "14959", "40203", "1387", "25749", "47174", "21260", "7072", "57607", "51643", "59093", "48967", "14452", "38554", "4819", "53756", "23497", "27543", "28391", "37865", "69493", "63808", "38322", "30501", "64110", "68414", "57109", "49620", "45720", "63845", "9155", "952", "17136", "23359", "68786", "43174", "41679", "45173", "57917", "53310", "11996", "65024", "54169", "54216", "57510", "53129", "44088", "72274", "4867", "13916", "39493", "42427", "50307", "57769", "13173", "58650", "12445", "48132", "62051", "71491", "29797", "22198", "55590", "39721", "11628", "644", "41356", "31258", "32530", "39895", "30293", "45544", "22934", "33988", "53134", "4552", "32568", "37318", "70800", "61925", "59610", "29877", "11936", "5097", "47427", "7193", "16057", "22947", "67741", "60581", "6811", "26897", "23658", "4682", "44080", "46711", "33017", "32031", "69877", "65948", "67305", "18161", "52803", "60066", "27168", "71138", "6937", "48920", "3819", "5173", "19967", "37685", "26477", "26180", "8311", "15154", "9196", "33189", "46804", "24558", "64874", "58816", "13798", "3220", "34738", "28471", "55421", "6253", "49767", "8734", "63409", "16818", "33503", "30504", "72101", "38104", "34062", "44693", "60783", "38098", "62024", "39856", "39893", "34569", "71623", "34247", "12186", "28477", "3735", "13392", "38806", "46838", "57991", "63362", "28109", "71335", "48487", "5206", "57227", "61243", "14871", "4801", "70378", "67727", "40146", "38754", "37299", "13113", "24992", "30237", "45475", "19704", "43891", "21080", "70446", "63179", "23578", "49280", "18193", "52519", "19104", "28197", "23257", "67222", "61469", "18559", "1634", "4473", "56917", "46424", "3307", "10546", "49631", "67439", "48937", "30902", "19570", "43810", "5621", "996", "19719", "6936", "27603", "24704", "30166", "33861", "41854", "53050", "35967", "10875", "57294", "7354", "72687", "48278", "12434", "2306", "50918", "57635", "65721", "20420", "7897", "62253", "28921", "48405", "72178", "41282", "53440", "48975", "31195", "64994", "42009", "57359", "17654", "58980", "2730", "41789", "70220", "41467", "54281", "21543", "58252", "56116", "67055", "5012", "64115", "60400", "24275", "60334", "64892", "24086", "55610", "72227", "9530", "43105", "48670", "28398", "67845", "42982", "28483", "46395", "50154", "32778", "63838", "42646", "13305", "29754", "52560", "63709", "60916", "20233", "41444", "39161", "40816", "71755", "8070", "21864", "19114", "53555", "63571", "37621", "71381", "72166", "22437", "32122", "72410", "40294", "59799", "54466", "38240", "12438", "30216", "52395", "18737", "56168", "20478", "50340", "142", "45978", "15013", "61960", "66643", "13525", "63230", "63264", "59454", "24809", "31900", "1179", "29902", "10473", "72989", "47671", "33683", "30190", "43663", "41416", "66556", "56102", "16521", "28769", "21991", "21648", "44879", "63211", "1279", "18112", "45907", "24651", "54326", "15799", "58764", "45982", "2887", "30036", "9967", "20716", "8140", "29800", "21684", "58507", "60271", "37429", "15521", "55580", "40154", "50054", "39178", "63861", "33830", "17196", "57066", "14687", "9127", "66806", "15610", "52983", "47297", "72525", "43959", "54708", "54239", "66204", "67882", "6967", "69555", "14572", "9662", "41493", "18033", "10510", "28687", "23888", "66523", "54221", "21094", "4571", "60625", "28784", "61168", "20683", "41809", "36235", "40965", "66566", "8068", "19161", "60056", "31948", "66444", "15349", "33747", "20265", "25863", "35428", "54198", "18263", "49675", "68962", "24341", "6707", "46899", "13209", "65342", "63210", "63181", "15908", "18208", "69266", "602", "68160", "32094", "42883", "16746", "35700", "12440", "61228", "34250", "8941", "69409", "16658", "57752", "62147", "57375", "50967", "51161", "53255", "8915", "6119", "57566", "27611", "38950", "47450", "16070", "59045", "26392", "30901", "64882", "62491", "2564", "69085", "27820", "14644", "60433", "43957", "14734", "26127", "68058", "36691", "66382", "27719", "14930", "64690", "13816", "22057", "1435", "48628", "10147", "31370", "32310", "35831", "35594", "32243", "16701", "52822", "59802", "51693", "57202", "56025", "51831", "65847", "19917", "4259", "17135", "36721", "18536", "44027", "32856", "44360", "51801", "5656", "32357", "47493", "27286", "54686", "44042", "57114", "53105", "59238", "59032", "70933", "58804", "13614", "12004", "24700", "23353", "8758", "46927", "16885", "12026", "41828", "66583", "64010", "40335", "60365", "58297", "58481", "47148", "54284", "11609", "43256", "34532", "38536", "57975", "5923", "58966", "43301", "32681", "51372", "65313", "70269", "10899", "18335", "30860", "7047", "43966", "72400", "67352", "24896", "30660", "41952", "35747", "10907", "39979", "57446", "27955", "29527", "28750", "5918", "38932", "20173", "62441", "11345", "11255", "68977", "60958", "15615", "2948", "437", "29815", "25083", "57744", "57963", "48212", "71471", "61275", "54935", "48277", "60072", "23841", "69962", "26512", "65195", "22636", "42761", "32380", "49317", "70247", "53076", "72962", "15725", "20848", "70756", "48381", "15541", "64102", "70390", "60326", "29647", "16119", "129", "46129", "43474", "15677", "65707", "9751", "68679", "62224", "45259", "59788", "67131", "71910", "57920", "32185", "22835", "29047", "57319", "6745", "57523", "34338", "58357", "13391", "66443", "46132", "52927", "18782", "4041", "19816", "40048", "30077", "21756", "68289", "27718", "64445", "45205", "40029", "11746", "64762", "35134", "49969", "28969", "23219", "64555", "37181", "54152", "62044", "2250", "8631", "41360", "481", "2286", "22215", "59296", "71932", "61588", "27124", "61770", "53606", "18120", "57431", "64803", "68199", "73034", "20785", "48080", "26935", "52465", "70118", "42462", "41525", "53965", "14346", "8561", "70257", "19745", "46939", "9354", "46349", "27129", "19892", "25514", "40388", "44127", "4381", "38558", "21989", "19703", "70592", "44031", "53632", "29634", "22952", "13318", "68577", "23529", "20373", "5655", "64482", "72380", "31515", "53259", "45547", "32982", "48375", "70614", "31194", "35670", "68371", "53592", "54178", "71395", "38573", "41323", "17409", "33202", "28771", "4577", "18264", "8849", "55764", "959", "11439", "38851", "65669", "12068", "69326", "50176", "16183", "41759", "51767", "42119", "27747", "54803", "47074", "63447", "21036", "67303", "64887", "21583", "70138", "32268", "57800", "68812", "65892", "30420", "13136", "20070", "36", "58721", "49285", "25494", "72353", "45701", "42162", "4229", "39903", "15336", "22963", "47349", "68238", "26305", "6508", "60470", "46174", "27729", "70724", "64187", "1926", "172", "44280", "31545", "4651", "66312", "5374", "13707", "12666", "6131", "68848", "18020", "20696", "41098", "32811", "48776", "62723", "8190", "21005", "5558", "7783", "73026", "61830", "27673", "43110", "29026", "24037", "53473", "15751", "43489", "22195", "8306", "37520", "44839", "27567", "60049", "35321", "48542", "56837", "42714", "43052", "55340", "19031", "41349", "20266", "43801", "390", "8783", "5311", "68527", "23608", "4180", "54209", "9206", "50792", "27836", "20114", "49129", "66006", "50645", "37955", "16111", "25790", "19069", "47977", "14872", "22334", "56622", "17570", "43935", "2714", "28141", "53268", "59693", "52896", "23147", "35188", "70755", "62556", "7369", "16309", "62869", "52730", "57935", "69907", "8871", "29108", "5476", "48019", "60201", "67020", "65340", "7315", "15781", "21719", "25594", "15465", "37490", "45033", "72567", "33653", "9709", "53126", "38578", "32828", "73065", "30591", "71082", "26113", "18168", "36876", "35367", "313", "39542", "72374", "28790", "16761", "73057", "35040", "58324", "65368", "8782", "41397", "62983", "55618", "42858", "37445", "72057", "70391", "57753", "8039", "24819", "46746", "48821", "17749", "42141", "52442", "41181", "27688", "938", "3749", "23156", "54919", "55095", "55250", "33328", "11148", "72648", "40124", "8292", "54535", "25970", "19483", "48718", "28402", "29239", "53077", "64514", "58281", "2105", "67208", "2224", "55257", "44505", "50877", "64310", "37963", "62364", "40459", "65232", "66109", "71535", "21570", "61077", "8482", "15759", "56155", "20218", "20207", "30471", "22044", "36758", "57239", "20981", "60340", "16867", "31022", "58021", "14073", "60813", "29319", "56522", "67271", "18614", "36192", "43515", "19367", "23463", "15915", "33677", "42282", "66525", "32800", "60141", "52100", "63149", "9061", "67404", "67308", "16143", "20454", "53758", "4643", "65453", "23544", "16711", "3866", "40460", "2032", "9289", "60472", "18365", "69177", "70282", "13871", "33517", "57600", "8616", "30783", "47345", "70237", "35296", "58069", "66360", "41897", "22639", "66442", "21081", "39187", "34948", "32428", "25754", "31835", "28365", "26912", "25859", "21334", "14921", "18360", "17193", "72759", "10863", "55320", "36368", "37907", "63872", "9065", "46676", "27505", "31747", "51948", "57196", "5268", "8222", "27221", "16271", "47794", "23897", "13399", "55379", "21514", "12640", "21526", "29684", "6242", "13487", "16112", "32946", "16622", "53459", "30547", "21951", "40987", "40359", "22614", "58091", "11363", "14861", "47375", "62306", "8937", "21507", "9162", "25908", "36908", "71618", "65255", "48319", "59618", "30065", "4484", "31620", "37687", "41773", "64498", "29193", "15559", "49259", "7305", "30084", "49641", "32484", "40898", "1571", "29869", "19848", "62338", "38833", "818", "58626", "64518", "7975", "44493", "1373", "26530", "14415", "37062", "51408", "22661", "53677", "8551", "56069", "57475", "67590", "26513", "48505", "10259", "12267", "242", "27786", "27711", "1625", "9016", "42852", "46972", "1979", "68121", "70020", "27065", "69128", "4453", "52008", "60998", "23561", "69083", "40332", "73102", "38839", "12025", "61908", "51547", "13661", "57148", "27619", "48154", "6483", "64084", "3850", "8846", "62749", "8767", "37413", "5036", "24163", "69969", "50515", "7817", "57854", "24265", "41972", "11706", "66551", "35685", "27677", "66091", "5821", "71933", "17726", "60526", "13808", "23562", "52899", "36673", "32932", "27966", "29212", "57602", "62069", "17623", "15770", "25465", "70588", "41520", "10775", "69355", "53998", "49205", "26101", "57875", "30825", "4531", "26482", "36585", "51349", "42422", "1247", "42032", "62093", "72093", "59374", "46581", "13490", "52631", "20161", "59020", "71870", "4166", "35765", "46836", "49058", "23338", "52805", "61778", "29628", "36228", "49268", "23107", "5395", "45282", "60816", "64389", "46640", "244", "40521", "30360", "54412", "45669", "37055", "2424", "65082", "71611", "22801", "26565", "21541", "59024", "32579", "22917", "45121", "9585", "65152", "5233", "19182", "46912", "68173", "61044", "33854", "5593", "15777", "64950", "49703", "45051", "69058", "56344", "28142", "32111", "26179", "8290", "31389", "27758", "585", "64148", "28710", "56529", "65321", "28916", "12152", "67227", "1160", "12472", "10743", "24644", "63939", "34864", "44479", "18720", "39216", "14463", "72063", "52825", "71330", "57685", "51338", "23464", "46817", "39137", "30062", "2890", "47650", "58714", "34246", "67389", "36451", "55132", "39414", "21565", "10778", "63258", "19193", "16514", "57993", "64989", "58594", "62643", "5034", "43580", "46279", "4098", "5763", "43864", "17173", "29297", "21211", "26000", "12843", "57344", "71916", "35962", "21990", "25800", "58777", "51303", "455", "4870", "55569", "8067", "71114", "52144", "48695", "43523", "7473", "25845", "27854", "57563", "34526", "34778", "64457", "66498", "16121", "10052", "67052", "73249", "63821", "67481", "3538", "11070", "72957", "43453", "64497", "55736", "66240", "34528", "64342", "326", "37408", "10328", "62878", "38630", "74", "7788", "18917", "22221", "8098", "48874", "21107", "60904", "2367", "54989", "27371", "20906", "32112", "17214", "59508", "32585", "47018", "8188", "48524", "37960", "40469", "39676", "27396", "14849", "52486", "24467", "4905", "31709", "48274", "59119", "14356", "70983", "18411", "40434", "71606", "34003", "18481", "46682", "33441", "21161", "52483", "17835", "24747", "10317", "25993", "10700", "38262", "25774", "51281", "25316", "38464", "50605", "44053", "32634", "1303", "16574", "65357", "72415", "59944", "32475", "35742", "50721", "1706", "65188", "56564", "37946", "49391", "6421", "22968", "18067", "63926", "49948", "30181", "33270", "57070", "208", "34704", "60781", "30543", "60244", "38994", "26059", "68434", "70892", "26125", "32149", "24766", "41752", "18716", "45847", "41590", "8894", "71081", "12451", "62809", "32450", "30225", "58614", "41049", "40371", "50528", "52777", "54965", "51727", "12748", "62828", "68396", "36484", "69526", "49859", "31536", "8834", "23956", "45650", "72481", "67494", "6346", "55494", "68715", "820", "25716", "7994", "17939", "72124", "17620", "67537", "72828", "60311", "5513", "72261", "71172", "55349", "56984", "29456", "1475", "12598", "17646", "27427", "20090", "59666", "45629", "23301", "50564", "11685", "9041", "47535", "31933", "20097", "40623", "24652", "17722", "5771", "43001", "45496", "72369", "31524", "24930", "35469", "29359", "47948", "36377", "18393", "33234", "24662", "12616", "43934", "30708", "44621", "14506", "45114", "67913", "42045", "56664", "8334", "57334", "6001", "56577", "13021", "35992", "55507", "27607", "11051", "16139", "52181", "33436", "21475", "49104", "30299", "66963", "49462", "68270", "18895", "53274", "15458", "30064", "6641", "66289", "53060", "66009", "32170", "39165", "31947", "12104", "34794", "2094", "3306", "7871", "22078", "21446", "57964", "16963", "59363", "69291", "14675", "18344", "70986", "36523", "53073", "41185", "41332", "58159", "12884", "3709", "42622", "10306", "67984", "51141", "72476", "55597", "536", "22250", "33169", "4401", "23234", "46745", "6998", "13240", "18691", "32210", "37396", "8022", "22867", "53006", "19330", "68062", "9500", "29110", "7492", "10016", "14657", "71599", "50685", "27165", "35618", "45505", "44487", "44491", "71551", "20492", "16913", "38891", "6884", "61315", "24102", "11012", "21274", "63048", "61689", "52723", "3118", "71927", "59328", "44657", "8327", "58993", "61412", "70740", "32737", "8642", "19468", "56614", "33812", "56018", "32761", "14766", "42763", "63839", "61926", "51733", "34963", "37624", "29539", "26910", "53308", "5642", "70637", "55975", "30845", "17863", "49522", "42421", "44733", "8257", "67472", "52873", "51475", "288", "17183", "3595", "2001", "63521", "22305", "15044", "16165", "18960", "31636", "43725", "64512", "59938", "73097", "10982", "61114", "3981", "67339", "32498", "781", "18837", "41276", "57046", "24256", "54847", "7208", "7577", "41441", "58273", "69181", "67471", "10001", "53218", "1673", "48632", "17533", "37517", "26697", "21202", "69787", "3332", "47698", "43615", "58377", "40007", "17391", "11645", "22895", "57276", "46425", "42048", "68114", "4469", "10012", "45356", "59278", "56134", "36457", "2128", "1732", "31606", "59648", "14016", "63261", "55929", "10513", "10946", "61236", "31259", "61091", "73209", "789", "4065", "31789", "61495", "68557", "54174", "69345", "48769", "59108", "67936", "51660", "68674", "24844", "65978", "19622", "53194", "18212", "57825", "4465", "14866", "50055", "28341", "40637", "1464", "13172", "35753", "30730", "25769", "44267", "8821", "64992", "34608", "41776", "38257", "14382", "11925", "17141", "1827", "58677", "61379", "23127", "31434", "12351", "24239", "65967", "15867", "60210", "7674", "11544", "51550", "21589", "36825", "25743", "29249", "18171", "23537", "55005", "31188", "44109", "3250", "66358", "61664", "19447", "15370", "11759", "16763", "73123", "44920", "31251", "48931", "6374", "20897", "3996", "71718", "23426", "47679", "6635", "11852", "12746", "2279", "25926", "61934", "18560", "51484", "13444", "4647", "17731", "9938", "34734", "61525", "7639", "40667", "60655", "45543", "21929", "48150", "8688", "58815", "57300", "49296", "67713", "34328", "48961", "61941", "36051", "71865", "71299", "8677", "7531", "41031", "60812", "30325", "46071", "3409", "31500", "17373", "35239", "32012", "56213", "44592", "38927", "53249", "68091", "62929", "52624", "21728", "56802", "61895", "62399", "67698", "12690", "52052", "12208", "43108", "66777", "73048", "44145", "45351", "13266", "55574", "20892", "41944", "65780", "39236", "8560", "67312", "25531", "28639", "54226", "60907", "34508", "29436", "51837", "12290", "12895", "35703", "16625", "53506", "9436", "6215", "69136", "42992", "61107", "46075", "20288", "33453", "36790", "27834", "26152", "42676", "65468", "33014", "20249", "60973", "47543", "23989", "5661", "72254", "26129", "71533", "50698", "56373", "26407", "9226", "6169", "14588", "11680", "40120", "16735", "16170", "67253", "15099", "57363", "57979", "63247", "38153", "37074", "31700", "44557", "24890", "25543", "30238", "20670", "67347", "35053", "26494", "21089", "72507", "4426", "31217", "47183", "22704", "47031", "56601", "61075", "5728", "45876", "17766", "42062", "4717", "5952", "54976", "8285", "17027", "34361", "29753", "64054", "34207", "62686", "10667", "19324", "67474", "31775", "19079", "7901", "25070", "25319", "25408", "55385", "45645", "33763", "34155", "43564", "9827", "20357", "3780", "14732", "2254", "40489", "27076", "47356", "39902", "57124", "31960", "60374", "70216", "2631", "67823", "5406", "47239", "22452", "20691", "17004", "34801", "51959", "11017", "32859", "241", "8154", "16334", "577", "46629", "66564", "65899", "14205", "30798", "16078", "41983", "72300", "9681", "54556", "42736", "59017", "20012", "12244", "70011", "11635", "32767", "10780", "24430", "72682", "11897", "59980", "57084", "11276", "42989", "23543", "25999", "36598", "67888", "27621", "11836", "49220", "30813", "55735", "71530", "40775", "16021", "12673", "55230", "29641", "17438", "55046", "60613", "38471", "22833", "43702", "41655", "33963", "61993", "60978", "15993", "23859", "540", "32921", "69624", "19281", "49909", "45443", "29413", "35721", "27015", "37679", "70145", "59483", "4188", "16796", "63485", "42180", "56908", "52734", "56328", "67251", "12752", "31817", "1262", "3973", "39182", "56308", "60170", "43721", "70995", "45056", "69603", "27126", "31540", "43078", "5522", "29547", "32324", "9378", "17240", "21610", "9248", "17505", "32089", "42654", "67259", "70882", "13629", "4659", "31718", "2731", "22431", "36459", "36535", "54054", "17023", "16870", "54428", "21549", "71121", "63465", "3777", "56735", "63776", "12031", "6750", "13726", "34371", "11474", "62707", "34019", "16909", "30521", "34138", "10075", "36745", "33245", "70566", "67697", "30709", "25559", "3439", "2954", "5942", "8204", "25617", "21124", "28760", "6834", "71570", "8747", "63205", "67791", "9545", "60562", "64313", "43727", "30224", "65598", "18200", "56541", "54201", "58664", "1462", "58952", "30075", "52325", "43027", "22063", "55545", "68809", "72000", "926", "63378", "38766", "22446", "35863", "29245", "40404", "2132", "57095", "44445", "20746", "234", "14247", "11837", "60505", "7533", "14683", "72265", "54582", "20005", "29669", "16667", "42369", "7203", "4160", "28636", "58031", "17241", "56648", "49570", "36848", "32835", "7375", "51778", "39090", "41709", "59594", "14341", "6144", "28715", "7336", "55258", "39211", "56969", "59037", "6069", "60022", "24801", "21503", "72025", "26519", "48565", "10191", "21888", "60818", "17827", "4055", "7913", "24989", "49776", "40227", "70274", "58313", "47791", "19852", "39512", "31077", "61175", "42153", "51802", "45937", "37976", "12854", "71225", "52341", "37666", "3004", "32586", "34259", "3670", "11725", "8282", "52992", "6526", "30438", "24565", "44395", "45010", "496", "51653", "59528", "14003", "27385", "8099", "39425", "36171", "69436", "54357", "15084", "60011", "27369", "52056", "5207", "12384", "24678", "55093", "5525", "65558", "48317", "44602", "44219", "40773", "15357", "8544", "34205", "16741", "13638", "64263", "45310", "3664", "53447", "25613", "8545", "4319", "70727", "40325", "43373", "72802", "34044", "59139", "52313", "64285", "9575", "57781", "28061", "27772", "47409", "755", "70736", "55755", "51926", "11599", "20952", "9893", "53659", "14482", "11053", "61211", "70974", "66075", "28193", "49485", "54849", "63960", "11189", "32725", "2266", "67361", "49978", "554", "65757", "30969", "39015", "19157", "54177", "27292", "58133", "10356", "23708", "28392", "19272", "10068", "51039", "45421", "35096", "29464", "66244", "37811", "8568", "38862", "33212", "37332", "66079", "38357", "12944", "35652", "61754", "36474", "53412", "4768", "61538", "35905", "29682", "32680", "63025", "63144", "56572", "17692", "34970", "61309", "14868", "14558", "46960", "16424", "53032", "62009", "72052", "25899", "43129", "46592", "45462", "4548", "22994", "7809", "35309", "4431", "1773", "12261", "38097", "5237", "3013", "54411", "1088", "55248", "72491", "50873", "31687", "5856", "5328", "53807", "21051", "58758", "21025", "49102", "71536", "57233", "22140", "54954", "49743", "15345", "20235", "12226", "18384", "40086", "15351", "29295", "60645", "48991", "67288", "58100", "28850", "57594", "36009", "63734", "26600", "21513", "61411", "37786", "24169", "34721", "65555", "60053", "59954", "63855", "68332", "68225", "38084", "32364", "27990", "29363", "59660", "7640", "10405", "28085", "4195", "27413", "14071", "19133", "59052", "45318", "66386", "72255", "18607", "20455", "33358", "29644", "46790", "48385", "51673", "64784", "23959", "40191", "52616", "68356", "59817", "15768", "36551", "14146", "33694", "55080", "56221", "13821", "44582", "60669", "9365", "8760", "36819", "6553", "720", "43388", "73242", "37160", "50506", "39504", "39538", "59712", "43491", "8453", "62536", "18101", "13833", "57592", "36952", "8367", "17893", "71740", "30129", "57113", "21020", "72947", "46904", "69073", "33580", "3687", "65337", "7332", "29499", "30357", "66625", "39179", "69704", "2074", "69835", "42942", "71380", "57063", "7025", "39189", "63011", "4599", "33481", "27363", "17504", "50626", "18881", "65377", "47598", "69434", "56668", "65303", "14411", "67707", "1791", "41846", "30631", "61480", "4684", "3185", "12832", "16301", "54571", "57590", "34243", "21699", "35154", "16326", "7732", "9733", "46116", "17139", "14406", "40083", "61819", "37678", "34052", "17161", "67060", "17078", "64097", "64413", "16724", "57001", "12183", "1444", "31480", "14108", "47263", "38359", "10354", "4443", "72420", "55100", "73154", "40526", "56525", "51490", "68090", "36047", "3584", "36275", "38188", "54246", "30221", "18253", "65133", "59399", "48020", "31893", "62466", "72637", "20947", "62261", "5742", "13736", "60233", "70615", "1456", "10830", "28250", "23594", "55066", "39847", "27457", "7594", "35103", "58058", "39073", "55242", "29112", "69122", "48099", "48231", "15078", "33352", "30047", "14805", "60181", "24232", "54982", "62372", "19259", "39143", "73165", "40089", "63196", "28138", "30995", "50652", "36956", "17090", "15560", "57667", "34517", "58702", "26943", "24550", "20044", "66159", "43242", "61850", "35539", "49590", "27565", "46732", "15686", "31270", "49275", "10937", "70788", "55633", "36805", "34736", "34501", "28619", "8865", "52391", "61141", "52014", "4911", "53055", "68272", "23469", "58193", "53577", "50499", "13147", "42432", "58971", "58812", "7163", "67970", "21429", "8951", "53374", "66670", "22243", "20271", "7637", "9524", "6271", "11761", "37215", "4397", "60724", "2068", "6433", "70942", "70009", "51351", "73095", "68548", "54526", "32862", "21788", "61235", "70215", "36211", "47030", "64769", "35974", "50016", "30206", "1398", "49073", "62801", "18310", "16750", "8407", "58046", "13022", "57990", "8343", "63558", "48814", "28651", "11698", "55268", "31539", "27420", "12223", "49594", "18516", "4010", "9586", "62142", "57037", "55276", "13813", "66458", "68486", "49242", "42399", "17808", "60105", "38518", "27378", "61877", "64255", "39945", "52502", "27681", "15947", "16710", "60502", "17759", "35000", "4476", "24492", "51531", "49410", "5973", "41159", "26342", "31684", "27384", "33832", "25438", "56845", "69561", "28009", "35445", "72393", "4961", "11542", "70053", "21012", "335", "1891", "42767", "12279", "62182", "35299", "56651", "56602", "35277", "45417", "37738", "44514", "52194", "72137", "49705", "11877", "39348", "64979", "34440", "53449", "68756", "22500", "12235", "23356", "49661", "21959", "10944", "8046", "11022", "50472", "36111", "3815", "1644", "31198", "11763", "643", "21769", "48942", "44161", "67975", "20382", "46852", "69811", "15767", "51429", "48910", "24349", "59655", "3634", "964", "9723", "37663", "14939", "42593", "41063", "45868", "62911", "6002", "68208", "31847", "66426", "48907", "41164", "17435", "66220", "11018", "27945", "21792", "56608", "4303", "21117", "67121", "51168", "63266", "11974", "9380", "7274", "10071", "1478", "15584", "17392", "8428", "30848", "8109", "69022", "6607", "72073", "49210", "43569", "7510", "58218", "63985", "35381", "15971", "14158", "72784", "13060", "15137", "28553", "18941", "29344", "31114", "38479", "43131", "5835", "22973", "71642", "65781", "27227", "39905", "12108", "30563", "44206", "25226", "49812", "32643", "36178", "58136", "11597", "49830", "11739", "14726", "3237", "59111", "63410", "38183", "30058", "2849", "59656", "48709", "36560", "17126", "15891", "6413", "18334", "28936", "29256", "20854", "55119", "24752", "12830", "68249", "48153", "71024", "1324", "27422", "42626", "32241", "7461", "33952", "63019", "639", "31375", "38551", "10451", "34179", "5013", "33102", "20109", "38773", "32059", "50422", "64372", "30892", "26862", "16726", "56093", "68603", "70946", "42444", "7896", "26029", "37227", "41589", "37734", "41102", "51274", "29978", "19516", "65659", "66810", "39733", "40368", "2115", "29920", "43243", "12147", "58269", "1430", "64442", "4568", "56556", "66785", "42964", "5393", "5752", "17929", "51797", "68826", "22113", "62371", "25015", "39832", "44875", "44620", "23440", "24808", "56910", "33448", "63561", "12822", "47758", "48737", "57521", "66252", "28357", "7573", "13648", "43742", "42508", "63155", "13093", "43273", "24255", "32657", "39460", "72897", "66041", "67338", "70219", "30743", "47452", "25312", "28460", "22481", "63934", "68044", "20144", "64486", "9400", "43484", "45728", "35819", "21489", "37581", "60792", "44006", "32390", "29933", "54774", "1168", "21535", "69629", "3255", "17576", "68622", "8349", "64495", "70886", "38941", "63609", "20763", "3868", "53151", "34032", "59070", "64694", "25575", "38805", "12336", "36243", "56656", "66154", "45307", "43378", "15091", "46490", "11918", "43686", "57792", "6678", "11969", "19748", "9877", "68133", "11976", "4080", "383", "62708", "72972", "41665", "59184", "22184", "41479", "34828", "7307", "22476", "55694", "51063", "69119", "4690", "70304", "71767", "20561", "30820", "24884", "52999", "30090", "56139", "48640", "68771", "33938", "22091", "59161", "52037", "36183", "9120", "30179", "9323", "449", "72717", "49981", "8571", "47259", "21118", "61643", "15341", "35807", "70010", "5928", "49107", "28494", "23407", "59859", "38728", "25835", "1983", "20700", "20949", "36808", "19693", "15619", "47665", "50254", "11346", "41798", "68804", "40443", "24833", "53276", "32525", "72326", "49200", "48589", "42125", "4743", "4566", "73009", "54624", "32697", "25030", "27585", "10644", "34714", "72847", "52382", "55652", "5795", "9305", "47195", "56278", "1781", "29739", "16043", "39117", "10504", "31530", "39613", "21405", "63472", "45632", "18331", "14214", "62964", "58019", "12297", "69419", "22989", "17118", "57371", "34779", "26990", "10464", "72981", "3772", "13121", "36334", "33965", "67544", "35485", "53306", "40664", "4990", "14681", "48039", "21612", "42971", "5578", "56460", "29629", "61765", "36046", "60117", "27845", "35933", "52024", "69292", "65329", "47019", "25706", "7949", "39285", "66111", "39061", "8778", "58970", "42150", "56534", "42742", "57507", "34822", "5812", "7377", "15690", "13958", "72676", "19819", "8410", "54581", "53504", "11885", "23019", "47688", "26568", "39057", "6437", "3603", "13170", "67038", "24542", "72590", "39657", "25423", "1607", "9712", "31930", "44523", "71563", "63992", "51783", "17330", "54538", "19130", "50250", "1442", "38227", "13684", "46879", "12331", "54525", "14631", "55562", "24057", "57502", "3533", "57948", "19363", "2821", "54449", "61621", "60849", "72450", "58848", "4674", "17891", "2930", "42174", "17364", "19189", "15284", "5925", "42738", "51216", "15943", "46055", "35601", "52923", "33254", "26115", "72660", "444", "51154", "56452", "72930", "56563", "46462", "61165", "67844", "8087", "65708", "53572", "51895", "9568", "47645", "62514", "2278", "50339", "33142", "1578", "68714", "61273", "63218", "24104", "31965", "71674", "10702", "24653", "69453", "10180", "65447", "37973", "57265", "7012", "40420", "47865", "5692", "62948", "17186", "57240", "12808", "10855", "19934", "14054", "20600", "9009", "28057", "39770", "44103", "3808", "62296", "63650", "38883", "17544", "45059", "64152", "59758", "69974", "37591", "23699", "54580", "5330", "19027", "33267", "6934", "72290", "53859", "56241", "47312", "40055", "38827", "59855", "42910", "20911", "42612", "22749", "58368", "67616", "71772", "37726", "245", "38949", "57783", "50015", "53812", "48872", "67024", "61871", "11321", "18859", "17473", "23251", "21900", "5146", "29895", "29004", "60868", "54628", "49191", "23759", "52157", "72539", "34129", "20757", "20708", "36572", "58220", "57409", "20762", "49448", "4849", "60886", "2404", "2417", "68642", "36098", "68662", "13188", "14666", "43532", "3931", "56024", "38004", "36556", "69283", "20247", "53800", "27605", "32935", "59104", "42973", "18850", "58997", "39510", "22039", "31117", "46793", "64696", "37167", "10530", "69752", "59741", "11416", "73255", "59453", "73083", "56127", "72152", "45542", "21770", "60129", "12257", "3056", "41865", "19826", "55301", "59646", "3767", "22105", "30569", "458", "27670", "5190", "44810", "36073", "13252", "20286", "4428", "246", "2794", "19146", "49170", "60874", "60136", "41086", "48250", "2357", "68874", "29307", "60758", "6166", "72358", "39113", "1308", "29546", "23132", "58367", "2241", "47912", "1539", "44079", "19971", "51412", "61564", "61921", "64586", "12816", "32718", "5387", "66188", "42420", "64073", "25793", "66117", "29818", "35858", "20991", "39582", "27696", "70675", "52791", "16499", "16539", "50511", "15995", "49261", "33994", "68875", "28757", "8462", "50277", "13125", "36629", "10172", "59584", "62587", "37562", "5961", "22089", "62388", "5791", "23926", "66260", "59661", "15702", "9735", "5458", "1033", "51050", "65034", "11693", "30324", "28129", "37816", "54163", "1861", "7349", "43495", "41139", "20641", "14134", "21932", "45428", "13579", "34437", "47869", "41120", "840", "23496", "52979", "60122", "31196", "28131", "20656", "45519", "43713", "63970", "55973", "27890", "62998", "72462", "38878", "41847", "71227", "4777", "27367", "63591", "13410", "41855", "24449", "68526", "16878", "11563", "64707", "8426", "40494", "63325", "61060", "37658", "38791", "35513", "61936", "18605", "23386", "9569", "47372", "38867", "631", "58144", "1103", "36191", "26619", "395", "34158", "47053", "46566", "61321", "63036", "36960", "4704", "70209", "26216", "40880", "46176", "72102", "11404", "23058", "41543", "11529", "55860", "812", "66991", "32708", "37949", "41840", "26304", "29538", "70148", "18474", "66692", "12995", "41455", "41373", "56209", "4447", "23167", "59991", "19435", "12543", "14177", "43851", "71578", "10391", "50065", "21082", "39758", "38289", "50697", "8295", "61519", "63878", "34482", "44903", "17828", "40222", "35892", "15037", "17901", "13389", "39625", "29723", "32637", "66368", "26105", "48715", "71513", "43786", "55164", "25784", "16265", "5840", "42498", "70431", "34683", "58418", "36549", "42154", "34363", "72610", "60263", "8011", "60744", "5912", "72143", "20706", "37121", "57904", "70577", "49083", "67940", "61115", "58476", "7847", "68718", "20213", "55827", "53657", "19290", "26643", "33311", "801", "15306", "51135", "28210", "53708", "41112", "10536", "7102", "66745", "35800", "1918", "42737", "15940", "72623", "68201", "17833", "15633", "13259", "58738", "62269", "9007", "69250", "12975", "46854", "70417", "14154", "39938", "62349", "22652", "23387", "46726", "55942", "56010", "59721", "58227", "21388", "19430", "63870", "28502", "57433", "16171", "62986", "59290", "32992", "52350", "12254", "48612", "29037", "59679", "38687", "56609", "47365", "4464", "34556", "29096", "11161", "4418", "32731", "58124", "68939", "24081", "73253", "49469", "24357", "28137", "46807", "1754", "32088", "17403", "55568", "49839", "43462", "24343", "49544", "67183", "60373", "8797", "42249", "10506", "889", "50723", "65586", "18937", "64462", "67502", "72807", "20148", "69589", "37524", "58277", "33986", "35033", "1029", "39734", "37567", "47637", "30173", "5563", "32834", "28218", "45296", "36921", "26984", "20484", "17879", "36034", "46561", "40953", "2901", "45606", "57509", "18692", "20497", "46985", "46257", "28019", "45089", "34836", "420", "42230", "43079", "49258", "56396", "21954", "42305", "13411", "57071", "57175", "70901", "17951", "71010", "12793", "35582", "66734", "53600", "6789", "19813", "10101", "38127", "57757", "2571", "16317", "8626", "37322", "53154", "18292", "5990", "7489", "55588", "28094", "51999", "72547", "65099", "48535", "67925", "46455", "25870", "48", "72470", "55244", "50494", "25752", "26374", "32225", "1031", "10264", "2838", "684", "3417", "64404", "15253", "26380", "63171", "51158", "43250", "34089", "20351", "18187", "4525", "62836", "37397", "14608", "43152", "49134", "14399", "38221", "23149", "64502", "70873", "29075", "55870", "40103", "18109", "54102", "40819", "62118", "61528", "36030", "45992", "27795", "31404", "1761", "52817", "43036", "65893", "37111", "72609", "32724", "2598", "7907", "26399", "1098", "1859", "47105", "71898", "8933", "69908", "61006", "32030", "14430", "52240", "64188", "32844", "6792", "31115", "25747", "55133", "47494", "10614", "58790", "13800", "52324", "52101", "34291", "35934", "9262", "18590", "40463", "61776", "10976", "32507", "9898", "4970", "32141", "15648", "66675", "43260", "23323", "27593", "69992", "53738", "27031", "66141", "46033", "15811", "20390", "32105", "58183", "49551", "43447", "40480", "18293", "661", "41912", "16855", "6984", "50614", "36799", "14256", "11305", "8686", "877", "24984", "23348", "3908", "39275", "19885", "44177", "47712", "29921", "51981", "34583", "13928", "33527", "26590", "18057", "49863", "26016", "39148", "61587", "63749", "7431", "345", "45285", "8152", "62675", "29450", "26622", "70184", "58223", "37250", "47836", "64330", "40142", "59531", "42026", "44594", "13434", "29152", "53201", "42821", "26485", "29169", "16529", "67005", "28536", "28000", "20666", "46249", "35066", "71057", "54806", "59925", "45021", "70957", "61032", "43239", "64640", "69455", "12478", "27202", "1587", "21476", "883", "27045", "51899", "1014", "53670", "69897", "38474", "55302", "19867", "73053", "35994", "68556", "55386", "33622", "72342", "31350", "50349", "64377", "1629", "39588", "28310", "29308", "7807", "58531", "20469", "31394", "40275", "2026", "67174", "10098", "62424", "41078", "58213", "25871", "55288", "17689", "24399", "60967", "72856", "19407", "60840", "38587", "23553", "4012", "56698", "40214", "34231", "35757", "70396", "60732", "13694", "70462", "47681", "9399", "39749", "34312", "195", "37007", "58344", "27407", "60456", "43846", "63557", "9637", "25818", "42083", "30843", "20224", "25037", "62721", "48050", "62434", "27486", "70878", "2088", "43161", "49155", "47620", "72232", "24894", "46277", "60798", "70683", "22050", "5739", "19904", "21171", "26707", "41209", "72914", "21850", "56421", "62575", "68782", "64763", "17093", "29745", "36953", "429", "23126", "57301", "47703", "52055", "62228", "7666", "71026", "34542", "40616", "29201", "56591", "20276", "20400", "33906", "69816", "10386", "45143", "42451", "10794", "45308", "5535", "26283", "34800", "37481", "22032", "25161", "11784", "12116", "71066", "35382", "7927", "66118", "61276", "6990", "43710", "13970", "24315", "71731", "15608", "69445", "31845", "34277", "46672", "48766", "13918", "3174", "59342", "42874", "55922", "2481", "2459", "25436", "18638", "68781", "42237", "33437", "67790", "44714", "47217", "60817", "31815", "17039", "12461", "56039", "64", "69072", "15243", "56744", "57302", "19555", "26362", "56016", "22962", "34264", "60547", "61876", "29595", "19720", "4569", "56830", "49233", "12504", "55698", "27944", "59087", "13492", "65828", "55758", "31979", "16971", "39688", "65254", "47462", "15164", "12982", "9776", "59378", "61563", "55745", "47870", "24763", "46831", "70479", "14754", "4175", "2935", "27755", "23912", "34460", "45292", "9836", "26672", "8749", "8287", "58066", "24995", "15391", "5560", "68200", "57348", "48329", "73153", "5544", "62927", "56596", "44740", "15840", "53729", "15673", "16822", "58540", "2343", "53188", "66540", "3100", "14719", "55768", "69724", "72033", "30194", "5017", "39449", "7213", "49998", "59633", "51263", "21904", "49521", "12775", "42403", "27912", "73010", "39587", "46925", "64568", "24593", "40780", "33538", "48454", "19110", "2734", "69728", "52312", "64100", "71856", "70567", "5137", "61249", "24882", "39361", "33404", "37533", "33744", "23419", "71192", "51306", "12780", "25113", "41880", "70511", "16076", "6344", "21906", "34048", "67833", "33972", "29407", "47985", "37043", "5854", "58359", "22881", "43217", "25120", "41489", "37835", "11923", "36683", "9270", "44800", "57864", "10403", "17537", "37234", "47336", "3075", "59609", "1535", "53689", "67560", "16946", "1406", "59578", "17158", "44435", "29080", "56994", "2193", "45253", "56248", "8663", "25510", "69492", "51986", "56228", "43505", "25173", "71108", "5275", "30773", "25938", "70900", "64379", "29643", "45484", "15590", "7125", "62070", "14020", "62577", "3658", "10393", "59178", "29106", "60584", "34399", "30273", "5227", "42879", "49446", "8279", "2441", "36167", "66550", "4250", "36942", "4314", "19974", "9450", "2020", "17698", "52792", "51084", "48553", "24022", "52645", "60990", "22909", "43253", "25662", "53341", "39069", "66477", "11945", "52563", "47592", "4586", "49912", "17402", "51373", "45355", "56680", "46", "33308", "22600", "58571", "62242", "25205", "71443", "16922", "63640", "47094", "4915", "13497", "20128", "49375", "2729", "71848", "70742", "29980", "7970", "39129", "60503", "40113", "62868", "17859", "6501", "19162", "70507", "40091", "72714", "16250", "14586", "70761", "42990", "1733", "32511", "73002", "25729", "3520", "4009", "65373", "42105", "10129", "16819", "62793", "8134", "62126", "46559", "36272", "4081", "32070", "30794", "40225", "11205", "57949", "58109", "31525", "15670", "23628", "61998", "5347", "70622", "63907", "32720", "28455", "13086", "26331", "49076", "29370", "59219", "5644", "24999", "69344", "50501", "44626", "68203", "44565", "68602", "52172", "8361", "44225", "4044", "52013", "65422", "54417", "67033", "52159", "26247", "54027", "46965", "50294", "16402", "12544", "27437", "61483", "12456", "67491", "66300", "63216", "27750", "4246", "35245", "58469", "35552", "2818", "36215", "26261", "63231", "53397", "28589", "13473", "68694", "50839", "28765", "2373", "57897", "14619", "69183", "47033", "22079", "6143", "71336", "37637", "39080", "64977", "52068", "66917", "67800", "41867", "33200", "38109", "59330", "36448", "12233", "30152", "24118", "65090", "15145", "42374", "65361", "30512", "6108", "7770", "38270", "22660", "41109", "44691", "17426", "61461", "22053", "44713", "25918", "15070", "72164", "18709", "64779", "23209", "60111", "46183", "54251", "25648", "21497", "50623", "713", "21764", "18402", "54786", "40690", "8061", "63434", "9696", "36045", "63154", "40632", "55691", "31554", "43199", "45921", "64257", "34417", "60423", "34464", "25883", "49489", "9908", "51865", "43606", "44196", "24362", "38620", "58645", "9603", "6375", "64615", "12088", "9983", "31221", "50019", "13730", "68214", "44596", "47181", "28271", "5001", "60050", "38593", "65692", "24090", "33145", "69732", "23686", "7548", "41386", "64455", "59163", "64751", "61843", "51934", "63523", "11008", "4446", "13550", "58114", "9729", "69068", "25249", "41267", "23620", "70598", "35843", "15443", "56755", "28283", "50914", "881", "20811", "38410", "23586", "70266", "67576", "63813", "15377", "8588", "17858", "55029", "33886", "966", "8554", "55260", "13836", "39329", "25352", "49329", "11387", "54157", "48521", "26839", "10841", "25603", "58830", "36781", "46432", "16013", "49549", "34122", "72659", "37902", "71991", "58374", "29362", "2918", "70521", "68024", "360", "45943", "69485", "30699", "1258", "32132", "12834", "65153", "13484", "51848", "2482", "28328", "17878", "11913", "25324", "61691", "16752", "50282", "34185", "70505", "22972", "53680", "59011", "57328", "10956", "2229", "37909", "52683", "72468", "65117", "15223", "45278", "60298", "42430", "35322", "65826", "41891", "188", "60747", "67966", "37802", "8479", "51396", "46411", "31084", "9533", "59131", "49610", "24835", "52882", "19223", "33029", "64119", "34082", "48866", "3243", "57658", "7430", "38769", "22626", "70862", "56511", "66163", "63476", "29331", "55142", "45471", "30802", "25757", "61439", "51555", "31942", "20623", "2056", "32836", "41322", "3583", "59922", "2452", "54518", "46898", "13998", "36832", "45896", "55298", "529", "28258", "53686", "66843", "31552", "30714", "37760", "14650", "70425", "23849", "34768", "46787", "441", "46409", "33187", "6784", "72546", "11047", "30266", "60983", "38896", "4283", "65118", "14090", "31088", "69698", "28412", "14502", "50069", "59479", "32081", "63333", "59086", "70630", "55638", "45799", "68384", "15287", "43875", "52099", "6125", "49656", "15174", "9550", "55666", "36438", "67929", "50439", "67539", "58511", "58140", "65513", "7708", "59159", "51990", "67986", "67382", "31918", "40890", "5015", "19542", "70771", "15654", "3985", "30074", "62202", "4757", "50826", "68366", "16592", "48497", "51730", "66504", "29671", "54706", "10908", "42955", "1084", "19552", "54601", "73204", "53971", "39104", "3958", "1667", "68878", "18189", "63723", "22711", "4894", "61534", "71247", "56187", "10589", "71726", "19145", "66672", "45062", "11327", "25812", "69658", "65539", "15612", "47055", "68918", "6853", "41297", "50316", "51968", "17896", "22413", "31291", "31638", "37978", "56130", "7331", "45220", "2466", "21574", "53599", "43162", "64754", "37673", "10006", "5252", "64301", "66353", "67660", "10760", "8330", "65323", "58288", "33121", "33373", "67014", "41784", "68137", "41124", "27743", "23948", "65562", "31613", "8892", "34258", "52267", "13015", "26647", "29382", "42381", "48549", "8914", "51527", "62314", "38246", "29166", "34675", "15848", "29068", "32638", "45778", "21596", "27746", "66295", "13004", "57697", "17933", "40312", "40256", "27166", "30899", "33468", "42031", "36447", "15191", "22544", "73162", "69284", "6576", "35696", "24524", "53714", "63516", "23746", "20766", "39899", "62561", "34491", "28874", "21577", "6572", "9142", "22866", "51402", "38717", "55760", "31966", "54620", "21694", "58240", "7868", "34805", "12201", "72744", "18657", "21859", "38002", "63695", "21369", "59880", "6918", "63136", "50699", "61383", "16712", "15107", "21270", "28825", "21861", "44149", "1894", "54012", "14223", "60130", "17530", "60524", "16467", "56509", "61640", "10810", "24860", "293", "33414", "48618", "31871", "47943", "31495", "69282", "42392", "13973", "21665", "43069", "26453", "20130", "14196", "72035", "15933", "26719", "54127", "16488", "33920", "43880", "53994", "17253", "43909", "44340", "26322", "58327", "8143", "24798", "13099", "4400", "16394", "41563", "9024", "65872", "5080", "64395", "70161", "34578", "5240", "13498", "56419", "60890", "23287", "29819", "13713", "4207", "53989", "28458", "72884", "21389", "48180", "961", "20698", "20444", "5016", "53123", "61125", "40697", "41336", "55017", "38947", "47826", "41914", "35494", "34450", "20931", "21196", "39122", "1394", "279", "47399", "19599", "1341", "35847", "58126", "47340", "18154", "23491", "9926", "3482", "31053", "68467", "15309", "40419", "45604", "71014", "57057", "19368", "27783", "40355", "5930", "14721", "26640", "54191", "49415", "16938", "56616", "25127", "31629", "45088", "24786", "16075", "16526", "49647", "61263", "55634", "25792", "58981", "48828", "909", "32080", "32180", "38913", "42996", "29373", "38389", "71587", "53158", "20898", "63298", "65764", "48310", "54950", "53036", "35851", "50039", "2217", "44422", "4294", "30851", "41255", "62546", "50427", "48242", "66331", "44665", "39206", "19115", "14299", "68922", "40476", "3084", "5302", "12022", "48467", "49172", "59723", "66684", "37431", "1820", "25025", "8831", "53353", "62601", "27791", "39083", "46553", "60166", "47784", "16105", "16336", "38745", "30809", "33542", "25199", "24858", "62321", "30526", "33388", "35353", "66847", "55490", "62420", "62392", "29", "55547", "24395", "6063", "30755", "18835", "43635", "62034", "36925", "70858", "44496", "57174", "68447", "20189", "33931", "32774", "36290", "22330", "65103", "35891", "72331", "25134", "45623", "61398", "65884", "44338", "34357", "72921", "39523", "52383", "2052", "9197", "47231", "21902", "72882", "19727", "54293", "34971", "13755", "8439", "21633", "18785", "50272", "27127", "23924", "19644", "5884", "19769", "51652", "29565", "62778", "41529", "18043", "48037", "4715", "9739", "47693", "42791", "10070", "72963", "4040", "31666", "25589", "45440", "47854", "28890", "34908", "2592", "34033", "33756", "69884", "53145", "39949", "9334", "28416", "10110", "59171", "45863", "58823", "71845", "63804", "45794", "27572", "27502", "41477", "4090", "66824", "21000", "7266", "62259", "5814", "9734", "50252", "33996", "19087", "15663", "43179", "31749", "55740", "59015", "65526", "47212", "18572", "21894", "47449", "21017", "8954", "38820", "5303", "5386", "64167", "30865", "26130", "33656", "72843", "71953", "51765", "21723", "20321", "45431", "10606", "13139", "68757", "12940", "23824", "52149", "47132", "2401", "27549", "14417", "5572", "16916", "34398", "9833", "57554", "26570", "71595", "28854", "10228", "13784", "69696", "30793", "39051", "36269", "12968", "9888", "25415", "18025", "3460", "50725", "51064", "50479", "71901", "30849", "36238", "56807", "63630", "10696", "23643", "42965", "20442", "32558", "7024", "30885", "51755", "38894", "36699", "9077", "68524", "15268", "9543", "42835", "57867", "67820", "53264", "19188", "44865", "58001", "33673", "23521", "68293", "55462", "34780", "36008", "72028", "7686", "66416", "31941", "37054", "46325", "59982", "57957", "48346", "42678", "39155", "12926", "29269", "14538", "1863", "58829", "19113", "56927", "22055", "31138", "34953", "23929", "65080", "63391", "68006", "49897", "29070", "31874", "40220", "5184", "2471", "339", "40954", "45599", "48275", "1588", "27418", "13534", "25221", "33146", "33572", "38235", "31526", "11128", "33357", "70838", "10280", "32115", "28339", "72307", "46355", "69909", "22009", "11251", "59772", "61512", "16615", "819", "21650", "24031", "49650", "64931", "36337", "48629", "44", "20775", "35095", "71404", "38493", "19257", "23279", "9562", "64536", "10622", "3177", "7460", "31710", "39789", "45502", "48125", "42570", "44694", "32307", "55026", "24351", "52905", "19580", "67141", "13548", "69162", "20030", "8844", "16052", "53819", "68614", "31282", "14606", "60605", "73129", "64449", "49935", "674", "48554", "38206", "1423", "2053", "45044", "66359", "69722", "5288", "37892", "60152", "65509", "38266", "40052", "50557", "25197", "51333", "148", "16273", "44631", "28304", "23579", "13617", "36234", "3133", "11654", "20301", "37879", "64493", "63002", "14796", "31725", "46877", "28929", "51563", "12012", "32198", "50108", "3861", "43508", "15459", "14087", "6371", "6958", "13697", "49861", "45433", "34236", "14012", "48435", "19913", "64368", "12963", "45233", "56567", "56941", "15067", "71863", "65961", "19551", "21426", "12833", "52083", "25432", "46266", "35575", "39760", "33598", "21146", "69613", "64765", "43286", "70716", "18748", "16341", "2213", "10072", "43168", "61000", "32013", "32907", "31910", "19497", "42704", "8307", "27690", "12586", "3971", "55704", "56131", "17882", "1980", "18329", "35226", "63547", "70731", "27556", "40448", "44428", "12910", "53394", "60828", "37377", "61253", "66446", "18567", "62490", "37123", "28493", "38812", "34606", "38507", "28720", "27655", "35822", "17502", "17385", "32107", "73158", "26804", "58300", "39785", "319", "51329", "61353", "50377", "2788", "14948", "13927", "41176", "46760", "53355", "37220", "33261", "7032", "12890", "50104", "53968", "49050", "13450", "4827", "35939", "66440", "13263", "48438", "1173", "36658", "70401", "11229", "45283", "10997", "1669", "72978", "71207", "27301", "62006", "68453", "37551", "45065", "49192", "46239", "58497", "3318", "4972", "42935", "2222", "67464", "47127", "68315", "19121", "8275", "10577", "11831", "19758", "61634", "5108", "15258", "9595", "41701", "38199", "12620", "68836", "9025", "13023", "42759", "70059", "65522", "8890", "21381", "56300", "1622", "63198", "68101", "8126", "1536", "72218", "16358", "16523", "57006", "36761", "5043", "3039", "57430", "61780", "21671", "37794", "64331", "54534", "35590", "25325", "58155", "10444", "23928", "66915", "2659", "13747", "18387", "72418", "27315", "20379", "59339", "17851", "57491", "7242", "24695", "2290", "24647", "20965", "14265", "6779", "71733", "66089", "58836", "54551", "23099", "19784", "53734", "11", "549", "50718", "37395", "67824", "7339", "2682", "3946", "53410", "14261", "31183", "66921", "61093", "48003", "48085", "20599", "29431", "49390", "30657", "13272", "43147", "9610", "47317", "47390", "2408", "14190", "12554", "9825", "27343", "43240", "49766", "55236", "23973", "25152", "44635", "13423", "66680", "31230", "58800", "30124", "43211", "68897", "66735", "65113", "10378", "33452", "67955", "48761", "5969", "29323", "62331", "43364", "66209", "28729", "11665", "43649", "15611", "66568", "47506", "61441", "30283", "59319", "175", "33221", "23719", "13955", "45448", "37027", "15964", "27257", "36382", "2349", "50820", "36052", "55749", "23029", "33262", "49902", "47888", "63422", "62287", "28190", "70640", "42985", "33925", "61788", "21879", "15088", "41381", "11381", "52938", "27606", "397", "2457", "57453", "22598", "25452", "58693", "28168", "33535", "65947", "14011", "20781", "17784", "65000", "11299", "69678", "65862", "231", "1011", "68648", "16267", "47453", "21253", "8557", "2936", "31107", "53596", "69234", "53041", "20580", "62238", "34014", "45697", "52034", "37208", "64367", "29696", "7933", "2446", "40687", "44780", "28637", "11465", "60631", "52145", "2980", "46181", "19215", "31771", "35512", "52548", "56410", "27423", "23617", "18090", "36814", "53696", "16175", "61150", "67711", "33556", "58096", "59716", "34306", "26923", "43350", "65434", "35453", "23698", "47684", "49639", "71003", "67895", "13539", "349", "27594", "34818", "41615", "30372", "19348", "6649", "68916", "35616", "49441", "9032", "22584", "27006", "37710", "61331", "40682", "59647", "71842", "48741", "47558", "3390", "61003", "9078", "2709", "7777", "69362", "43464", "21709", "3095", "5257", "9396", "10466", "19195", "14110", "5383", "1380", "28421", "66050", "19946", "32117", "24338", "53546", "6817", "50617", "24573", "56124", "24771", "23900", "9581", "32229", "54092", "52122", "6983", "25770", "23374", "39629", "44449", "19604", "18421", "51902", "30319", "60364", "18106", "21392", "33730", "32493", "5104", "70021", "68250", "6386", "18027", "42089", "55897", "33421", "52440", "44932", "2669", "39024", "55337", "65049", "43285", "62315", "19918", "283", "1018", "32722", "5926", "21614", "21516", "61533", "1222", "16739", "43695", "28823", "12428", "27940", "70167", "65229", "61581", "27217", "61234", "10852", "64697", "72271", "27999", "65400", "34267", "60937", "4459", "30249", "42188", "20055", "48536", "43739", "11511", "21842", "50793", "48040", "35133", "47560", "14801", "28744", "53071", "9250", "39342", "45000", "37183", "7464", "6751", "68719", "1338", "31542", "65937", "30976", "35403", "62316", "23883", "32295", "63007", "57486", "15564", "6380", "67546", "32728", "500", "33430", "58200", "41214", "45453", "47396", "29906", "54291", "10493", "3254", "3263", "60319", "45293", "51060", "72344", "20570", "33945", "44766", "35965", "9680", "28169", "69883", "6595", "5438", "73196", "45818", "33402", "48357", "21384", "55073", "18407", "44910", "16453", "17369", "66910", "48016", "42545", "52602", "41604", "15634", "24455", "14739", "18804", "13386", "24879", "47807", "14501", "35853", "64171", "47327", "64891", "1509", "29734", "24572", "11949", "59625", "19213", "9538", "68005", "59003", "63809", "46044", "52711", "32603", "54610", "69440", "34812", "11071", "13054", "18770", "20354", "33530", "18751", "72244", "15842", "21390", "14694", "10118", "63081", "59089", "17637", "40574", "13756", "64179", "14958", "7800", "17641", "9853", "35186", "9048", "38164", "36184", "30384", "66356", "30043", "17580", "50705", "68624", "34626", "14688", "28819", "23170", "3653", "57150", "42691", "34776", "12293", "25439", "10646", "5547", "33682", "24421", "65508", "52801", "68470", "34123", "71244", "22517", "49302", "16944", "18869", "22147", "20860", "53046", "17386", "28235", "51024", "70080", "23809", "51743", "20450", "40270", "45382", "71271", "64759", "60964", "2403", "17834", "32028", "55472", "40351", "20859", "32148", "7132", "10661", "19788", "4267", "13257", "37568", "67934", "68365", "29807", "42243", "33702", "25833", "26410", "68882", "49916", "68426", "39442", "60385", "40386", "59702", "22220", "33138", "52291", "47134", "27782", "13488", "22516", "2156", "29392", "9387", "4850", "60517", "51907", "4029", "7268", "34459", "48667", "67", "12305", "67946", "23886", "9767", "25189", "49300", "42472", "69744", "32336", "39922", "28188", "46855", "60037", "39862", "1497", "50445", "9282", "34321", "30116", "10032", "68319", "14918", "52680", "18138", "54422", "53035", "29657", "9634", "73195", "42435", "31032", "17177", "42909", "68194", "36777", "50147", "13827", "73251", "39085", "48479", "69679", "50322", "48312", "3236", "47034", "21827", "40400", "21093", "28828", "8113", "37945", "16477", "73135", "48308", "70254", "66296", "60504", "26019", "50380", "63603", "20968", "59748", "3355", "67885", "26073", "66966", "47347", "36686", "11104", "39270", "30118", "56546", "67272", "58285", "46031", "13859", "29283", "2209", "33576", "13027", "65115", "64065", "15413", "30750", "52642", "47765", "68275", "54262", "1307", "6713", "6588", "12069", "28500", "14949", "70230", "64459", "1424", "11591", "3033", "7235", "40756", "54129", "12824", "3849", "38309", "56827", "71879", "20066", "47896", "6476", "31288", "20722", "32769", "26792", "6460", "53509", "64112", "50572", "16745", "16581", "22359", "19285", "32495", "22097", "40988", "52160", "14788", "48179", "51090", "45562", "5941", "66031", "3988", "11259", "12638", "70376", "27026", "67461", "872", "41137", "16042", "13771", "7054", "66920", "72289", "53838", "52165", "58729", "39591", "19331", "44311", "13296", "22884", "22354", "56963", "56870", "63001", "9783", "62769", "62488", "18952", "9326", "31939", "69659", "18026", "73023", "63225", "92", "13658", "13654", "25842", "2523", "43463", "17300", "66062", "34171", "36144", "2090", "44271", "68525", "49706", "17572", "28112", "5025", "54434", "8838", "68197", "7184", "59081", "3278", "34388", "47046", "46723", "14646", "7386", "11933", "53377", "72600", "7328", "1896", "43196", "73149", "37642", "16849", "70564", "22724", "46636", "45255", "40577", "16969", "59080", "59225", "72352", "25101", "25469", "17820", "64424", "18795", "41783", "47085", "6873", "69556", "12998", "55082", "61076", "10917", "61197", "39007", "63929", "43765", "60997", "14693", "37602", "47988", "51096", "37483", "1837", "9126", "55410", "55324", "29701", "51324", "72236", "28851", "46159", "52855", "20704", "53668", "18185", "8698", "68649", "69244", "25563", "27204", "72602", "12350", "46863", "30346", "61792", "41984", "66517", "20731", "3122", "52935", "68246", "30088", "57824", "28344", "24689", "29164", "3810", "69216", "15704", "16330", "6142", "25866", "6429", "55447", "12792", "33721", "41240", "9038", "35495", "70457", "50459", "62834", "43121", "52196", "28886", "15692", "11235", "72645", "40548", "17354", "39208", "59574", "27236", "66010", "36828", "32391", "32555", "28089", "29581", "29415", "18845", "14175", "21209", "61024", "9210", "14066", "13286", "29761", "15510", "67287", "43705", "14618", "34340", "46260", "6109", "342", "4906", "26462", "38599", "72631", "60428", "20895", "55992", "52524", "18856", "12144", "47731", "60863", "39218", "41607", "49086", "63904", "47945", "10283", "65339", "10445", "34618", "17194", "43189", "14633", "25707", "52776", "51037", "38299", "25236", "11375", "28205", "28840", "52811", "59820", "18113", "28780", "53741", "39617", "55705", "70496", "31493", "30637", "25367", "64917", "55541", "53336", "24580", "69987", "41494", "21834", "40491", "10327", "45696", "8382", "65046", "14097", "31345", "1917", "38890", "10151", "66105", "50094", "70126", "19550", "26041", "39686", "24067", "67765", "62933", "22467", "31410", "40809", "39531", "5234", "36943", "11917", "33933", "65386", "5319", "51508", "57547", "31744", "38757", "35975", "21391", "66000", "4162", "8085", "10051", "28879", "59286", "31896", "38130", "3702", "64689", "36429", "20989", "10008", "2646", "50394", "42740", "51094", "17872", "63894", "26671", "47142", "19355", "73178", "21237", "14578", "30544", "20466", "40860", "19317", "37188", "64329", "59947", "61824", "70794", "66390", "49577", "37891", "67528", "62077", "71716", "27878", "58141", "56557", "56011", "46362", "14966", "4863", "6447", "16038", "25880", "14471", "7462", "40306", "25365", "28505", "6252", "46828", "16914", "63993", "53548", "41812", "8517", "48645", "58950", "40145", "35273", "61458", "28564", "9971", "31674", "49483", "57994", "18337", "12115", "22406", "65433", "57911", "63314", "61858", "900", "65546", "38960", "11288", "72417", "9885", "54854", "9537", "66844", "21814", "3892", "65397", "8773", "24907", "15313", "58370", "29668", "18695", "30165", "42557", "70868", "44816", "20041", "896", "57069", "32256", "19504", "43879", "4656", "9316", "4760", "59537", "64302", "54415", "4805", "70147", "42727", "17180", "55635", "60305", "52595", "66186", "38715", "57755", "15440", "36227", "56645", "5495", "72068", "40877", "51138", "39188", "57085", "45743", "53469", "11893", "44754", "28642", "31694", "70686", "10331", "69864", "71346", "31973", "50270", "299", "66469", "11149", "69317", "35828", "20571", "7917", "17616", "34867", "34230", "38813", "39301", "18581", "21759", "36927", "36498", "48456", "33141", "72700", "35669", "60029", "4043", "22746", "65517", "45783", "49007", "35642", "59727", "50401", "1004", "678", "2383", "55291", "895", "66321", "62117", "65486", "21744", "71303", "17155", "19343", "63033", "36134", "43565", "44448", "55775", "60654", "11582", "42624", "19380", "24144", "48528", "46957", "28572", "61721", "67843", "33069", "57998", "17814", "17782", "60540", "45585", "37826", "35396", "44799", "64035", "13235", "32302", "67996", "54824", "45012", "18889", "6288", "58586", "53101", "2487", "44668", "4544", "20719", "57907", "24759", "17380", "70235", "24269", "17497", "67605", "17075", "7084", "4383", "50910", "34283", "45707", "37204", "44043", "45185", "26103", "12256", "9705", "72681", "19173", "67770", "40891", "71246", "48809", "58648", "20196", "66947", "38378", "33889", "12654", "11334", "25504", "14268", "64814", "7189", "62648", "46485", "51662", "60677", "71162", "14084", "22469", "15311", "54087", "62504", "58618", "34842", "6749", "26339", "66429", "65467", "1246", "53865", "59021", "14632", "63623", "56749", "49952", "49855", "23645", "4550", "35253", "9496", "52588", "21335", "68691", "21112", "61605", "67124", "18236", "27937", "58704", "27297", "43560", "44832", "42617", "61435", "35448", "8637", "58261", "31782", "4311", "6512", "4352", "27313", "13219", "60717", "20202", "44455", "55799", "24482", "40038", "49408", "20278", "42257", "37434", "16063", "7792", "12774", "15149", "6155", "37487", "33791", "60197", "27635", "72404", "63538", "35563", "46232", "55061", "65299", "24790", "60158", "69761", "3928", "46739", "3589", "30459", "41950", "49335", "25675", "66455", "21004", "58088", "34534", "6278", "57776", "14427", "9368", "33690", "64585", "10767", "36578", "14007", "24672", "38181", "4537", "16468", "8723", "10244", "24502", "56435", "6096", "15977", "53340", "9890", "61802", "42347", "15858", "59654", "47392", "68929", "70552", "54857", "3146", "11007", "38930", "67609", "23734", "28809", "4698", "40128", "16324", "28843", "72323", "52839", "12150", "13840", "32878", "60873", "8273", "2256", "21044", "2551", "22292", "21635", "46478", "70911", "7022", "54843", "24358", "6877", "20071", "7655", "68705", "59950", "53369", "70504", "40864", "23196", "51478", "45048", "975", "14283", "32753", "19450", "2246", "66388", "15452", "21255", "29257", "65319", "16758", "6504", "67011", "41684", "19607", "52256", "21125", "2245", "37967", "58212", "55010", "32053", "61180", "23109", "34180", "5048", "32514", "8922", "1599", "4233", "50449", "37772", "20545", "23048", "14653", "35198", "62797", "30856", "63831", "11929", "57221", "8858", "734", "32339", "63281", "73166", "56583", "53973", "24663", "14605", "34894", "5126", "39471", "20432", "35166", "31983", "46526", "60297", "17057", "63617", "72953", "57690", "18456", "63703", "51339", "52958", "47236", "21239", "45061", "49718", "11417", "2133", "73114", "5398", "38753", "48111", "5127", "43602", "1898", "49398", "60831", "32805", "26396", "19226", "45771", "57714", "2641", "9726", "35295", "43629", "43488", "31360", "67857", "52011", "38102", "52561", "70572", "51806", "71956", "30272", "22615", "30492", "6203", "40473", "73122", "55028", "24815", "63997", "348", "68914", "57805", "1364", "30265", "52778", "11102", "44526", "53518", "35681", "28152", "49991", "12385", "8441", "70293", "28433", "33434", "57311", "732", "69571", "710", "14468", "9407", "65264", "37997", "16496", "35576", "34550", "56582", "17243", "31131", "1449", "43480", "15689", "16894", "14238", "41190", "5082", "30785", "27256", "31626", "41609", "15362", "19315", "60644", "11294", "2139", "28452", "14912", "72201", "57530", "65423", "45583", "11871", "43316", "27235", "40427", "25709", "49217", "4215", "45824", "23441", "21415", "67109", "21344", "40592", "48616", "27959", "57754", "57235", "58339", "63062", "45715", "62581", "71157", "40861", "53847", "56077", "51074", "60438", "1271", "60674", "51114", "45795", "69131", "38521", "7050", "7316", "56351", "31125", "25058", "72440", "44272", "59035", "24912", "32804", "24768", "1105", "51086", "57379", "33984", "11593", "55203", "36795", "42620", "13789", "27056", "1015", "45174", "49035", "8717", "57257", "3026", "16834", "73103", "68323", "47540", "17200", "43542", "26721", "33904", "1811", "24736", "58670", "7848", "65362", "15939", "15308", "30815", "17442", "33408", "23873", "57268", "71479", "54966", "10740", "537", "21702", "19559", "35787", "64134", "53636", "40608", "37566", "56198", "70885", "35952", "33513", "44685", "8465", "20498", "9982", "60510", "18779", "72601", "2233", "8102", "59181", "3121", "47607", "65006", "19100", "21966", "39890", "70805", "66182", "31898", "30922", "33661", "40132", "60399", "33766", "62058", "10932", "42974", "70463", "52685", "29733", "53893", "35178", "55815", "68437", "60283", "59703", "21995", "445", "34055", "243", "64590", "66461", "22252", "10559", "18424", "68678", "513", "21580", "65561", "57107", "33548", "42854", "7135", "51603", "36386", "43653", "68507", "57271", "14800", "70970", "53811", "1479", "3922", "71460", "13044", "21851", "56291", "32536", "45592", "47605", "49677", "66759", "70850", "11239", "29294", "30004", "55712", "44535", "36493", "70635", "54095", "42900", "34650", "57434", "14353", "72876", "52302", "28213", "34612", "42391", "61646", "18632", "34797", "18794", "3123", "39537", "60657", "40114", "3636", "17444", "11910", "46140", "13580", "8133", "19033", "25628", "59213", "18389", "35634", "46606", "37124", "35323", "67013", "33600", "50319", "51363", "43374", "66965", "43317", "3280", "70348", "61929", "24800", "9552", "72292", "9263", "49847", "50599", "22108", "58009", "2344", "29444", "37210", "451", "35857", "12055", "17320", "18302", "12249", "8945", "47115", "63918", "17423", "36307", "6246", "54690", "2470", "46286", "67864", "2204", "26197", "364", "20642", "53923", "40457", "27888", "67201", "29750", "65797", "34684", "775", "70936", "20636", "10831", "40594", "16967", "67979", "65929", "5588", "14804", "11397", "6716", "59495", "41921", "22300", "32376", "18038", "21793", "18529", "44253", "41146", "19053", "27148", "56824", "64299", "52114", "71042", "64044", "27104", "23677", "16650", "56533", "43807", "62458", "8586", "45972", "21425", "35250", "21903", "26775", "24514", "72330", "63840", "67868", "28605", "41048", "37134", "42794", "6122", "43859", "56269", "72510", "15793", "35708", "66103", "26250", "36345", "13818", "57460", "12698", "25917", "10679", "33847", "20445", "53029", "48756", "72932", "47348", "42351", "55659", "223", "32867", "46924", "53620", "8447", "4171", "19953", "32846", "16442", "16220", "5570", "17294", "35366", "61586", "39094", "41587", "18080", "68155", "58334", "65658", "35430", "41128", "43187", "17994", "54161", "68405", "22689", "56165", "57160", "71408", "37641", "4922", "41971", "29260", "44584", "73067", "61707", "68", "3314", "17445", "35070", "42792", "22718", "35094", "66637", "9847", "10486", "14839", "55256", "695", "11811", "32559", "70854", "39256", "32488", "69572", "58674", "10808", "39468", "53503", "67396", "15021", "44117", "66974", "47503", "26276", "22375", "33372", "72162", "1207", "21852", "51784", "4126", "45457", "26840", "22558", "47386", "30017", "47939", "18377", "35635", "40099", "28814", "64129", "16036", "28434", "33050", "43370", "35335", "31715", "25240", "59282", "16193", "25697", "2181", "53830", "60196", "63484", "41845", "52674", "31227", "16860", "4854", "32865", "23559", "59795", "4785", "53325", "62327", "50607", "28778", "52670", "47789", "38523", "12979", "36261", "3972", "26896", "23216", "23563", "6477", "44853", "5452", "35212", "33306", "2686", "46384", "68954", "32373", "24043", "47694", "20879", "12401", "57951", "44589", "43645", "59505", "31945", "15963", "41631", "6024", "62308", "42120", "66064", "19771", "16800", "64471", "14428", "16169", "53980", "40635", "23761", "17336", "24666", "11942", "67566", "48658", "16106", "31555", "26104", "40527", "14655", "43783", "65110", "44132", "14722", "63566", "1890", "73206", "11723", "66832", "68482", "8668", "43914", "55128", "26269", "46244", "68017", "37761", "65949", "59543", "29165", "3125", "33351", "68637", "43779", "48689", "36270", "14636", "9034", "55173", "13537", "13445", "69497", "2636", "11127", "23148", "49392", "12112", "36571", "70199", "19274", "17506", "36574", "34427", "25110", "66537", "56937", "45734", "59072", "31104", "15653", "8300", "22349", "41197", "66043", "27903", "35257", "40603", "31614", "9278", "34292", "237", "56470", "46791", "21133", "46994", "16634", "28324", "66094", "30808", "44874", "48692", "29664", "26346", "67778", "69157", "62815", "15216", "16940", "61343", "9042", "71921", "35768", "41531", "27724", "58017", "21276", "21713", "50071", "35437", "73086", "51710", "54274", "45469", "7810", "61782", "43439", "29690", "40158", "61392", "43716", "47979", "16051", "57770", "65704", "4091", "52090", "66811", "12370", "3165", "41125", "8225", "58629", "22987", "72666", "21595", "38549", "52119", "19635", "50297", "3832", "72436", "56673", "7196", "47437", "30832", "68136", "4896", "40395", "50063", "14690", "28143", "46984", "8982", "4355", "32601", "31430", "21373", "65407", "21267", "42425", "19369", "14083", "3955", "33802", "58043", "43458", "40567", "9973", "50110", "33992", "48087", "58921", "50325", "9119", "22858", "67122", "2538", "54932", "1172", "56415", "19050", "28727", "43062", "13830", "16174", "69932", "63407", "70388", "39522", "3046", "50096", "52604", "40644", "58316", "56555", "27767", "17137", "5196", "56600", "36628", "41229", "67095", "44277", "37546", "9219", "1026", "57236", "47299", "24615", "55662", "53595", "29855", "65820", "70025", "68832", "40704", "20272", "57333", "9493", "12033", "36838", "3443", "10333", "24213", "30172", "73172", "72911", "39403", "15869", "65998", "62614", "25756", "50066", "28068", "65050", "10576", "44252", "31660", "40741", "16530", "58612", "46503", "3164", "56002", "72235", "67492", "3876", "56588", "4862", "69401", "13351", "11794", "41258", "53147", "8181", "11930", "594", "56222", "32334", "28023", "35664", "49037", "34563", "33188", "68297", "17527", "13084", "5213", "823", "600", "34279", "19220", "71104", "20824", "43421", "67064", "50059", "69934", "26592", "59158", "53207", "71088", "2951", "33469", "16439", "37001", "52626", "34945", "32476", "39915", "27201", "68567", "47569", "48390", "3848", "38572", "28226", "31518", "61706", "13768", "41998", "68218", "43975", "8979", "59075", "39737", "50531", "10676", "25881", "30968", "47488", "60589", "26736", "7988", "53393", "25417", "38774", "37747", "23396", "17848", "27374", "21126", "27825", "5220", "66354", "13137", "68988", "20470", "62715", "26778", "45407", "10985", "53716", "49203", "8543", "5049", "50529", "26774", "46340", "61628", "27638", "48836", "52939", "35248", "51376", "4325", "59522", "16418", "62295", "56745", "48018", "30879", "66215", "38463", "62205", "52469", "13702", "43165", "52911", "34901", "53978", "13592", "35261", "4669", "60597", "61962", "48484", "23443", "33072", "53398", "63318", "52779", "9274", "17676", "60038", "20060", "71073", "32858", "44095", "22734", "33780", "68508", "14149", "31269", "63762", "3574", "15474", "49584", "15132", "56826", "14528", "40546", "38971", "52679", "23694", "12590", "22451", "24692", "66519", "5679", "58170", "24259", "34531", "53181", "512", "37513", "17159", "15548", "63706", "54081", "63336", "7715", "18451", "33489", "46231", "45574", "23601", "12419", "37536", "64547", "37042", "38598", "4199", "29202", "58545", "45324", "55884", "50681", "17507", "61151", "42164", "53921", "58079", "25679", "36804", "47037", "32161", "44036", "29128", "49363", "42653", "3044", "59444", "5858", "53673", "3135", "45075", "26700", "14969", "55368", "24976", "27613", "56072", "67459", "43533", "51410", "33674", "26424", "70061", "53897", "14304", "41440", "37974", "66669", "17712", "31148", "24209", "32022", "64548", "21001", "56372", "16776", "8576", "47654", "45099", "47960", "23", "54548", "28126", "26815", "7195", "45572", "3590", "61661", "46253", "28265", "16180", "13888", "37037", "4908", "29023", "37194", "64727", "30096", "22658", "35548", "1947", "44659", "56484", "41013", "15505", "13344", "36879", "72514", "51042", "52555", "7588", "30864", "2862", "64228", "43059", "47111", "34917", "72060", "36501", "26357", "18326", "33276", "22047", "3820", "15658", "60113", "59877", "107", "33364", "20166", "12957", "49690", "72772", "40823", "22873", "36507", "30573", "7622", "37009", "3838", "63717", "62002", "43844", "47248", "58577", "7801", "65717", "32850", "54406", "60024", "28173", "4034", "10107", "48010", "70542", "69536", "31055", "24986", "18766", "54155", "6824", "5318", "40437", "29943", "36420", "13685", "69513", "46375", "71011", "66191", "11627", "44756", "1319", "69427", "56192", "54779", "47626", "52283", "1195", "53321", "1909", "55996", "68490", "40173", "7751", "32325", "64608", "19663", "52018", "63588", "61348", "8185", "18562", "46577", "17334", "6209", "58863", "23954", "23146", "1743", "26036", "61339", "56279", "69462", "21913", "62019", "48583", "37171", "61242", "31001", "49453", "17938", "59836", "56356", "71949", "70190", "16644", "31249", "27664", "60947", "61444", "35287", "66316", "55679", "52292", "52127", "15450", "28164", "36042", "8059", "41760", "25049", "53193", "18526", "44414", "65946", "12435", "41442", "44017", "3032", "11539", "25963", "24251", "20866", "6241", "31616", "14438", "22965", "70114", "23564", "17343", "46600", "57192", "14075", "16142", "24655", "23613", "38823", "2110", "64174", "2311", "15748", "9627", "53850", "24392", "12796", "22422", "38128", "53116", "25965", "47334", "36154", "15630", "9131", "67747", "29244", "36972", "507", "34914", "13602", "46098", "4342", "48539", "33662", "27299", "518", "48686", "24673", "71070", "25103", "54493", "41290", "11330", "6662", "21198", "68466", "55096", "10090", "71961", "43302", "72646", "22379", "54898", "56627", "33086", "23690", "10691", "52757", "18466", "60861", "47748", "28542", "39683", "38297", "25795", "55504", "2984", "32134", "53160", "36378", "1650", "18097", "69695", "26200", "29189", "3089", "28510", "38311", "56310", "46538", "72979", "50539", "71749", "32710", "39335", "55500", "66422", "12199", "44897", "10298", "11173", "7561", "12389", "32715", "31371", "32848", "38601", "32967", "69039", "35879", "70307", "64761", "3317", "38490", "63963", "224", "10503", "39282", "12837", "56329", "56991", "68476", "26155", "70528", "5592", "52102", "46798", "8131", "63107", "56390", "33370", "63419", "49228", "45022", "53293", "29453", "3234", "65874", "14206", "21586", "36370", "49461", "3548", "21601", "29328", "30132", "59186", "30202", "11085", "52012", "64798", "34936", "63174", "67454", "37279", "63302", "41935", "50854", "49152", "17678", "67223", "32063", "30674", "1749", "44553", "66828", "23165", "12514", "30007", "38067", "50585", "52612", "4164", "37985", "56623", "67799", "61887", "65165", "50211", "15529", "26971", "51640", "12381", "14085", "2553", "42324", "4136", "70532", "55522", "54110", "48378", "24091", "29237", "57341", "48561", "57636", "19631", "42248", "66588", "51887", "7910", "15244", "63491", "72574", "67586", "58396", "33657", "49304", "9683", "66986", "51323", "1949", "56636", "7383", "45494", "25670", "58143", "34252", "68712", "8247", "51272", "11372", "65251", "3005", "31416", "14337", "45939", "39245", "16068", "14798", "13092", "11686", "18573", "15352", "17717", "28556", "28806", "60342", "26291", "27349", "30604", "9578", "43970", "69109", "51546", "39547", "3669", "27676", "37467", "37502", "50230", "64782", "1367", "60607", "69429", "54752", "7119", "59094", "17223", "63727", "55877", "29881", "13479", "11926", "9684", "22708", "793", "50332", "54909", "7528", "44723", "60864", "68560", "58231", "40258", "11134", "41141", "17918", "5360", "35360", "61002", "29699", "12877", "71239", "19562", "16646", "50120", "2136", "5944", "29045", "70408", "48455", "49292", "14641", "47566", "47663", "56238", "72741", "8466", "32191", "39939", "5462", "17535", "61879", "63412", "5056", "65995", "31584", "13431", "43633", "41692", "15307", "3566", "5970", "9534", "59789", "16032", "35468", "40318", "14841", "206", "57263", "40517", "59617", "65449", "51360", "8027", "47561", "44477", "69093", "51116", "8539", "68549", "24696", "38313", "49420", "43918", "66580", "6663", "14532", "19757", "35244", "6287", "27023", "11760", "68996", "39357", "46005", "23152", "64101", "51100", "35827", "50632", "28070", "16831", "36433", "58049", "49713", "29845", "46789", "35377", "47720", "37622", "59826", "26325", "38088", "24795", "10873", "50101", "70609", "61654", "17372", "10396", "65979", "3291", "30095", "68612", "33443", "28836", "36116", "63338", "696", "36765", "8906", "33613", "66621", "72997", "53909", "61964", "6836", "37180", "31701", "32587", "18473", "35440", "50769", "57458", "29460", "5204", "10063", "7197", "22787", "72366", "67418", "56571", "35183", "69751", "73072", "25394", "44497", "21993", "54156", "40115", "19610", "45126", "30842", "32909", "70089", "22316", "61888", "24764", "69032", "45789", "69592", "67002", "60637", "12319", "7453", "14335", "60178", "31868", "31621", "19579", "52474", "282", "56999", "35727", "25579", "55829", "16610", "12821", "72582", "42265", "73161", "59927", "4750", "63959", "58529", "9056", "1274", "28408", "16854", "32313", "52729", "22319", "14354", "20966", "21301", "18258", "22995", "4470", "44719", "9821", "44305", "18535", "22654", "56891", "14079", "70270", "32378", "11034", "69537", "31141", "47931", "11279", "28488", "16567", "776", "2341", "73068", "63034", "24045", "72266", "40330", "43720", "11083", "33505", "8504", "17528", "44769", "43088", "32689", "48106", "68593", "62845", "70894", "50018", "50930", "54861", "70973", "50148", "8927", "73089", "60618", "6883", "41853", "73175", "58333", "23470", "72890", "21243", "9273", "58937", "62763", "47161", "59013", "11857", "38859", "17850", "7348", "51651", "30626", "71045", "35533", "13547", "6117", "72319", "43326", "27566", "35293", "26136", "58106", "6646", "58675", "16080", "11450", "52653", "12506", "28049", "27560", "45408", "71283", "46285", "69303", "58928", "8814", "37893", "14851", "4678", "11853", "50473", "63035", "53350", "20606", "41958", "37033", "30777", "33867", "45405", "2244", "45858", "17049", "70674", "33316", "33033", "37407", "10252", "27391", "14974", "61898", "38616", "69444", "35714", "65806", "40022", "39128", "70048", "61251", "29078", "55161", "7615", "11267", "50857", "29801", "52694", "70471", "34617", "38435", "2505", "66943", "30377", "62469", "30144", "66173", "42708", "34335", "66400", "52479", "69762", "42075", "44071", "8826", "34576", "16365", "36479", "1354", "11580", "18004", "47907", "59824", "28363", "44711", "22501", "19424", "63382", "41818", "52686", "58694", "42370", "21370", "29439", "30806", "561", "55213", "54744", "23644", "47487", "22563", "7631", "29180", "29743", "29225", "34112", "72173", "17513", "12127", "35030", "6848", "10094", "44888", "26484", "62042", "47315", "41274", "72163", "55729", "10902", "56864", "41325", "934", "33919", "19102", "42049", "54702", "67513", "6978", "72285", "42562", "68280", "67848", "28711", "31333", "70928", "9486", "25126", "38617", "56292", "60172", "13763", "19931", "43185", "54286", "38979", "21341", "22757", "56049", "60794", "13619", "43522", "30768", "27599", "11884", "32177", "47379", "10222", "42657", "26184", "63317", "56898", "6393", "55074", "44750", "13881", "3071", "50450", "49817", "18012", "68303", "18034", "33781", "17459", "34559", "22808", "30480", "33762", "11827", "19662", "9593", "53528", "67261", "57291", "10106", "8675", "28115", "66427", "21037", "69302", "43953", "26789", "46954", "54846", "40415", "42939", "54634", "53510", "61753", "12945", "73243", "50150", "29575", "11900", "8230", "63810", "66467", "33028", "41862", "43049", "66980", "24757", "49021", "21499", "4064", "25107", "52963", "35398", "56589", "58081", "41640", "59601", "48913", "40836", "39022", "12619", "67257", "65786", "56625", "32422", "11429", "41623", "58916", "47954", "14213", "53898", "53835", "32406", "56586", "46587", "36013", "67016", "68368", "45218", "29828", "44218", "63879", "15547", "29443", "56929", "62065", "17702", "41745", "65549", "2840", "57392", "43920", "34470", "68308", "19297", "60436", "61912", "12200", "58290", "24156", "19398", "34632", "62357", "5367", "31090", "44384", "31712", "38247", "39920", "68790", "65713", "47875", "26024", "37058", "46045", "48279", "7638", "37971", "34595", "13180", "61963", "33424", "42719", "60620", "41601", "7775", "23683", "5632", "66095", "37995", "44590", "54647", "71294", "61911", "55796", "26319", "24181", "11100", "25767", "17816", "29535", "68514", "2360", "4129", "57483", "49932", "1991", "50784", "72005", "41180", "53251", "26795", "3388", "18279", "8827", "22058", "15303", "22984", "56075", "23908", "51946", "14869", "51291", "61861", "63613", "66698", "47220", "43318", "10062", "64378", "49234", "27311", "15198", "9991", "29448", "59036", "52046", "20368", "23889", "31791", "73229", "33993", "36607", "65491", "24958", "239", "60315", "13211", "10376", "8648", "10933", "65667", "5896", "8114", "28878", "8354", "57404", "4218", "23481", "32977", "20614", "54517", "41116", "57102", "65815", "638", "67415", "33253", "2308", "34784", "31488", "18256", "62606", "27288", "13831", "62918", "50690", "64500", "45821", "7151", "44044", "60165", "9992", "2284", "43221", "56599", "42304", "64438", "71764", "47281", "25262", "9658", "53655", "33936", "40539", "9571", "68591", "22315", "30757", "50141", "29825", "528", "27661", "61405", "56273", "53087", "49783", "71717", "66470", "27518", "58224", "49922", "30323", "43897", "55459", "22107", "26071", "60702", "70895", "62720", "59449", "42046", "16146", "48068", "36454", "26942", "9881", "61915", "26963", "39524", "67827", "26256", "68202", "40285", "51924", "57533", "71686", "47060", "9394", "50390", "21396", "9713", "61715", "17100", "13956", "67075", "20182", "27220", "54024", "4848", "32614", "70208", "67191", "68452", "70883", "53505", "9223", "21073", "68697", "34991", "59805", "181", "438", "1964", "10934", "52281", "28041", "63438", "32927", "34434", "59806", "31650", "46280", "14405", "26832", "2588", "27069", "34896", "20805", "37880", "47062", "41508", "72105", "11291", "1407", "41975", "69172", "49331", "2058", "31996", "68013", "46392", "17273", "52400", "23556", "38197", "40728", "20436", "24233", "52655", "64541", "49450", "43790", "14934", "38537", "64185", "20809", "35868", "9088", "49745", "40702", "67123", "58139", "35274", "63289", "52666", "53335", "3927", "72665", "64865", "45548", "69236", "12265", "32423", "15386", "56417", "73224", "18674", "27666", "23236", "20011", "60261", "8293", "35075", "56878", "1806", "48173", "9314", "28927", "57929", "19876", "1506", "47603", "19761", "36213", "70195", "38184", "35878", "16736", "38560", "34490", "70453", "5381", "13067", "54075", "56099", "41355", "504", "71098", "40385", "45535", "45426", "30609", "26870", "56286", "57062", "660", "9193", "7981", "64755", "38577", "38284", "21096", "56528", "61821", "34809", "67634", "35438", "17418", "25306", "37419", "66952", "52961", "17836", "61301", "2134", "72464", "13995", "22462", "23762", "1840", "17813", "60592", "1365", "56505", "52135", "3964", "5369", "43930", "4193", "8315", "49089", "23410", "14649", "59652", "24645", "20617", "52494", "11250", "25949", "1280", "14380", "42775", "18870", "28856", "36764", "45961", "297", "20662", "19175", "7930", "43836", "34597", "29095", "5710", "32540", "36329", "8238", "7167", "20179", "43916", "20176", "27907", "66848", "25056", "27526", "64324", "34589", "59834", "48290", "51076", "18551", "50832", "47718", "37954", "44138", "60542", "35849", "57682", "15558", "39477", "35225", "68462", "65686", "47495", "29848", "56007", "66486", "55013", "27248", "49686", "6680", "13595", "7363", "41708", "28295", "24857", "55950", "31571", "4851", "116", "38632", "52004", "57316", "22988", "43311", "4858", "63452", "35960", "36996", "50406", "39654", "68111", "51252", "59937", "15424", "23204", "6705", "23289", "38814", "62568", "21247", "27700", "60807", "33472", "24946", "61421", "26492", "1297", "54856", "33930", "20819", "23110", "47582", "63841", "5382", "62223", "25726", "30603", "54712", "46245", "25848", "47614", "4779", "55408", "26057", "72622", "31512", "54912", "36880", "55222", "30565", "42220", "7614", "52143", "17002", "52173", "47898", "14864", "61046", "13290", "38543", "2150", "3447", "13073", "69437", "23890", "58420", "52856", "54812", "8012", "26617", "31804", "12237", "66264", "25690", "26298", "51654", "33628", "33381", "34698", "48806", "7836", "23778", "17773", "14130", "11848", "4415", "32103", "46876", "43298", "8542", "61445", "37295", "55561", "71553", "20850", "42949", "72010", "8486", "32894", "8350", "24594", "18806", "42649", "30113", "33174", "25345", "70338", "55688", "45539", "7576", "26109", "66961", "59476", "25519", "58283", "72043", "68717", "47482", "3879", "37128", "32034", "68911", "7415", "10500", "38625", "71769", "56215", "68776", "53231", "42158", "30186", "48861", "32259", "26946", "24369", "66086", "3155", "62559", "57472", "37798", "60531", "45497", "44644", "16429", "27430", "35937", "48353", "30728", "15607", "43241", "44968", "57497", "9669", "12248", "13590", "63490", "51489", "48682", "48226", "21310", "6540", "60415", "64481", "41236", "61174", "27022", "64945", "6868", "6612", "54387", "9883", "57657", "41556", "9140", "32343", "36317", "7591", "67402", "34432", "18131", "73247", "68903", "26282", "23214", "7698", "36995", "22281", "5466", "54578", "7131", "1540", "72133", "25004", "14566", "19566", "35639", "17410", "39954", "66724", "14862", "30362", "41622", "36159", "67916", "33036", "42671", "35581", "56026", "33533", "52512", "5861", "32648", "56476", "6924", "60482", "50363", "64288", "48207", "2456", "30839", "62191", "21129", "57832", "58415", "48462", "47306", "34367", "61597", "60971", "55265", "479", "16364", "14831", "14224", "20870", "12210", "19228", "72346", "23460", "2680", "50992", "5630", "25847", "37372", "69098", "40883", "30308", "54044", "51750", "56895", "292", "20045", "53701", "66861", "23282", "50638", "15481", "22112", "44240", "58016", "68149", "24049", "43233", "45881", "19186", "19781", "12315", "22233", "12730", "68207", "9866", "51166", "28560", "5373", "68663", "41803", "45057", "15022", "6628", "38263", "50619", "62376", "9849", "25936", "7796", "23650", "29692", "57999", "10802", "12295", "20576", "22163", "15587", "36818", "15599", "54344", "6986", "38352", "56254", "48849", "68510", "69848", "33363", "55058", "38841", "1353", "56444", "64983", "2807", "6932", "13495", "18024", "64172", "28728", "5843", "16975", "5943", "70261", "31902", "34314", "42877", "73151", "51670", "57789", "25659", "66954", "47536", "41470", "69749", "57476", "65456", "11602", "58898", "42968", "10459", "47759", "38134", "10779", "32436", "72693", "28638", "30945", "14678", "31820", "16621", "9356", "60833", "55833", "31828", "55143", "56551", "33315", "19782", "64289", "72088", "33838", "39626", "62380", "31276", "60402", "23632", "51153", "52539", "49884", "67670", "10116", "15182", "70300", "1187", "42911", "61026", "55269", "31186", "785", "62777", "4189", "49729", "19901", "39100", "42288", "52393", "66849", "3293", "38178", "64913", "57860", "24782", "31146", "56128", "41304", "12723", "21717", "34465", "73092", "45579", "17809", "58346", "63697", "30734", "64571", "2815", "1876", "40498", "10127", "45439", "7916", "45029", "72297", "12160", "65273", "53084", "66116", "27817", "22001", "48532", "1417", "48336", "59544", "33922", "66431", "41884", "10770", "52434", "49990", "58854", "30352", "13031", "7260", "6416", "30376", "32662", "50072", "3399", "50807", "22072", "20172", "5514", "45019", "44856", "60852", "18282", "54438", "57129", "21897", "60689", "56909", "40320", "15260", "15843", "55208", "18509", "46008", "6945", "16156", "42600", "26078", "25994", "6685", "49945", "37830", "50119", "25681", "37989", "37243", "17664", "19507", "28818", "51251", "32546", "31951", "86", "1241", "53348", "42040", "19960", "8784", "57051", "25740", "69643", "45613", "50785", "30695", "11472", "31065", "55020", "39258", "8898", "42905", "34616", "18746", "9413", "56083", "8986", "47003", "6702", "22196", "58060", "11129", "30305", "8037", "39980", "71759", "47020", "30199", "8712", "36986", "1628", "50399", "12684", "52362", "41058", "26551", "40189", "10427", "61589", "26438", "64854", "23975", "34619", "39244", "68233", "52733", "16362", "35058", "13555", "57254", "24601", "58763", "11491", "27215", "51472", "8034", "10401", "51377", "23202", "2507", "63551", "8700", "21992", "58838", "6423", "25986", "14794", "44287", "72705", "72268", "55589", "62607", "47858", "36959", "3992", "41747", "57823", "43823", "36661", "72838", "12526", "23510", "65548", "651", "69219", "59775", "7574", "27571", "10095", "54689", "15447", "26964", "64722", "47389", "33507", "62183", "68108", "51781", "402", "66838", "61739", "40830", "35362", "70326", "60065", "17478", "21816", "43144", "20575", "24010", "20518", "54145", "22165", "35684", "8936", "34075", "52007", "19234", "22992", "2697", "41961", "54953", "10305", "49495", "18848", "15150", "45188", "52947", "49323", "33968", "33985", "65406", "199", "20651", "13657", "46917", "19598", "51213", "55169", "17463", "41555", "18076", "251", "10613", "58881", "36999", "17229", "18987", "53691", "40172", "49605", "62136", "5107", "62120", "68906", "36363", "60780", "43268", "73156", "56686", "65027", "50447", "52258", "51885", "51075", "21550", "52977", "26927", "60954", "62187", "51753", "892", "42540", "45691", "1132", "45947", "44700", "64335", "51983", "67787", "57041", "1316", "40058", "10245", "30021", "32981", "27643", "5933", "28562", "13919", "47964", "18725", "29248", "50475", "71701", "14061", "8212", "30122", "25766", "30301", "30104", "15960", "11886", "64132", "27600", "564", "8359", "51841", "12154", "37221", "17703", "64209", "20174", "37613", "63763", "15461", "59132", "12123", "19942", "48223", "16982", "37176", "13799", "16256", "26244", "50351", "15886", "38406", "34873", "10913", "40305", "56637", "12561", "42126", "42376", "56267", "66830", "48196", "43903", "1728", "55152", "24475", "53025", "33423", "26301", "21090", "53040", "62409", "63130", "16505", "41072", "30263", "57939", "21938", "25939", "63533", "2198", "69733", "41321", "72248", "33326", "67445", "50771", "67314", "72212", "72384", "27036", "68779", "41165", "41505", "29034", "16264", "21376", "25674", "10605", "72773", "22620", "35745", "71935", "62089", "44256", "66817", "61701", "8981", "56370", "902", "5421", "6277", "7211", "57405", "72612", "10446", "7392", "62779", "30756", "72291", "7290", "8770", "48624", "67092", "28382", "69103", "19403", "22274", "20988", "68057", "66628", "57915", "59863", "3309", "15169", "65997", "54980", "59065", "42417", "9924", "61678", "36739", "7773", "67961", "48027", "60727", "19359", "70802", "58505", "40720", "27468", "67793", "39385", "55427", "47521", "60214", "39708", "57538", "21121", "53342", "65629", "42611", "21632", "63208", "55963", "32726", "3552", "38111", "42297", "71817", "35504", "11643", "38924", "45334", "33836", "22152", "33377", "67453", "50956", "30079", "55782", "17885", "47191", "35072", "55499", "69097", "68643", "10322", "166", "6818", "52175", "45991", "20726", "33966", "59155", "71085", "37539", "10995", "34518", "33946", "11623", "51097", "60054", "66480", "52039", "34043", "7673", "9457", "56226", "73197", "49992", "71924", "69412", "26149", "15235", "19794", "29838", "20969", "73100", "22721", "35623", "65162", "41969", "71741", "2600", "8393", "50738", "56821", "1270", "31726", "27203", "32320", "27703", "29326", "16269", "41417", "43503", "53191", "21651", "23259", "41335", "26823", "61593", "26518", "64038", "28487", "67959", "64710", "56432", "24973", "47247", "67859", "64972", "43591", "62770", "34922", "61191", "24521", "33500", "18433", "39948", "42169", "2478", "63788", "44039", "52289", "57790", "11486", "33133", "52853", "73138", "57758", "47575", "9414", "14535", "37435", "7092", "29722", "40710", "32349", "42632", "42506", "33280", "61341", "46815", "50383", "59088", "63766", "9021", "9623", "215", "58151", "40629", "60583", "63135", "6666", "38743", "4051", "22197", "39848", "23390", "40761", "12008", "69833", "29871", "62552", "60714", "39427", "69140", "39526", "56214", "67299", "28986", "52672", "51210", "22246", "22181", "24165", "64162", "56138", "21573", "57281", "31532", "38500", "28571", "61836", "43862", "59985", "15853", "46034", "61832", "65939", "47469", "70725", "36558", "62106", "37921", "56561", "891", "64281", "66055", "14273", "13337", "2079", "5099", "48077", "68243", "62277", "65505", "25953", "53183", "47444", "40426", "53371", "8886", "40849", "56111", "60156", "21327", "26038", "8168", "17634", "39735", "45247", "6519", "35936", "40918", "22414", "8210", "56260", "16983", "71430", "57089", "59147", "1769", "52263", "33179", "28924", "26471", "46188", "557", "43397", "38302", "24837", "56402", "41648", "301", "38481", "1704", "44648", "41735", "634", "58169", "66856", "31631", "41401", "12986", "57977", "11069", "53675", "19965", "18997", "51704", "13379", "35039", "45773", "63707", "51696", "11624", "49274", "66189", "27713", "45339", "49315", "9259", "36977", "55855", "48260", "14367", "49087", "63424", "37890", "35061", "10103", "43772", "42969", "71562", "69014", "71974", "38327", "36137", "2506", "1382", "58619", "43878", "66853", "56184", "69272", "34646", "65306", "59120", "14063", "55336", "62494", "68292", "32163", "23738", "66190", "12153", "29816", "69475", "67277", "68852", "12285", "47457", "10272", "65431", "48683", "27459", "43200", "59631", "62237", "55079", "34899", "52475", "39227", "47568", "9541", "15803", "12874", "30515", "23728", "1291", "43678", "6318", "15685", "49589", "71824", "38591", "54124", "30880", "23105", "11073", "8947", "21676", "23894", "37541", "23851", "23803", "31953", "66168", "65936", "43053", "16896", "37186", "19982", "48820", "48720", "52431", "62291", "27329", "19886", "12182", "57110", "19634", "59393", "25295", "9167", "4975", "62001", "59429", "4391", "72069", "72857", "52774", "45232", "49961", "15490", "27546", "29371", "32529", "22350", "3947", "16595", "60730", "7286", "33612", "35138", "55657", "31272", "60749", "56203", "47208", "10377", "69026", "43355", "71758", "61977", "35347", "72557", "11904", "2303", "31911", "64034", "9149", "38424", "54957", "39545", "59898", "3818", "36856", "60186", "48513", "44250", "54997", "71909", "4788", "1174", "6398", "60328", "71131", "26442", "10037", "24610", "64723", "20873", "66034", "588", "30180", "30156", "1632", "24307", "24606", "72221", "40751", "32116", "71411", "34435", "32713", "24023", "802", "40921", "72184", "35719", "72999", "7582", "15985", "31277", "37379", "24001", "47267", "29935", "53262", "20566", "38151", "46712", "11487", "5313", "28672", "14961", "50604", "69210", "44612", "23133", "66829", "53100", "10560", "34425", "8363", "26356", "739", "46632", "30648", "11145", "7233", "45344", "6146", "59499", "27749", "54916", "60403", "1829", "19766", "6633", "64221", "15254", "19312", "40245", "3193", "72230", "63525", "52446", "26141", "64371", "70695", "39377", "17417", "63506", "70040", "64685", "16811", "46795", "64526", "53890", "54026", "42202", "15226", "45765", "10713", "39002", "54740", "30687", "8353", "17434", "27011", "34500", "46946", "22814", "40707", "10890", "23238", "7201", "38108", "22728", "55665", "41121", "50314", "59297", "57581", "4781", "12032", "73248", "37159", "19996", "55352", "48824", "70045", "28254", "32270", "34541", "41725", "23384", "19944", "13128", "23619", "36865", "68000", "18959", "68393", "9722", "60002", "17534", "19632", "58565", "13201", "37569", "3273", "39933", "42016", "71132", "72347", "50413", "30940", "15665", "41300", "57798", "59073", "50208", "18482", "43867", "62632", "52199", "19716", "17591", "68253", "20222", "35511", "33375", "67343", "66436", "41382", "45251", "45938", "67696", "48808", "17204", "53090", "36577", "43381", "26360", "2297", "40490", "16599", "58620", "66664", "263", "6907", "5060", "6481", "19777", "40558", "17430", "45442", "4183", "69648", "55286", "13157", "23064", "40468", "31961", "48795", "34410", "58900", "58024", "8323", "58807", "5556", "24939", "59071", "52868", "72604", "9415", "54292", "37634", "71662", "71770", "50024", "21703", "72940", "3204", "30589", "50130", "44561", "27495", "70618", "22365", "35780", "2537", "71273", "32216", "21136", "19687", "46083", "64843", "31059", "44525", "55672", "64303", "13862", "44472", "57078", "6359", "16295", "38195", "65185", "42247", "56349", "11422", "6050", "66171", "63427", "44571", "57382", "37313", "40540", "45746", "52553", "146", "67306", "30335", "30185", "6127", "7621", "37608", "60204", "58727", "48912", "49062", "23931", "21420", "29985", "15003", "49023", "10140", "27194", "69023", "46192", "28909", "20194", "18792", "31145", "72822", "25586", "23120", "55677", "57534", "39453", "17575", "4113", "72790", "62140", "21393", "51927", "50757", "34265", "22692", "60894", "936", "31784", "41394", "43656", "46444", "22280", "61072", "60077", "57656", "49985", "3632", "67898", "17452", "1992", "54959", "6430", "18343", "34748", "50774", "61991", "25947", "16812", "50191", "6140", "6330", "64796", "2002", "9311", "6213", "44931", "14170", "45451", "42575", "66366", "4031", "23255", "34157", "17866", "4123", "8473", "65466", "7083", "22680", "30554", "16113", "56218", "48774", "26644", "38918", "7858", "31972", "44988", "25698", "46507", "60883", "27764", "66924", "46218", "15265", "59539", "21791", "58251", "50052", "48047", "61797", "28833", "71712", "56508", "54258", "17640", "72496", "56420", "15238", "63355", "46010", "51334", "68484", "48348", "49383", "63615", "18397", "52916", "55021", "28358", "59261", "43021", "27936", "32879", "25921", "34359", "18446", "27856", "33163", "26620", "56524", "906", "26053", "728", "11767", "32289", "27174", "47261", "6759", "63344", "29489", "58033", "37071", "16088", "51130", "35090", "52737", "45031", "61061", "10742", "28620", "1090", "30700", "19314", "61677", "36888", "19624", "4821", "9359", "34441", "49176", "811", "39052", "66693", "22745", "5179", "1801", "5422", "65489", "12771", "51629", "11902", "13563", "62209", "44099", "56876", "51143", "5231", "67789", "52771", "32695", "69664", "36886", "42092", "32165", "43572", "32368", "2183", "69498", "20229", "23046", "42770", "5789", "24938", "44692", "45988", "65777", "38585", "41472", "12172", "12185", "3232", "58496", "23807", "16518", "48337", "72059", "11943", "49785", "35735", "30514", "7983", "15048", "56118", "5264", "49917", "35340", "3041", "45172", "67530", "32590", "28832", "12799", "40621", "2810", "13018", "49560", "57617", "56721", "36468", "29020", "68251", "58633", "59106", "36897", "31266", "13446", "36408", "1391", "36862", "26174", "18780", "39420", "21494", "62996", "73087", "71129", "16709", "67721", "29543", "43928", "72117", "3363", "10467", "37499", "44891", "25169", "28067", "64259", "9555", "41860", "18213", "18403", "47093", "6088", "42521", "37977", "11744", "65839", "56914", "28960", "250", "40319", "34220", "29666", "26088", "50836", "70817", "28301", "2054", "27300", "33590", "61015", "11750", "7840", "31311", "72572", "4013", "32499", "18462", "43320", "63620", "43648", "17602", "13388", "16589", "41614", "47932", "58053", "64852", "27775", "52463", "1012", "32023", "24619", "60260", "49270", "19611", "35520", "29884", "16830", "34103", "29352", "44847", "58376", "1900", "70759", "29081", "30584", "23525", "44829", "45865", "40339", "8987", "2724", "50251", "22748", "48354", "3794", "39675", "9363", "54224", "15102", "33524", "59632", "52357", "56772", "38244", "15236", "26845", "1598", "19775", "29588", "63761", "26802", "12047", "14589", "1199", "38658", "11821", "51462", "52224", "50660", "51752", "26403", "2508", "37356", "42127", "45367", "35089", "30145", "54314", "7641", "61717", "18394", "21974", "10693", "70238", "64794", "68374", "27501", "49695", "65030", "51139", "38992", "64832", "68170", "23682", "33021", "65265", "11783", "43619", "39265", "10813", "45191", "59170", "44906", "48987", "19725", "41757", "10674", "56080", "21604", "65747", "2545", "44436", "66255", "70073", "48463", "59573", "69611", "5654", "41038", "48482", "34927", "66700", "33562", "63913", "33631", "49963", "22800", "15049", "10381", "33874", "19920", "11652", "29519", "31653", "60887", "59326", "12962", "27192", "4351", "18656", "13922", "25808", "66291", "13676", "37756", "72558", "22690", "67592", "61204", "24314", "22915", "623", "45274", "30942", "113", "65581", "65520", "2095", "3633", "37843", "7033", "47116", "7595", "50385", "54514", "66372", "46530", "25506", "8666", "52129", "5678", "48530", "65237", "58474", "63987", "71340", "41681", "29936", "27450", "8985", "28775", "14936", "52994", "59888", "62576", "45491", "18641", "52416", "21943", "13866", "23903", "21762", "60591", "8189", "4132", "8286", "9791", "56361", "32439", "55692", "2363", "33775", "8565", "12167", "20904", "68387", "15604", "37184", "53619", "27759", "143", "41299", "67126", "68707", "49470", "46578", "55435", "50694", "70518", "25285", "32812", "11533", "29507", "51689", "29805", "66419", "51467", "4916", "28087", "13268", "34983", "25470", "43785", "48002", "20689", "16320", "5329", "2843", "32413", "33464", "30911", "15326", "42036", "54741", "54059", "8090", "46321", "34324", "33582", "18303", "62706", "19678", "41841", "67953", "33393", "58791", "21465", "69143", "30733", "17026", "39822", "52349", "71890", "48110", "69605", "5986", "63753", "50023", "36294", "40772", "13458", "33976", "3615", "70075", "25948", "53277", "26899", "29015", "63285", "15668", "13982", "68198", "29303", "49262", "52069", "65956", "18753", "19936", "40968", "67992", "58436", "70890", "22174", "43362", "27927", "22671", "64974", "6195", "14900", "33787", "69967", "3877", "290", "16512", "31691", "7446", "57529", "42368", "7702", "37984", "63487", "40957", "62538", "26904", "21947", "22788", "6928", "68316", "17837", "20569", "18653", "70251", "4001", "3159", "9183", "45403", "54227", "43248", "70539", "3769", "901", "34285", "39618", "72806", "66993", "35680", "16778", "57451", "35327", "4631", "66661", "17568", "5451", "49160", "159", "51170", "55889", "3268", "60957", "13417", "53648", "28204", "41483", "44329", "9996", "45052", "20369", "47741", "25811", "26611", "43267", "66018", "31517", "36794", "20395", "15895", "6302", "26616", "65770", "46077", "49365", "61708", "29584", "23655", "2449", "47391", "71816", "59571", "61095", "50310", "66495", "61194", "47623", "29621", "26211", "50393", "59440", "5120", "66804", "1865", "50921", "45328", "29178", "46546", "67365", "42494", "51368", "9814", "7001", "22049", "9110", "14827", "68222", "15715", "72349", "48805", "72719", "59977", "42511", "25677", "8006", "17175", "2842", "23554", "46064", "31293", "67798", "61171", "67758", "72223", "15932", "59923", "22384", "5761", "5696", "31124", "40595", "34915", "66605", "16049", "7968", "28489", "34895", "53148", "51293", "38113", "24581", "57954", "16798", "70013", "12785", "2914", "12737", "4133", "55546", "41108", "29412", "44118", "4392", "44851", "911", "37696", "42863", "26812", "57264", "3610", "6632", "28918", "8416", "4489", "23801", "10479", "9844", "48367", "18074", "43469", "44914", "65226", "11058", "40104", "9697", "16495", "30303", "49413", "69227", "33112", "70541", "4131", "8614", "45993", "21876", "29892", "36552", "46053", "55090", "4254", "48072", "9026", "22064", "37282", "62766", "34350", "59419", "69391", "25705", "11590", "66195", "49411", "58494", "56515", "69156", "25591", "6697", "54138", "14407", "21882", "46738", "65880", "62189", "25712", "42311", "4799", "31541", "12533", "59222", "24574", "17611", "59450", "65800", "20332", "16476", "12885", "439", "1892", "19302", "33841", "14119", "46774", "58301", "13807", "9561", "73019", "36772", "62874", "67667", "34819", "6316", "49527", "32702", "10351", "9351", "53265", "30792", "27072", "35531", "38674", "30765", "13824", "6227", "41939", "72394", "26699", "48144", "41026", "40731", "1396", "41702", "21230", "38897", "46389", "50186", "2227", "9485", "25278", "70909", "62290", "65125", "30009", "50891", "33258", "23501", "24495", "64087", "56836", "59156", "35816", "10471", "45144", "17980", "29590", "2321", "191", "50193", "65805", "11653", "38709", "72538", "12023", "43031", "10850", "59796", "68993", "9010", "71171", "67552", "38031", "16577", "16835", "2287", "16118", "28484", "4998", "25912", "12040", "1805", "68775", "43773", "7653", "2316", "27013", "66703", "27258", "19177", "67501", "29896", "9792", "64600", "50989", "56740", "39084", "5260", "38161", "51649", "27207", "27441", "24461", "62814", "64830", "60467", "66863", "8771", "5478", "23079", "2009", "3793", "21485", "18810", "19378", "29795", "22101", "42843", "17906", "71913", "19451", "61349", "5", "35746", "32814", "58636", "48611", "70661", "16541", "40064", "4196", "39746", "24598", "49303", "55653", "60738", "35442", "34481", "58191", "28719", "71515", "35602", "30429", "9818", "45132", "67519", "29606", "42380", "17768", "52016", "6660", "31134", "66546", "27717", "59744", "13636", "41217", "43191", "19360", "13969", "8040", "65388", "23704", "10816", "63754", "17127", "58493", "49526", "8384", "57695", "49436", "46920", "65795", "60434", "71438", "14504", "63680", "63932", "33698", "8735", "67214", "54994", "32308", "44809", "46967", "60976", "52607", "13587", "33332", "5893", "15487", "71455", "54398", "22117", "62991", "70115", "55144", "22474", "63190", "3282", "6568", "10067", "42280", "18599", "12950", "67045", "65307", "55475", "56309", "14057", "45415", "4414", "23218", "59308", "13847", "70487", "69273", "39459", "35235", "39240", "50780", "40754", "31377", "19420", "66482", "21790", "24011", "71267", "64574", "61835", "51165", "62109", "55446", "71834", "35157", "8738", "46683", "17735", "69383", "27842", "6741", "19475", "41117", "4084", "36053", "62396", "54768", "44574", "49121", "71323", "44078", "58536", "52663", "58588", "9961", "56321", "47230", "44941", "53404", "3690", "31634", "37449", "25604", "43212", "8112", "54657", "59313", "11272", "54219", "68290", "20928", "34431", "34731", "43802", "68635", "35665", "25277", "19956", "71111", "10164", "39636", "23091", "61143", "42217", "53103", "303", "69640", "54592", "70565", "44521", "48696", "33642", "38788", "47619", "66662", "48765", "65609", "59431", "998", "53192", "61182", "33475", "28035", "69356", "34040", "52418", "33239", "24435", "32652", "17457", "52651", "42921", "37855", "70330", "10776", "29060", "49832", "16684", "45998", "48848", "45466", "44237", "7763", "16921", "63860", "60449", "1693", "12841", "25488", "52152", "45550", "20946", "15207", "25699", "5259", "55664", "45134", "14672", "43835", "14460", "10725", "22812", "20799", "39433", "52587", "52922", "48452", "38260", "7841", "61440", "12194", "5516", "57895", "32056", "32554", "18799", "57038", "54539", "65274", "705", "65907", "39445", "30710", "70827", "47572", "1431", "2378", "25548", "70879", "5444", "34802", "65993", "45398", "18041", "41337", "65848", "65752", "3508", "53005", "40783", "65002", "40717", "61757", "59455", "40593", "17381", "40639", "43608", "65804", "52476", "52346", "52974", "28145", "44848", "15439", "35037", "60867", "59645", "72944", "54389", "341", "10520", "55053", "20365", "21933", "31882", "38122", "28945", "37527", "59838", "64579", "11536", "69516", "34381", "44488", "21447", "70939", "17349", "45874", "46848", "34181", "37775", "7170", "63328", "73234", "15968", "60182", "65067", "4275", "52096", "29570", "57214", "13806", "58884", "49561", "67156", "461", "28575", "62", "40407", "63785", "60443", "25303", "51575", "12842", "45360", "18749", "47812", "37735", "47295", "62747", "5098", "45060", "64139", "2753", "8949", "7542", "1066", "53560", "33585", "50560", "69258", "30557", "55839", "40654", "48630", "26614", "24840", "41790", "61975", "61501", "4988", "30478", "32684", "36595", "55703", "64094", "13041", "35659", "29404", "39792", "42844", "50804", "2548", "40911", "24248", "9976", "7752", "31223", "72663", "45222", "6775", "45948", "57413", "369", "66973", "55772", "23749", "10539", "13639", "55599", "62387", "37558", "15681", "16263", "38448", "6057", "16828", "3252", "35903", "27096", "50552", "9750", "47644", "53403", "67023", "14233", "18767", "68195", "30168", "49755", "54724", "11570", "3863", "24762", "13107", "70097", "38802", "44852", "2405", "49346", "31813", "26202", "58115", "31531", "64401", "66135", "59473", "44255", "41823", "43759", "68499", "38634", "54474", "36763", "8775", "6788", "57709", "32912", "19863", "16125", "52551", "12168", "5855", "22284", "43574", "57831", "65639", "63755", "37688", "53664", "34840", "51562", "37366", "9147", "21300", "66263", "31421", "38063", "42343", "58696", "48843", "55602", "27161", "36624", "409", "44996", "65212", "61856", "71531", "10215", "60748", "17073", "33799", "32283", "25570", "52824", "22642", "58985", "39614", "53764", "71197", "70456", "16576", "43046", "68511", "21529", "63331", "31740", "48145", "49954", "63814", "3886", "67266", "27618", "64146", "22828", "67641", "20532", "60061", "52170", "50346", "27098", "1067", "46627", "59414", "42635", "18961", "46080", "11125", "32356", "71110", "69025", "67490", "57372", "64537", "33150", "63324", "51099", "37895", "10926", "47021", "31651", "52745", "6499", "71296", "36787", "39690", "15765", "30091", "42721", "70473", "15645", "46027", "3827", "5485", "18249", "33778", "53508", "3187", "24640", "10755", "71100", "16635", "54442", "52635", "5450", "41624", "1471", "71542", "62969", "50242", "5705", "16131", "55536", "52189", "39916", "49983", "70610", "44995", "8681", "24627", "6838", "36521", "23235", "16654", "15224", "41002", "66463", "43818", "67847", "36381", "31887", "7830", "28004", "20800", "41118", "18165", "36885", "71525", "62161", "5255", "33213", "48217", "25139", "58949", "5129", "54062", "44113", "34865", "36812", "21352", "21220", "14530", "60490", "63857", "66842", "41638", "18359", "48399", "13448", "31663", "49235", "23256", "25590", "67606", "18624", "17144", "45936", "33815", "3918", "620", "51961", "70421", "51637", "809", "34883", "22228", "51888", "12682", "68432", "12717", "56401", "25530", "7468", "9775", "37773", "15436", "9810", "42242", "48751", "42266", "14010", "55076", "134", "36373", "39231", "28579", "65436", "69287", "14540", "60875", "41010", "71236", "59763", "25464", "45332", "51626", "21359", "272", "65498", "48218", "7586", "26271", "38889", "55487", "43415", "11896", "39262", "42187", "63563", "42790", "31801", "59529", "23024", "63583", "25879", "33987", "39469", "67198", "67120", "65988", "33755", "55954", "35071", "57842", "23742", "14574", "44987", "59804", "31878", "44297", "29348", "46704", "50946", "62907", "33115", "5875", "17877", "56831", "12787", "39484", "37040", "19955", "36985", "41565", "21410", "5092", "17968", "66258", "8959", "64792", "33954", "21231", "11726", "52091", "5037", "29993", "67619", "55129", "70387", "44238", "48191", "62243", "43032", "72478", "47117", "7781", "63240", "41068", "11289", "37627", "52415", "46049", "18214", "48477", "36295", "54260", "61267", "37017", "8505", "7417", "69135", "34503", "28288", "43945", "11268", "62036", "47729", "33153", "4239", "49580", "64655", "53793", "45932", "35883", "65974", "52208", "42509", "3274", "552", "61359", "36054", "34614", "32620", "43595", "27279", "56790", "7554", "34111", "24803", "2910", "50279", "34382", "31997", "56674", "42194", "12065", "18132", "42967", "38304", "32297", "29437", "40561", "29268", "30764", "35764", "7646", "22279", "6239", "63920", "59639", "5209", "50890", "60198", "44817", "53021", "11660", "70547", "63988", "24368", "48076", "25568", "8558", "28581", "16242", "47171", "17665", "1209", "41042", "12148", "55295", "62540", "12560", "57444", "22214", "66881", "41763", "31358", "47773", "46901", "64489", "70881", "41792", "31859", "62820", "61482", "7101", "52835", "9307", "38177", "37212", "40085", "17949", "53789", "58373", "25274", "11310", "51079", "43440", "34841", "56056", "8667", "51344", "69554", "54900", "57847", "58786", "52027", "14424", "30943", "42518", "43952", "34509", "63335", "35963", "35181", "42272", "646", "58888", "47447", "4886", "15166", "70249", "72595", "58660", "70264", "70954", "13225", "30803", "16378", "71282", "72509", "35339", "51829", "30142", "26010", "18893", "20243", "66589", "23355", "14043", "47226", "27633", "72130", "23906", "44549", "3411", "38456", "15632", "6066", "30294", "37718", "19898", "58522", "1737", "26549", "68422", "48777", "72071", "51486", "23933", "58309", "23865", "60483", "28678", "642", "58551", "42275", "4574", "40833", "9019", "40740", "63350", "1591", "9111", "37206", "66266", "12358", "57664", "43758", "15545", "57081", "7065", "47012", "64837", "11503", "36774", "68669", "40714", "34928", "46948", "69387", "57277", "37871", "64278", "15498", "6334", "37671", "26764", "7301", "6000", "52656", "48812", "54509", "35007", "27896", "70930", "6799", "24731", "20939", "36482", "20633", "71889", "57495", "66325", "40280", "5805", "72213", "1136", "14797", "3897", "36752", "24285", "49175", "22405", "61065", "28492", "58805", "6221", "20429", "1455", "43432", "5076", "48444", "4217", "35286", "24870", "48985", "54905", "2376", "47360", "67180", "69937", "56495", "55055", "50142", "6434", "46669", "18803", "45714", "69381", "47673", "49910", "38096", "12829", "37142", "37326", "30955", "12572", "43729", "58802", "45181", "65203", "18672", "63875", "53621", "65266", "59890", "60932", "51661", "37344", "42695", "38361", "35432", "53867", "59582", "38265", "3420", "71004", "57007", "3917", "33665", "68036", "72720", "489", "59896", "65774", "6468", "63577", "59359", "54073", "38981", "32693", "61303", "26591", "4942", "21695", "10084", "20006", "31745", "41804", "43435", "52850", "62103", "71036", "25450", "18721", "68423", "4462", "11788", "15605", "71607", "36257", "71985", "62775", "26925", "38748", "54990", "54566", "66784", "62437", "34406", "973", "34543", "40922", "9806", "40631", "34551", "59385", "43549", "15178", "72475", "9640", "52038", "51312", "31405", "25402", "49071", "32653", "65140", "71979", "70466", "48188", "23237", "26028", "53411", "54369", "42779", "49802", "56152", "55296", "61515", "45683", "71196", "37414", "30924", "72175", "3149", "4822", "43287", "36506", "57863", "3683", "29427", "14252", "63283", "49195", "8599", "17687", "44257", "37411", "7066", "44050", "53150", "61696", "56550", "54888", "66821", "59190", "7057", "23232", "68347", "54924", "3539", "66731", "35918", "66530", "69954", "12240", "26272", "52854", "66800", "30115", "16679", "49849", "52818", "55207", "59110", "18648", "45646", "52151", "6281", "29680", "33477", "4213", "12372", "65076", "1696", "54253", "8682", "20732", "13292", "66144", "10436", "21275", "34116", "14645", "63873", "43149", "7373", "1110", "66732", "60900", "41821", "33386", "36208", "68915", "63241", "61019", "64739", "22830", "59436", "41174", "8281", "3405", "49222", "39014", "25380", "27101", "46120", "30197", "3454", "22891", "68899", "23226", "43685", "65176", "51028", "21374", "12274", "71388", "17303", "58150", "20782", "1996", "20333", "48328", "69086", "40216", "23990", "32280", "66335", "8115", "57814", "37757", "56227", "61507", "23754", "67413", "72792", "44007", "58464", "54968", "5549", "51091", "38094", "72619", "2432", "3954", "50991", "39563", "34695", "22617", "27140", "27058", "55461", "30434", "38371", "24266", "32129", "71176", "58398", "36286", "54559", "59597", "6471", "66338", "19596", "37008", "58630", "1936", "42566", "36347", "69620", "13335", "21591", "2568", "31012", "49874", "30364", "36660", "61584", "43839", "28331", "61334", "45030", "63054", "38564", "54288", "46301", "40996", "10586", "45709", "24124", "26231", "18817", "18071", "4652", "48956", "46686", "22441", "46069", "68849", "53826", "43760", "73069", "67098", "42479", "20681", "41929", "46589", "12394", "28418", "49265", "18495", "27326", "5199", "7610", "62076", "10670", "55847", "24850", "15483", "19158", "71226", "46651", "35326", "63606", "48142", "67430", "25311", "58725", "71313", "18920", "3907", "6837", "31302", "38575", "40452", "14957", "6225", "1361", "48894", "10456", "57736", "1768", "33010", "25072", "43706", "31665", "38613", "22831", "45093", "3952", "11850", "70163", "61298", "11648", "16475", "36500", "21323", "63614", "14088", "56729", "54987", "49999", "49780", "47428", "4693", "56516", "18541", "41433", "58564", "40364", "22665", "59719", "61286", "47808", "2607", "20285", "37317", "72763", "61232", "54719", "33798", "49649", "2979", "11222", "6385", "31250", "8709", "48485", "48863", "5088", "25637", "28211", "68578", "47612", "54437", "57266", "37235", "31729", "43874", "39647", "28695", "72263", "46975", "61456", "48400", "24152", "20267", "49123", "66227", "49336", "30853", "40655", "530", "719", "49219", "54100", "9377", "70548", "42897", "20857", "35316", "10929", "48575", "3775", "30448", "51279", "28267", "17904", "50200", "9239", "39088", "39940", "47539", "42548", "10293", "59562", "32872", "19373", "23898", "12120", "15239", "44857", "26228", "53273", "25234", "54808", "216", "46748", "68655", "29258", "36671", "517", "19756", "61159", "6498", "38533", "68399", "52687", "31071", "33860", "73111", "14226", "41020", "15800", "72128", "43398", "38665", "66904", "59397", "22026", "62663", "41099", "24432", "7716", "71071", "57003", "3048", "67116", "41663", "47656", "37969", "39438", "70352", "59613", "57355", "8965", "41298", "546", "27570", "44823", "32960", "29765", "42651", "6034", "29983", "13118", "54144", "69906", "45155", "13991", "16333", "55343", "12483", "38496", "50963", "34064", "17607", "50903", "45184", "22562", "3811", "33742", "27978", "68053", "54238", "60133", "12189", "62007", "65597", "13241", "7694", "66167", "38418", "37736", "3080", "13817", "38132", "15827", "28559", "69945", "25333", "48700", "67814", "71937", "54922", "65311", "73049", "49927", "16508", "43887", "48108", "3150", "68726", "63751", "16637", "12227", "26798", "34843", "71124", "68348", "44389", "22021", "28861", "11214", "26807", "70222", "73192", "12402", "69799", "41314", "67908", "38921", "56875", "29730", "50073", "13486", "42088", "29992", "48495", "50977", "68055", "1221", "31969", "32173", "50497", "70899", "68671", "47991", "10460", "69189", "52433", "829", "66850", "39135", "36518", "28978", "10348", "15240", "3751", "65510", "7566", "40947", "33222", "71308", "17928", "64007", "52698", "23884", "61292", "40545", "45358", "72004", "45890", "255", "56172", "35419", "66852", "28183", "29030", "61111", "32196", "3378", "66407", "51093", "39913", "42507", "72830", "52848", "61231", "6719", "51833", "48030", "72866", "6452", "22683", "16321", "57146", "21772", "32570", "41418", "5105", "23737", "54967", "65976", "63348", "19536", "64011", "3277", "13078", "52316", "20709", "3796", "14806", "40652", "31158", "16704", "53690", "66758", "16296", "10447", "4214", "25159", "64957", "20110", "21372", "11163", "14503", "63895", "39389", "17884", "30502", "52861", "23616", "27516", "53787", "5675", "34287", "8969", "16603", "61386", "23199", "43119", "27408", "36910", "22508", "68656", "30244", "38011", "38686", "29315", "60041", "45003", "59983", "44004", "46943", "67066", "40095", "65052", "50554", "63160", "3582", "28106", "65355", "27998", "22244", "20140", "1990", "29981", "4371", "53244", "40093", "3024", "7828", "14018", "49974", "16094", "49463", "30241", "50644", "68471", "23220", "39457", "17892", "20027", "45460", "1516", "64133", "22381", "17653", "58653", "12036", "59512", "24154", "64006", "27440", "63782", "40675", "71505", "67145", "26850", "30128", "16691", "20397", "6070", "38610", "31231", "16164", "5689", "15310", "32779", "49257", "53010", "51204", "60012", "52620", "69693", "27448", "54817", "54652", "5599", "38983", "51570", "24512", "41265", "9632", "55865", "34512", "60804", "4996", "50264", "9482", "54020", "33465", "49782", "19265", "4957", "54716", "25937", "34636", "21468", "68580", "67062", "37789", "17361", "50418", "70429", "24665", "54307", "9176", "24642", "61397", "23394", "27", "38529", "46574", "64808", "8882", "58004", "60427", "61336", "38779", "16561", "37684", "23917", "58242", "58638", "10119", "53252", "27462", "21649", "1491", "50814", "70313", "24528", "4412", "5408", "39556", "16104", "62111", "9519", "46383", "29462", "49119", "61511", "8419", "9205", "9862", "23788", "65756", "38362", "323", "64360", "50207", "724", "73164", "47643", "71668", "3784", "67526", "58920", "26515", "9136", "43159", "39210", "73003", "559", "1564", "58601", "17926", "68604", "12409", "37900", "60773", "72135", "1034", "37376", "45194", "60013", "44532", "13269", "162", "13855", "56195", "13134", "722", "17348", "23201", "63576", "24474", "71946", "6961", "17311", "41911", "55233", "12923", "56499", "73235", "15416", "35977", "1047", "30363", "16797", "22046", "51236", "33638", "71888", "19884", "34845", "72688", "44863", "61161", "23324", "1383", "36647", "41568", "4337", "47636", "51505", "21077", "24099", "45590", "4907", "50490", "1428", "778", "37327", "69112", "23615", "6247", "48526", "31181", "64919", "50625", "40030", "32969", "66197", "12396", "67819", "1169", "45703", "196", "46170", "37200", "38576", "53872", "66893", "3737", "47596", "65592", "52310", "6248", "19485", "16846", "69294", "55057", "66859", "55449", "61681", "60306", "71163", "61351", "27995", "71602", "42687", "38194", "60092", "49177", "11762", "39269", "58225", "60430", "70893", "56017", "28291", "22357", "29341", "22964", "28787", "29977", "8897", "45384", "52470", "24855", "59906", "8101", "12475", "13186", "62683", "65684", "69845", "66944", "59059", "18085", "5175", "42170", "14651", "465", "17645", "17429", "22283", "8824", "8863", "8491", "49834", "3477", "664", "62934", "51647", "71436", "52546", "45243", "31457", "36274", "25951", "6579", "71957", "41749", "42431", "44090", "11383", "48141", "40686", "22200", "19005", "67561", "24187", "70903", "33252", "23868", "47054", "72294", "50410", "36700", "35662", "50373", "61283", "32086", "43127", "39944", "49826", "31326", "29242", "17355", "4798", "54217", "69742", "64533", "64361", "50935", "20884", "10088", "11840", "53239", "1170", "33186", "65211", "29678", "57930", "53754", "47197", "8388", "21428", "54231", "47069", "17407", "32409", "53882", "38756", "18475", "58576", "22155", "71272", "24931", "61570", "24317", "10284", "69669", "4533", "48330", "24064", "13576", "31242", "41718", "51857", "50268", "71012", "9331", "6709", "61540", "70919", "34857", "23861", "5002", "39237", "36455", "25777", "47512", "48362", "64405", "25298", "35175", "4551", "15010", "73140", "49530", "40456", "44880", "56934", "61209", "6301", "39255", "22672", "52048", "11307", "3109", "33522", "29155", "72363", "56672", "12713", "65298", "27632", "17184", "52918", "72686", "34463", "32054", "52260", "14148", "35223", "18434", "40779", "61278", "5683", "24092", "20902", "17819", "7973", "8258", "5735", "42097", "54609", "64545", "57803", "11410", "28384", "22212", "2653", "36409", "43941", "26199", "19764", "24866", "23603", "3327", "7997", "57640", "39249", "53174", "18752", "21661", "23988", "43092", "63248", "13984", "2352", "39801", "69164", "60132", "20237", "72465", "39349", "69007", "23069", "59737", "4407", "29501", "34969", "25688", "54212", "49310", "42143", "26685", "45928", "20542", "11215", "16421", "67804", "9428", "65672", "2296", "69063", "26383", "12852", "51861", "25451", "67503", "37424", "39404", "50522", "57893", "32156", "54143", "2684", "60938", "70192", "41281", "14324", "3906", "16284", "57199", "48879", "6592", "47182", "52534", "31274", "66484", "28380", "37384", "34152", "32632", "60740", "57336", "11847", "15836", "30119", "12329", "4008", "36649", "42517", "36223", "69737", "18660", "1092", "53146", "36601", "4773", "37498", "29736", "50432", "69470", "24204", "13151", "46892", "34035", "58890", "48349", "39431", "52444", "25975", "18455", "28482", "36132", "11105", "35055", "22153", "7852", "3518", "40875", "44871", "3304", "14143", "66130", "8595", "64403", "46142", "66070", "25084", "29648", "54279", "40009", "51034", "19645", "69163", "70152", "16696", "11328", "26749", "13142", "22571", "10159", "65641", "10061", "70569", "5296", "43406", "17232", "64602", "61031", "23263", "4738", "48314", "8494", "16687", "28906", "28323", "15350", "66202", "31235", "60299", "25762", "19732", "53383", "40938", "10832", "57659", "45642", "39415", "36168", "71067", "2030", "36190", "21655", "17864", "19090", "62325", "9093", "26756", "50825", "54282", "3551", "57859", "798", "58555", "68441", "55191", "57141", "45904", "31950", "29039", "5326", "41129", "68185", "62660", "3457", "68565", "66585", "69370", "28463", "567", "56834", "32479", "72639", "5985", "63226", "2164", "50429", "44158", "30494", "10410", "24342", "8768", "16856", "22188", "58795", "1633", "45359", "1903", "39924", "38824", "6781", "4121", "7243", "60282", "66574", "1500", "55910", "58537", "55538", "53444", "29101", "13102", "48468", "70722", "14564", "50574", "15292", "72868", "36625", "29767", "39600", "30548", "17549", "40439", "29604", "71929", "25933", "11843", "23503", "1551", "6956", "70937", "53796", "16030", "54223", "27776", "992", "63811", "3721", "52219", "635", "60435", "43814", "35457", "17757", "33127", "32964", "44316", "34242", "5055", "73044", "29591", "66027", "54339", "3340", "70087", "53286", "1003", "49231", "54511", "52348", "39323", "66587", "72552", "59567", "42960", "30582", "5890", "53304", "29035", "34733", "21720", "66226", "55742", "47708", "44022", "32730", "8689", "50909", "1476", "19738", "2768", "35014", "33236", "19882", "57574", "59316", "16605", "37680", "43018", "19588", "33427", "7798", "68015", "38734", "11710", "4384", "4726", "34368", "27885", "20868", "70914", "2420", "15651", "33191", "8660", "59138", "11717", "52802", "41834", "45449", "7665", "49403", "6940", "27195", "67025", "17099", "42206", "63726", "24427", "656", "15913", "61524", "27793", "68793", "35854", "62033", "12002", "52894", "61320", "39823", "67515", "68245", "11752", "7738", "8163", "61658", "70876", "7120", "20206", "17276", "55945", "40484", "66123", "27735", "12275", "47862", "65898", "55757", "68632", "5833", "29274", "23422", "12868", "4522", "10310", "68873", "53179", "47163", "52615", "8581", "67082", "37303", "13400", "1404", "41374", "32896", "71373", "1957", "19165", "23036", "9328", "69206", "7922", "57322", "39130", "9761", "51605", "65604", "24400", "41294", "16224", "4515", "69474", "54862", "7321", "3713", "47997", "36001", "34847", "21804", "10400", "44144", "28685", "55486", "71708", "60453", "21241", "23479", "61100", "69566", "22535", "55484", "67572", "3002", "37706", "50491", "14493", "5210", "54699", "13773", "17577", "23722", "56288", "54204", "63903", "31118", "55615", "6030", "47471", "71688", "60970", "60691", "7271", "70819", "21670", "7569", "41514", "17765", "44040", "22970", "20560", "59572", "66752", "57670", "18046", "37840", "42803", "39153", "64246", "41651", "68338", "5743", "30399", "24588", "11106", "44761", "9977", "52702", "40096", "52804", "8017", "68584", "58941", "42152", "30900", "61749", "5829", "47662", "37302", "9146", "53221", "38103", "8703", "37618", "21690", "14478", "22543", "7825", "6292", "46214", "18670", "43030", "36742", "39992", "14978", "29218", "12886", "47451", "13149", "42669", "57599", "14601", "59048", "50203", "43067", "55946", "27934", "45659", "7678", "12471", "26344", "46149", "17777", "28628", "44163", "46114", "42712", "4834", "14450", "66078", "52594", "43872", "13362", "16951", "9968", "11020", "11604", "2892", "33808", "2884", "13053", "71154", "67931", "54878", "514", "5152", "1727", "42051", "54275", "15573", "33285", "2961", "29748", "66385", "22439", "32913", "50006", "41577", "22695", "6451", "55210", "59604", "39467", "59201", "27272", "71802", "4299", "9440", "59366", "21726", "73226", "7390", "58101", "39472", "69562", "16216", "54365", "63063", "30542", "50980", "58509", "17638", "32206", "12555", "26297", "69933", "44913", "23949", "33361", "63793", "69013", "54097", "23177", "30549", "28844", "72056", "67909", "55224", "61099", "58427", "45797", "27159", "62108", "2656", "6223", "23328", "50551", "8930", "61499", "2230", "32133", "13711", "41898", "19713", "53431", "27906", "70962", "15012", "6210", "68742", "4954", "63849", "5401", "34234", "9275", "54465", "44349", "39055", "47425", "15059", "67557", "52946", "23050", "11633", "29309", "44279", "31328", "54939", "18294", "8284", "36904", "13098", "226", "65051", "39371", "13056", "23624", "60943", "68340", "20198", "7630", "43539", "67744", "59237", "38611", "51310", "10223", "60251", "43993", "33394", "23185", "71445", "36858", "95", "50215", "17876", "25556", "9159", "18040", "50371", "60486", "64274", "14242", "2468", "59140", "5212", "47532", "27142", "9204", "4003", "69188", "53557", "17268", "2644", "43482", "9580", "50492", "22237", "12974", "27008", "69224", "72740", "52284", "8130", "50949", "66076", "49479", "11556", "50124", "43375", "274", "45911", "9909", "27595", "44653", "46311", "7955", "65759", "26401", "35115", "68688", "57564", "63777", "3158", "2866", "68380", "33161", "54379", "38024", "1045", "31671", "13244", "5155", "37948", "35216", "4474", "19533", "44926", "27051", "64620", "60823", "60611", "22093", "19350", "61724", "11072", "30622", "42309", "16681", "29827", "62598", "61719", "46412", "53574", "51889", "70428", "43291", "58547", "34946", "51683", "49046", "18447", "939", "49499", "59154", "27646", "72160", "43026", "69757", "15787", "45638", "61863", "47268", "25125", "66681", "54397", "35908", "6025", "33566", "21672", "29691", "58628", "17096", "17783", "71363", "72198", "24242", "67202", "20366", "35885", "23538", "35523", "16762", "25073", "37749", "69584", "67118", "4487", "48722", "8817", "30776", "21594", "14236", "18229", "20510", "5584", "7310", "65723", "44200", "16772", "48234", "41212", "61971", "38215", "48768", "11859", "57891", "29338", "59302", "67807", "71441", "53927", "66818", "22391", "10297", "305", "5882", "26239", "57626", "43841", "3105", "38501", "68944", "17974", "4590", "58343", "2313", "33850", "51287", "54175", "33380", "64399", "12778", "38370", "26645", "55192", "13518", "32147", "39577", "70400", "14144", "35830", "14379", "35958", "72154", "15519", "1973", "48533", "27137", "16298", "60452", "44808", "50047", "2152", "46886", "39900", "63633", "42013", "61465", "9355", "15525", "41404", "25028", "18542", "30953", "32789", "49078", "39168", "31990", "64427", "64043", "33558", "28289", "40915", "43011", "11488", "42191", "68729", "6974", "2082", "9058", "45178", "69138", "30790", "31588", "63459", "57811", "25369", "15318", "50863", "73239", "67444", "14550", "2106", "47373", "4442", "32839", "32954", "33777", "63436", "47995", "61362", "11728", "12134", "66562", "23557", "45769", "7104", "39663", "8422", "25974", "69137", "59050", "8628", "54444", "62400", "6511", "42078", "6350", "19262", "29259", "61198", "6939", "39278", "56350", "18477", "3444", "4880", "53202", "24720", "31958", "45473", "52223", "17737", "10896", "2595", "23833", "67738", "17250", "11386", "25798", "44821", "59247", "65490", "60005", "46740", "61773", "45809", "9358", "70705", "44076", "19018", "62030", "18069", "51521", "68433", "68861", "55438", "37132", "3285", "72636", "45780", "34849", "23277", "61786", "39077", "1762", "67687", "6777", "8851", "25715", "16733", "63976", "32272", "41978", "8830", "23961", "72494", "43575", "9865", "15540", "11972", "44378", "32747", "61857", "41592", "40573", "8687", "13385", "1124", "9891", "62443", "17921", "13133", "42849", "58498", "46154", "2649", "67648", "25768", "33369", "54252", "52549", "55417", "7929", "14944", "66381", "36278", "63017", "11786", "43561", "50541", "1465", "32251", "70459", "64218", "56323", "17450", "9466", "65745", "39125", "9222", "15139", "54352", "58573", "59379", "63468", "43674", "32772", "38466", "68266", "47343", "43548", "45833", "66017", "3117", "30441", "59627", "42219", "50426", "59303", "56924", "25944", "33981", "10292", "42589", "34620", "44345", "19620", "38454", "19615", "12289", "32742", "14068", "51174", "2374", "30595", "51929", "66314", "70860", "13156", "45974", "66380", "33633", "4845", "55845", "38975", "41549", "7423", "44777", "52211", "26915", "69574", "20877", "51393", "47477", "7887", "30111", "45208", "67110", "71987", "16881", "23098", "13039", "12314", "32576", "65135", "67937", "19058", "73090", "62640", "19718", "37119", "42945", "48114", "7565", "3600", "8657", "57477", "6728", "46811", "57275", "14113", "44014", "35537", "43276", "23549", "36906", "36387", "28568", "8792", "71846", "33026", "32267", "43076", "43616", "23087", "64208", "52603", "60596", "43481", "27211", "18243", "12850", "18619", "19080", "3644", "29322", "58361", "41907", "6478", "5193", "62840", "61266", "49389", "60487", "24867", "66489", "28439", "15199", "32523", "56956", "71416", "71181", "63980", "42130", "40383", "25354", "69101", "24677", "53554", "10236", "42052", "32294", "57542", "54126", "21226", "20855", "64524", "47017", "12907", "71517", "47527", "13542", "17540", "64847", "70785", "52908", "61380", "62618", "51565", "33845", "17594", "10440", "52051", "57489", "28449", "34972", "8653", "43978", "36829", "4869", "32889", "49339", "29714", "42568", "24544", "49914", "53663", "58000", "63158", "33679", "61738", "42336", "19871", "44861", "66003", "67700", "51406", "8094", "34479", "35930", "39892", "64656", "8781", "38135", "37582", "36715", "48567", "72977", "26580", "31128", "9724", "51790", "36783", "20703", "59404", "14699", "29405", "70441", "38073", "8147", "12702", "19495", "25063", "62974", "44843", "29184", "26309", "51811", "54354", "15294", "48815", "63066", "9544", "52192", "56451", "14747", "31262", "9194", "62059", "52677", "45187", "29477", "33739", "16821", "34803", "57894", "17342", "72725", "64907", "70346", "30980", "29118", "61800", "2174", "58310", "38568", "33177", "36033", "30140", "55426", "46534", "51044", "30318", "42686", "5064", "24313", "45882", "34790", "27588", "11535", "70141", "66521", "26100", "35578", "12672", "61745", "21677", "2087", "29372", "60094", "21957", "26363", "24094", "64406", "20616", "26124", "37795", "39092", "62083", "49065", "66796", "52078", "1175", "46124", "1063", "37150", "10632", "43087", "17086", "3069", "43883", "13686", "48434", "47412", "69343", "42440", "1118", "63186", "1294", "40139", "31787", "54958", "57131", "68802", "60537", "61418", "24639", "2194", "31559", "9993", "43800", "29887", "49984", "27487", "67558", "9528", "59441", "12693", "16458", "12783", "56092", "36048", "41585", "72689", "6041", "34439", "28928", "10178", "8392", "72650", "33579", "16397", "68480", "56540", "47921", "15135", "72819", "69706", "6715", "8493", "38636", "71775", "53899", "34786", "35514", "8396", "46633", "25626", "44712", "27410", "41871", "7183", "62352", "59935", "34937", "70176", "60366", "69397", "66194", "46694", "968", "14302", "10175", "28417", "63582", "51732", "28677", "38064", "8236", "34005", "26240", "30836", "48092", "64614", "34297", "11180", "37021", "6192", "70201", "2999", "16639", "52991", "9423", "14050", "54443", "8836", "443", "10669", "7456", "28967", "69946", "68217", "48466", "10799", "46096", "6387", "19977", "58642", "65428", "68277", "54510", "52146", "21984", "71213", "34416", "26980", "61414", "70780", "22062", "50017", "17923", "63942", "34989", "39729", "22336", "51040", "4745", "16053", "67232", "35893", "49920", "71231", "15274", "30688", "69316", "55193", "24302", "52297", "49877", "57383", "42846", "19916", "11690", "28884", "40651", "17919", "21647", "25979", "10000", "28293", "28078", "6900", "9653", "14987", "27672", "54931", "42532", "53069", "35506", "23850", "3656", "1155", "2366", "13200", "42994", "62055", "54874", "16417", "25960", "57884", "64623", "43282", "60977", "62826", "49597", "56510", "72922", "7346", "45665", "3077", "51595", "67177", "4776", "52902", "69350", "17172", "37836", "45228", "15147", "19545", "10893", "63581", "66975", "18221", "30169", "21368", "66275", "23102", "31081", "4874", "4856", "19126", "62629", "60153", "45280", "69430", "35806", "55172", "15782", "19171", "22317", "10238", "25238", "30416", "64650", "41342", "20409", "59165", "72075", "47164", "41380", "66750", "3030", "37096", "1867", "64997", "59388", "29822", "12739", "35417", "6132", "72958", "10421", "19242", "24280", "16198", "57164", "31922", "35573", "28740", "66644", "49733", "14104", "36115", "70934", "36283", "44666", "49894", "50578", "55919", "9232", "49794", "1631", "32930", "4417", "66592", "12474", "2124", "27744", "24555", "69008", "2061", "72306", "67260", "6171", "52140", "50144", "49617", "37844", "57899", "69207", "62726", "32015", "21417", "45085", "40278", "18821", "9148", "22623", "26871", "4061", "40895", "69972", "36309", "4756", "68038", "38445", "27721", "66868", "10108", "39234", "327", "39109", "68807", "34676", "52255", "72967", "45648", "24398", "21843", "49256", "34985", "29995", "42769", "29724", "57258", "38540", "40326", "66620", "48374", "47187", "33509", "36504", "11021", "37415", "72382", "59391", "28774", "30055", "3936", "6462", "18735", "46377", "60600", "24804", "34007", "25856", "26328", "53220", "69781", "699", "4310", "21257", "10309", "10916", "45745", "52473", "23401", "39698", "59535", "37884", "22402", "21873", "50629", "66880", "70605", "36987", "43077", "12985", "645", "36150", "63730", "15138", "58362", "38069", "13937", "48748", "73256", "66200", "68780", "29926", "1839", "2495", "39344", "67442", "62915", "25495", "57278", "65018", "38000", "61761", "39501", "64639", "40229", "14579", "24320", "39", "8139", "21680", "63093", "49162", "38421", "31894", "14711", "29445", "4968", "61466", "43128", "15991", "8239", "35795", "2554", "10472", "62124", "59224", "37005", "43251", "59069", "12996", "37672", "31092", "51596", "39552", "29441", "3594", "2050", "53472", "13854", "18975", "50735", "49820", "11978", "46110", "27364", "22008", "2358", "44075", "32060", "11615", "66037", "41597", "31906", "32972", "64431", "37623", "23642", "25371", "16780", "57618", "18979", "69802", "19683", "66555", "70001", "36332", "35074", "2462", "52485", "2655", "61783", "57816", "56312", "40845", "47616", "40844", "13780", "5239", "44421", "68751", "23500", "14033", "2107", "47455", "43679", "33617", "61829", "31152", "61592", "8785", "21752", "2924", "54068", "37014", "60391", "62897", "34162", "10163", "67795", "8714", "15786", "370", "33286", "66166", "41135", "22978", "11233", "15626", "46882", "56813", "27099", "8400", "36751", "53820", "20169", "6590", "6717", "47078", "59337", "35009", "41512", "45731", "4974", "9040", "12651", "12484", "29055", "36955", "32057", "3537", "16839", "68942", "36755", "60344", "71502", "34030", "21002", "69296", "54696", "11516", "40680", "31677", "51881", "50166", "38179", "55405", "63124", "5859", "68363", "20750", "34677", "918", "65179", "29138", "31185", "38020", "33476", "45819", "23775", "14260", "43386", "44783", "58639", "19059", "36642", "57764", "57518", "7407", "24865", "7587", "31018", "32843", "9165", "9872", "52891", "62163", "18117", "58485", "51506", "62548", "51224", "68821", "1977", "20326", "70708", "26318", "13823", "70470", "33194", "35380", "19402", "34540", "10155", "39759", "59220", "71437", "566", "32709", "22923", "1072", "31203", "29490", "9641", "58480", "40141", "23397", "8467", "20660", "45489", "70688", "22463", "24006", "63359", "47805", "57878", "21870", "50622", "59162", "25233", "25012", "16629", "3578", "26443", "40524", "51796", "65131", "69333", "60376", "3489", "51056", "48899", "33705", "33790", "62946", "25502", "11705", "73157", "42326", "4394", "62806", "42954", "55531", "40584", "12282", "16233", "60398", "70105", "24933", "52564", "25165", "11901", "43132", "35208", "37076", "1692", "1784", "56078", "55898", "18531", "47479", "34567", "62203", "59331", "18490", "60734", "61300", "6869", "25517", "41976", "12286", "11588", "688", "3646", "67334", "56169", "8763", "26628", "73230", "23714", "18199", "47419", "70437", "42782", "57008", "48430", "1443", "37661", "68375", "57651", "16182", "44759", "36945", "9547", "37389", "27291", "70971", "5770", "24906", "45053", "40106", "37994", "38314", "67374", "45477", "25731", "45474", "68785", "54688", "32062", "32143", "54176", "49324", "67071", "64711", "72023", "53517", "53208", "53961", "62274", "38556", "8119", "61855", "9517", "3143", "69366", "16315", "57749", "43181", "49673", "64756", "59144", "41676", "13314", "35781", "37316", "26352", "69036", "55947", "17160", "794", "4809", "56154", "27737", "30338", "68762", "12601", "16875", "34951", "20645", "3503", "53892", "48215", "68735", "59786", "36546", "44093", "590", "40506", "26998", "18454", "11970", "62975", "2362", "70582", "21460", "45736", "30616", "71490", "57791", "11494", "25200", "71393", "37999", "66699", "59750", "47583", "32426", "38546", "62622", "19513", "67751", "21616", "30457", "47463", "33759", "15994", "35728", "15773", "52435", "71083", "14696", "19385", "15893", "48449", "49157", "37951", "61979", "33020", "43271", "19187", "43940", "18230", "70959", "14384", "21430", "9536", "16548", "65302", "36843", "15325", "48958", "29709", "13796", "42760", "67577", "11889", "38071", "20199", "61994", "64122", "36220", "68747", "67192", "1988", "69288", "38718", "45046", "11466", "56426", "9150", "20588", "31301", "1164", "25877", "52707", "25448", "13964", "56132", "38083", "63392", "2597", "70349", "10481", "49079", "65504", "44732", "25075", "1296", "70393", "50580", "39499", "6802", "9249", "68180", "54693", "25227", "25174", "16133", "6151", "56949", "67264", "17856", "52831", "49100", "71714", "46642", "65890", "12896", "61502", "22955", "60824", "47651", "51589", "50462", "49197", "51262", "18840", "63955", "34194", "39082", "45047", "9097", "12628", "36567", "69919", "21293", "69265", "16880", "29231", "52787", "40265", "38679", "2713", "46190", "32347", "24538", "46342", "18099", "64740", "29002", "47420", "59994", "64071", "69822", "7234", "7418", "20749", "65069", "27429", "24812", "30747", "67762", "57774", "26121", "4336", "30656", "63166", "19655", "71338", "14725", "71720", "48333", "39611", "68187", "51906", "13751", "47328", "37774", "27801", "30633", "64131", "11448", "65037", "61307", "10489", "13260", "1984", "24166", "64654", "47038", "61338", "61886", "36720", "21942", "28716", "8182", "3290", "48422", "61329", "15183", "410", "66067", "70908", "57443", "49759", "24923", "12522", "71068", "65559", "47480", "53173", "45791", "66738", "52389", "35056", "55361", "2440", "37400", "52704", "49774", "21520", "37746", "59266", "8305", "36675", "50133", "20190", "62028", "1139", "30191", "45600", "68095", "63276", "65881", "63274", "56841", "50919", "38459", "13310", "40187", "58166", "29149", "50957", "3730", "4611", "11241", "38976", "66957", "20456", "33960", "30838", "3116", "11238", "60579", "10383", "26500", "17411", "50687", "55101", "2870", "63371", "13302", "26599", "64833", "53626", "68613", "68745", "69187", "54360", "19520", "51002", "13472", "3336", "21128", "17263", "61450", "29190", "62771", "67079", "25184", "18984", "43856", "69834", "49341", "4679", "1436", "20938", "62880", "33039", "66611", "59592", "51031", "55566", "22362", "52864", "16447", "72474", "34933", "10977", "9691", "6045", "61373", "59041", "43687", "41859", "57984", "38414", "29018", "35595", "12229", "57167", "12960", "3522", "34049", "12883", "1934", "7118", "66867", "45733", "9004", "32120", "23229", "54540", "25074", "4212", "48580", "8743", "65650", "43158", "9584", "6321", "2904", "37550", "19465", "2782", "55888", "17976", "12870", "38609", "49082", "29223", "45944", "44508", "452", "67247", "1419", "34837", "33583", "39295", "58230", "35192", "69858", "23782", "63347", "51779", "52754", "9102", "4740", "66238", "51935", "46733", "34056", "54297", "10836", "29098", "66573", "45682", "13187", "22398", "66032", "3113", "62759", "70361", "12704", "7672", "50115", "41076", "70806", "57004", "41715", "9161", "23421", "59498", "49248", "14905", "39679", "4416", "19044", "19592", "30472", "71617", "62003", "49882", "15374", "42423", "68464", "20338", "26267", "71588", "49933", "5000", "29449", "53211", "29603", "44611", "21172", "43436", "67651", "67290", "72305", "20846", "53724", "17584", "28427", "25509", "2996", "17370", "51052", "64137", "21955", "67785", "39046", "11283", "4532", "11261", "28938", "40605", "45", "48425", "26552", "63268", "21830", "39465", "37122", "62913", "67362", "40298", "42004", "52006", "72907", "58579", "26987", "42481", "21285", "49149", "45776", "41220", "33015", "44450", "35266", "70472", "27280", "57034", "58954", "49762", "40040", "45112", "28671", "45768", "60424", "16246", "40478", "44911", "13071", "44178", "58419", "66088", "59511", "28964", "63829", "57091", "70991", "42886", "31384", "61041", "70399", "2006", "1390", "28982", "31748", "14240", "70160", "34691", "59018", "16999", "72654", "10904", "61328", "212", "64305", "28544", "35705", "4995", "27796", "30840", "3408", "1076", "4375", "40835", "43041", "44963", "67379", "9704", "40854", "31343", "35325", "34376", "9699", "32401", "60933", "57373", "32698", "24111", "22018", "35190", "27905", "53485", "32287", "39997", "12898", "25778", "51405", "24249", "35484", "20306", "36449", "61852", "39198", "38920", "27889", "59304", "30766", "9265", "58569", "16213", "29011", "28570", "69201", "23311", "63966", "35536", "66939", "15390", "36724", "24530", "2863", "12092", "45805", "59083", "44174", "55738", "27874", "48139", "4322", "59814", "73074", "20807", "66278", "43319", "42640", "71387", "37056", "72896", "17740", "55840", "58382", "53779", "65191", "41720", "35887", "70694", "35805", "61429", "41706", "36780", "52298", "18003", "38535", "2395", "70150", "65576", "22192", "50136", "65350", "20835", "4752", "26921", "27034", "11724", "34045", "34976", "71008", "66744", "10438", "45167", "11367", "45618", "19915", "15361", "58924", "21915", "11578", "45951", "19961", "44452", "66622", "26624", "31697", "61547", "32448", "36604", "54635", "32544", "28220", "9253", "72748", "13626", "43809", "40042", "73131", "59305", "15087", "14679", "28977", "20439", "63799", "26246", "1052", "43933", "761", "11433", "46242", "30154", "30162", "34135", "56964", "69404", "40975", "8301", "38079", "33456", "57442", "68487", "51226", "34735", "70843", "46797", "36503", "40622", "73237", "42661", "37274", "24345", "6275", "39509", "63740", "70185", "5287", "26142", "13660", "63806", "57197", "5901", "59287", "40696", "45811", "57500", "24481", "10379", "60559", "64949", "619", "29455", "5693", "5949", "9886", "47749", "12970", "70170", "27322", "41286", "34926", "62768", "37190", "18511", "25854", "39454", "13245", "10329", "22460", "37812", "55145", "52920", "32788", "58960", "62232", "18808", "60118", "42519", "54648", "71300", "33718", "55200", "54114", "9362", "7094", "49644", "46968", "3798", "42830", "72685", "19295", "31821", "35292", "44355", "52422", "19032", "63474", "50262", "31583", "16708", "70395", "57427", "42833", "30371", "60921", "66819", "69411", "26706", "49382", "25561", "70482", "48488", "65642", "33713", "32721", "27271", "66115", "40969", "42429", "55072", "37048", "3015", "9381", "4261", "42825", "36470", "47526", "17446", "45201", "51418", "67839", "16941", "71431", "37579", "15740", "56512", "27404", "1375", "27306", "67977", "63039", "32455", "29296", "39930", "26864", "51728", "33058", "7545", "17823", "23843", "3001", "16825", "29151", "15744", "54208", "51217", "52381", "12580", "68050", "68732", "39387", "46442", "44411", "70168", "23206", "34391", "61166", "64291", "30317", "16227", "28206", "43213", "7358", "24612", "30533", "18055", "39910", "23828", "53085", "36872", "25759", "28336", "59960", "26916", "44318", "3099", "3035", "60086", "4909", "47981", "17862", "65697", "5666", "4868", "56251", "671", "56800", "61018", "604", "25137", "47728", "22847", "42373", "31732", "47418", "17541", "24956", "19789", "45368", "54778", "35704", "13564", "47815", "55064", "69633", "24785", "31561", "16190", "652", "69567", "52226", "58964", "52986", "30213", "45723", "53229", "23250", "5400", "58908", "53878", "37380", "49844", "72573", "43257", "65104", "4127", "50414", "11304", "22838", "13215", "27984", "8902", "38422", "14867", "58149", "27730", "22594", "64284", "68112", "50335", "58556", "22832", "56890", "62765", "46042", "28528", "60295", "31713", "27931", "8736", "63890", "36209", "17045", "27134", "31341", "68956", "7803", "29409", "64990", "26107", "2365", "257", "61053", "14555", "41965", "49698", "7265", "45528", "1248", "848", "4709", "49622", "67103", "14099", "64229", "28058", "39490", "40268", "52028", "44660", "70824", "17499", "35620", "32486", "3382", "25852", "46316", "59566", "71615", "20597", "40943", "1102", "29731", "50830", "38831", "17846", "59657", "38346", "63233", "72312", "63764", "57546", "15328", "42065", "45501", "68850", "52579", "18207", "50305", "39126", "13690", "14623", "43416", "69530", "70397", "69646", "1874", "70906", "7248", "58903", "50536", "47110", "20837", "10829", "12858", "54607", "60114", "37053", "23137", "61647", "35035", "47811", "5610", "27064", "58229", "24224", "54572", "4971", "18065", "24735", "4343", "2283", "42019", "20452", "63868", "47294", "67547", "48945", "6587", "20533", "11342", "49514", "17852", "31682", "44394", "23007", "58837", "50524", "6056", "47993", "28317", "54964", "13224", "40084", "3874", "66710", "22016", "39507", "29564", "50392", "23518", "59851", "67294", "19597", "54431", "72564", "26945", "15429", "11431", "48129", "47802", "25493", "57908", "68269", "19950", "24597", "45817", "26744", "50057", "69278", "26270", "61149", "48599", "23921", "51549", "34538", "58292", "29520", "61202", "17399", "52481", "7959", "49547", "38594", "35724", "52020", "47559", "12631", "44397", "62415", "29775", "60031", "66671", "43334", "4369", "17910", "58203", "44642", "54881", "39267", "61928", "45211", "15441", "5171", "13900", "34307", "37156", "70106", "27680", "7063", "18690", "64709", "52773", "29358", "34070", "60349", "54821", "42418", "14454", "15073", "11023", "23295", "57087", "61145", "33490", "3334", "265", "19484", "18999", "46135", "52500", "9815", "66845", "10792", "12029", "33819", "21731", "19864", "64290", "59057", "9742", "49260", "61074", "11336", "49772", "53176", "48503", "16071", "72197", "13889", "7312", "55116", "48823", "18627", "19399", "16686", "23522", "17895", "31785", "62661", "59830", "7434", "71028", "44308", "62855", "52197", "56880", "45235", "40015", "38781", "34360", "44628", "35086", "26722", "29560", "62807", "11275", "53295", "46281", "13678", "20412", "58793", "8053", "58460", "13185", "42270", "7422", "27112", "38077", "68615", "42970", "57723", "26412", "60902", "9899", "7071", "27833", "33625", "1779", "67255", "2013", "2767", "23322", "11078", "69702", "18591", "63948", "64273", "46423", "48091", "72805", "31856", "49113", "62169", "43130", "46470", "31793", "60593", "33686", "40612", "33494", "8885", "47382", "64603", "72329", "31327", "55035", "69819", "65941", "1617", "12983", "14336", "7263", "17946", "44185", "24667", "46056", "27933", "47972", "6262", "60381", "43597", "43365", "17054", "64885", "43094", "64057", "42109", "5822", "60577", "70535", "25551", "54463", "66437", "53737", "35772", "65473", "44184", "24965", "24520", "20088", "5991", "30749", "37264", "21500", "62725", "55739", "32976", "51624", "19517", "30964", "17922", "72397", "28277", "59502", "11174", "49843", "60414", "52790", "25596", "1163", "54325", "63204", "19202", "47959", "67880", "49721", "45654", "10343", "25943", "25710", "71967", "54340", "29156", "56472", "34147", "12805", "37882", "45687", "21926", "13874", "15601", "52205", "19839", "47894", "396", "54399", "15641", "30925", "58667", "12241", "2605", "70111", "28117", "25755", "1766", "22766", "54257", "51622", "61177", "30707", "71092", "72324", "11642", "39575", "64686", "30740", "18068", "38915", "63303", "19151", "72203", "2064", "6335", "54564", "41970", "22753", "20499", "66737", "13024", "1694", "30269", "40430", "5765", "23286", "34507", "49043", "70574", "13624", "7323", "40729", "63515", "157", "49663", "20581", "10599", "60621", "20410", "50194", "45499", "53627", "6647", "28282", "28875", "14151", "20313", "53913", "7620", "15575", "2872", "12386", "47565", "5003", "4921", "45175", "64443", "16045", "56036", "15930", "1793", "67965", "70584", "4340", "29526", "39174", "68967", "19125", "37386", "40271", "34949", "51433", "14517", "48369", "52321", "51582", "1559", "47009", "48081", "29414", "11218", "354", "44872", "58963", "4005", "45297", "45779", "54478", "58875", "24191", "6157", "46550", "65331", "34457", "34334", "54845", "46347", "58572", "18136", "59516", "59231", "19160", "68037", "24282", "31094", "53751", "30895", "23231", "54608", "33518", "56339", "45679", "7827", "52353", "44807", "40998", "24589", "21480", "44519", "43264", "57410", "2620", "4291", "57469", "16537", "37630", "53327", "22523", "42616", "17953", "42908", "11861", "38417", "57123", "32398", "21356", "10211", "54317", "782", "44550", "61957", "61265", "41169", "11576", "32402", "52492", "57611", "63883", "58606", "25905", "18095", "52897", "40066", "3839", "21847", "56346", "14258", "32355", "1149", "39388", "10138", "13941", "44804", "12081", "46212", "5877", "26098", "69619", "60191", "9652", "7856", "45785", "49636", "24079", "11269", "12181", "33040", "71806", "24566", "12674", "60941", "51300", "13860", "61212", "66254", "52238", "11802", "50344", "62972", "8789", "33004", "25350", "6147", "3022", "48424", "4560", "37288", "10160", "5881", "56095", "24012", "65661", "37770", "3625", "14858", "65732", "29053", "7475", "21057", "59920", "468", "62987", "46484", "35969", "27419", "17280", "39328", "46477", "65793", "12348", "20036", "45967", "65808", "11941", "35145", "16037", "44805", "59672", "46466", "44868", "18930", "44403", "24009", "8701", "1333", "1505", "45164", "17264", "52437", "56777", "67617", "53762", "63774", "5907", "70203", "6602", "16407", "2651", "23623", "23566", "21191", "64475", "18288", "51542", "17322", "43858", "4706", "10139", "5074", "38258", "55249", "16964", "58138", "56028", "19357", "35585", "1042", "37450", "51507", "33063", "43857", "54921", "14782", "57772", "14518", "96", "61516", "29263", "67735", "48295", "35349", "13640", "20963", "47473", "46570", "14038", "13315", "58459", "62599", "32178", "67989", "69738", "20399", "69197", "55434", "2550", "55654", "27365", "30261", "19062", "44955", "51428", "18604", "66119", "55835", "11400", "46840", "51230", "6829", "28949", "65326", "34332", "55187", "114", "30488", "50477", "67594", "61215", "36298", "298", "62460", "25416", "11107", "55994", "63020", "13582", "21200", "56353", "34466", "35720", "32379", "39867", "54261", "66689", "41973", "13116", "38750", "70617", "29158", "14490", "66897", "6095", "9789", "3341", "44063", "15635", "52505", "65573", "59720", "44424", "43665", "71281", "24473", "68890", "48158", "18009", "48235", "50565", "49370", "28405", "64587", "22394", "65294", "60830", "41216", "56793", "42232", "50703", "31261", "5411", "12508", "42627", "41249", "23705", "61714", "42447", "56716", "34428", "32131", "27398", "50848", "10573", "60839", "14953", "12625", "4296", "41468", "2804", "32551", "51171", "36086", "1717", "49853", "14977", "60836", "45472", "46944", "70509", "21397", "56107", "67315", "3183", "16685", "43554", "52531", "25299", "44674", "14159", "45742", "46533", "21785", "69121", "3479", "22271", "43620", "70281", "63647", "22342", "53112", "36428", "49779", "4450", "59751", "29605", "65482", "22786", "25457", "6377", "8418", "22893", "35739", "10469", "70702", "25861", "34570", "44604", "11574", "44427", "56253", "61884", "54860", "57910", "44720", "19710", "64878", "36076", "31825", "4053", "60459", "22203", "55858", "35813", "59215", "27873", "5270", "12741", "27620", "58425", "15655", "2685", "73043", "49269", "54591", "72821", "27548", "49602", "60124", "4482", "69132", "62893", "53802", "58615", "8100", "53608", "31643", "8806", "5232", "21232", "65448", "52652", "68698", "5867", "10526", "50781", "62256", "26811", "21278", "38903", "11362", "54168", "63456", "25258", "21436", "9090", "25057", "41879", "33525", "50464", "46584", "36949", "61426", "54868", "42630", "63956", "39277", "45418", "61700", "27899", "691", "39286", "67878", "26165", "14685", "866", "13962", "50223", "62613", "57219", "67841", "15622", "42426", "26253", "61406", "2480", "727", "42171", "43170", "18355", "18054", "11476", "3930", "4231", "49444", "884", "2312", "46371", "6571", "3359", "13217", "49556", "31461", "34198", "57995", "16436", "61241", "213", "33454", "38316", "71222", "30979", "39350", "66383", "6724", "3242", "21701", "50062", "17924", "957", "3913", "38439", "27097", "66834", "21862", "27020", "66563", "43449", "20404", "59329", "41680", "70335", "57615", "42917", "56585", "11036", "67114", "65419", "20822", "32179", "59715", "15020", "5057", "62764", "17566", "21871", "24134", "54996", "41001", "54460", "16276", "38217", "38656", "32480", "26348", "38203", "23799", "14431", "8073", "18684", "11182", "32311", "18908", "36505", "43151", "33290", "11791", "60396", "10131", "40801", "10355", "58020", "71996", "17085", "11435", "6012", "70732", "54130", "39352", "72216", "58087", "67301", "38486", "3554", "23448", "41985", "39803", "51942", "49525", "32822", "10911", "59040", "23568", "41766", "42922", "13759", "8096", "8250", "60237", "26634", "44744", "59248", "54960", "68743", "52761", "25081", "55616", "67049", "29974", "35628", "50775", "59352", "11703", "70253", "34710", "9166", "41409", "26613", "69506", "3991", "54902", "53855", "47202", "43366", "48323", "70355", "29160", "51177", "48929", "49536", "65393", "3626", "57040", "25633"]]} \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn-test-split.txt b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn-test-split.txt new file mode 100644 index 0000000..cf6f03a --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn-test-split.txt @@ -0,0 +1 @@ +{"xvalid": ["int", ["1246", "11333", "18215", "16573", "11824", "8232", "18430", "2300", "12226", "25530", "5172", "22313", "14664", "3818", "2732", "21375", "19847", "1503", "14079", "12895", "2191", "25948", "7669", "1347", "14410", "14879", "22067", "923", "1453", "68", "13869", "1101", "15135", "6268", "4623", "10778", "547", "14171", "4328", "14238", "16544", "11730", "25833", "4095", "8825", "18508", "20865", "15259", "15729", "17011", "1109", "17390", "1369", "955", "18091", "24099", "23831", "16605", "8686", "22869", "12326", "8864", "6078", "21807", "13948", "13241", "8668", "13376", "23599", "20438", "17971", "1002", "16123", "16565", "9972", "6791", "3924", "5258", "15696", "17183", "21803", "24673", "25918", "7008", "22118", "9522", "25299", "18305", "20514", "1884", "8215", "5112", "21783", "10101", "6232", "11968", "10644", "15623", "16554", "5029", "4575", "19641", "22174", "9597", "21797", "7074", "16855", "20095", "10455", "4742", "22735", "9836", "8498", "16893", "4982", "19800", "21337", "17769", "17802", "2192", "18816", "21664", "24684", "23303", "19575", "12455", "13030", "2065", "21915", "652", "9252", "3870", "20072", "17511", "7928", "21004", "4217", "8773", "22983", "21379", "19147", "22578", "14580", "22181", "15446", "16503", "22596", "7072", "16731", "1582", "22280", "4832", "2423", "3015", "5341", "7513", "8482", "2844", "2331", "19665", "3030", "23439", "21923", "5003", "1926", "4645", "20129", "15871", "13751", "3438", "5804", "14666", "5518", "3397", "24861", "11600", "17835", "22167", "642", "25357", "9025", "13762", "14481", "4638", "11309", "24108", "10692", "24993", "10554", "17259", "8823", "13313", "21407", "1860", "19988", "21409", "2386", "20016", "18395", "5538", "11236", "4640", "20491", "4944", "25110", "7567", "3967", "3241", "24925", "22286", "16050", "10236", "19861", "20165", "16518", "19589", "1968", "8416", "1459", "10777", "25848", "25267", "2689", "20526", "4027", "24765", "22333", "22815", "7280", "6163", "12292", "10797", "5125", "22977", "18871", "15004", "23083", "577", "18503", "1", "13239", "1229", "4106", "19007", "10500", "19282", "17025", "11299", "17837", "246", "1160", "21698", "11563", "13763", "16763", "13639", "11900", "8797", "6764", "25553", "1488", "1776", "20168", "4393", "1346", "4013", "5997", "1813", "1490", "18354", "7130", "11743", "16608", "12574", "16709", "10449", "15382", "20401", "13272", "20181", "781", "3257", "4894", "13048", "7908", "14775", "25656", "1685", "13953", "16843", "7062", "2375", "7598", "11136", "7536", "19390", "10690", "19987", "21884", "15146", "2208", "18025", "21723", "25001", "25723", "6317", "5128", "24830", "6558", "9440", "17543", "1772", "25856", "22763", "12403", "21151", "4829", "22612", "2108", "9288", "9334", "20550", "4761", "1169", "19033", "5781", "684", "7197", "5001", "12126", "19782", "12566", "18873", "1177", "11202", "2723", "12493", "12014", "966", "12506", "8115", "21321", "20059", "15790", "20101", "20472", "12452", "14641", "8004", "17861", "25457", "25755", "14114", "7497", "1754", "4447", "17411", "7016", "24192", "14337", "3235", "21791", "8050", "12105", "7963", "21493", "10949", "7466", "17227", "12090", "16178", "18944", "21730", "22275", "18370", "5066", "25263", "4487", "9989", "8633", "8553", "16886", "19904", "1819", "19108", "10590", "22135", "19747", "23517", "16046", "22071", "20467", "205", "13531", "20057", "2180", "9127", "20539", "20633", "21340", "13281", "2443", "10640", "22865", "10599", "2549", "15191", "16941", "2430", "25991", "11468", "15953", "24942", "21879", "9527", "12809", "19343", "10525", "5114", "3284", "6372", "5813", "1861", "3409", "12067", "17603", "16460", "17110", "3206", "23112", "16830", "23577", "16414", "12015", "3126", "5778", "6482", "5330", "17480", "12454", "22683", "6499", "22451", "9118", "12618", "5797", "24693", "6818", "18726", "3571", "23276", "7323", "14746", "15493", "17749", "25668", "4655", "9968", "5691", "17204", "364", "12502", "14531", "10071", "9020", "19865", "19130", "16740", "18850", "24766", "10884", "18157", "20236", "2278", "15733", "18086", "8055", "622", "7571", "22438", "338", "9948", "24750", "25155", "25252", "15564", "14930", "24479", "8839", "11374", "602", "19571", "19527", "23665", "3938", "5293", "5555", "22939", "15388", "20404", "11080", "2548", "7595", "18094", "23862", "14682", "21007", "17984", "6568", "8474", "17499", "7375", "10108", "22552", "18359", "7568", "18933", "11464", "9264", "22422", "21498", "19129", "7477", "9289", "45", "10330", "5086", "15402", "12462", "17452", "24461", "20138", "23322", "8299", "14329", "24210", "21510", "10507", "19126", "14566", "2912", "23226", "17004", "10982", "11987", "18351", "3943", "3052", "3977", "16773", "8910", "3524", "20262", "22603", "20187", "9604", "4955", "24985", "21507", "25262", "24770", "10065", "13411", "25506", "4875", "15027", "13583", "19467", "18585", "11651", "22362", "5061", "2658", "170", "18771", "7505", "11179", "17181", "16030", "14584", "23029", "19526", "20971", "5847", "3815", "16245", "20381", "17199", "25015", "18012", "12890", "12758", "12667", "17897", "22270", "18277", "18608", "9061", "13622", "25986", "9644", "13533", "19792", "3111", "2674", "1059", "7732", "12628", "12129", "7944", "9489", "460", "13179", "13038", "13912", "12271", "15874", "5765", "2305", "25806", "8343", "8844", "23367", "14963", "18187", "21184", "14575", "1455", "12133", "7459", "15828", "20403", "9520", "24254", "9549", "13446", "7228", "7575", "20007", "16696", "10742", "4176", "24119", "25440", "537", "4625", "13005", "11379", "12838", "18241", "5762", "8385", "14333", "216", "16200", "7925", "3314", "12996", "17845", "22075", "5525", "17279", "4212", "15625", "8325", "23209", "15983", "5542", "7129", "16266", "16511", "14014", "5988", "25212", "19490", "21608", "19412", "14881", "19271", "1898", "25085", "25422", "12879", "14023", "20529", "18239", "14353", "18117", "14982", "19156", "22854", "16439", "17742", "5134", "16176", "11877", "21588", "5906", "9449", "8152", "10857", "296", "8441", "20541", "8801", "3509", "11676", "23194", "14951", "704", "2484", "15459", "23648", "22048", "14673", "24940", "610", "13344", "11960", "7594", "17813", "23939", "9391", "19097", "14800", "3635", "3823", "1650", "6882", "7101", "22522", "24825", "13032", "3277", "23105", "848", "15722", "11691", "24365", "12889", "11969", "16187", "18437", "12119", "8576", "4412", "8651", "20061", "5575", "23790", "6498", "14236", "20238", "25317", "15455", "22094", "25148", "22154", "774", "25897", "22766", "14907", "5790", "11954", "6799", "3918", "17605", "16498", "10487", "24599", "799", "20027", "6747", "14836", "8794", "5594", "12051", "822", "1608", "16916", "4677", "9175", "16471", "12125", "19367", "2144", "22199", "21200", "1639", "17715", "12510", "14893", "11251", "9704", "18732", "7113", "18630", "24633", "17894", "17419", "13644", "17476", "12353", "5374", "15546", "23445", "7044", "7393", "20628", "18257", "5834", "25450", "9054", "14503", "9180", "11550", "1371", "5333", "22066", "12595", "1451", "1581", "18158", "4763", "3735", "8097", "18514", "13424", "15964", "12591", "13159", "4138", "21069", "18940", "24334", "10106", "18122", "1747", "20773", "2916", "2156", "1969", "21828", "431", "5389", "10035", "21820", "21847", "3266", "22535", "8352", "12481", "7692", "4064", "1215", "17641", "1803", "24276", "11835", "9680", "21272", "10704", "18786", "20933", "12358", "24413", "24209", "17915", "6806", "26000", "12690", "17337", "11303", "172", "6977", "23889", "6206", "17159", "8636", "14298", "20036", "5960", "20989", "24241", "22594", "6913", "11796", "8246", "8102", "24754", "23582", "3216", "12501", "12173", "106", "5299", "989", "24817", "4282", "4287", "697", "10185", "14511", "23573", "7258", "15349", "15243", "21186", "19013", "1165", "18349", "26017", "15857", "1234", "11013", "8771", "1361", "6801", "9161", "10105", "3902", "23483", "8705", "17243", "21618", "23222", "8547", "5058", "4967", "10285", "11465", "24791", "6775", "14913", "6366", "241", "18473", "26003", "20442", "23434", "24010", "607", "16479", "605", "515", "11387", "21435", "12804", "2051", "13065", "12580", "15431", "16342", "4074", "10110", "13878", "25998", "14277", "7278", "850", "7086", "16669", "11718", "17266", "17298", "7721", "7647", "3659", "8453", "803", "14560", "11106", "2056", "16233", "21268", "24694", "18350", "24386", "383", "693", "7702", "240", "773", "7658", "9845", "511", "5354", "22487", "20420", "5833", "2933", "24284", "19410", "14771", "21261", "6837", "14384", "14544", "25693", "10844", "9516", "11250", "19361", "4179", "9024", "20873", "10947", "250", "7198", "5217", "8918", "25390", "15938", "8333", "8607", "10809", "14415", "21638", "1931", "21663", "20747", "15478", "14311", "5467", "3300", "12693", "18937", "23910", "20813", "22444", "15660", "6242", "396", "24086", "18108", "649", "20728", "6808", "25962", "13090", "21118", "11294", "9130", "6432", "15240", "23986", "12718", "24950", "21778", "2580", "25854", "7956", "16881", "1492", "10538", "3537", "23392", "22757", "12766", "8875", "10559", "13", "8879", "22994", "13076", "18374", "5097", "9187", "22321", "9209", "18162", "5010", "23755", "23675", "5639", "20156", "8631", "4743", "25129", "13068", "11815", "18562", "14901", "22662", "7205", "14369", "13785", "2905", "25882", "18050", "11939", "12286", "3790", "12509", "19956", "7427", "4131", "24990", "3301", "14792", "15281", "11484", "10006", "13936", "4508", "11338", "20906", "11660", "6449", "7175", "13790", "10024", "14573", "18754", "5701", "15409", "15031", "21817", "25244", "22478", "20012", "6776", "25853", "9041", "363", "5301", "25608", "11524", "6917", "6385", "2749", "21174", "6976", "1204", "7056", "25541", "22049", "22933", "20205", "18968", "21456", "10396", "8320", "17267", "9883", "23351", "23382", "2891", "7855", "8349", "7837", "11052", "5084", "14135", "18536", "7577", "6114", "21416", "21860", "14785", "25875", "10433", "4398", "1340", "17145", "24340", "17153", "23006", "13783", "17632", "17231", "8665", "23512", "10485", "1806", "9588", "7797", "24710", "12519", "1651", "14543", "6121", "3065", "2385", "12792", "28", "11416", "18211", "18135", "739", "8149", "4157", "17903", "14417", "14009", "9032", "18422", "1768", "4850", "11452", "10428", "21357", "11996", "10094", "4270", "6602", "6344", "11556", "4827", "17354", "14217", "18759", "16161", "13100", "6973", "8245", "20504", "4960", "4669", "24402", "5464", "13151", "8949", "11451", "17584", "11795", "21413", "1797", "12443", "3204", "2174", "20548", "15065", "3449", "9270", "21636", "1627", "8301", "10522", "7696", "4904", "2836", "10453", "15945", "21376", "12136", "4211", "23713", "3130", "13421", "9904", "11420", "5980", "11927", "2872", "22771", "9125", "12453", "20980", "20615", "18757", "15514", "13404", "22106", "14391", "9684", "1718", "16421", "7723", "18886", "21283", "21928", "14237", "2197", "19182", "23709", "13444", "15457", "18223", "9346", "907", "10457", "10361", "15171", "17280", "11178", "22264", "11698", "11339", "17471", "5600", "6409", "7359", "2688", "19092", "12421", "23272", "14474", "510", "13623", "8009", "23518", "22533", "10656", "23347", "24728", "11540", "7290", "11640", "14711", "23", "804", "16071", "20056", "22304", "23436", "1625", "15099", "10580", "17763", "16590", "10760", "18249", "8323", "5865", "8829", "16368", "6559", "4699", "5245", "7442", "14055", "16276", "17166", "4558", "1030", "11031", "10564", "2564", "20208", "9081", "25936", "9660", "5669", "7995", "25031", "19152", "6736", "17634", "10899", "17050", "13955", "21752", "14224", "17678", "15239", "22375", "10552", "24665", "882", "17737", "3889", "527", "11555", "10354", "10239", "10297", "1542", "8434", "18631", "8346", "13374", "5777", "12753", "4184", "16603", "16594", "15659", "976", "954", "6022", "19418", "6766", "3412", "1363", "24242", "25435", "18331", "6508", "1043", "11802", "24562", "16684", "21217", "16708", "1422", "195", "2477", "23664", "21173", "10839", "25322", "22343", "2289", "132", "17833", "1145", "16086", "2446", "2795", "2195", "10548", "19791", "17588", "15245", "4626", "17672", "25664", "1967", "16831", "8282", "1291", "998", "3382", "19173", "1961", "20478", "548", "3148", "5573", "13130", "1491", "17904", "20872", "19822", "10628", "23857", "17148", "15705", "4541", "3230", "8040", "6867", "7590", "15917", "17072", "12597", "23951", "11111", "23739", "18560", "24741", "1755", "24876", "10397", "22057", "6985", "19820", "2256", "20585", "22770", "12094", "21183", "18823", "3033", "10662", "4642", "3568", "12827", "17549", "8396", "25778", "24884", "13655", "11790", "3634", "22952", "13258", "24406", "3250", "16707", "18027", "9589", "13859", "5346", "1377", "11667", "13995", "18505", "3512", "18433", "25811", "13351", "9332", "15587", "11644", "8566", "8147", "7253", "2378", "8761", "8680", "12779", "3323", "12266", "19831", "13634", "8617", "23590", "4675", "16191", "6569", "5460", "4560", "21213", "13638", "12588", "10739", "2835", "7593", "12924", "18490", "25083", "20206", "3879", "16835", "11357", "24265", "1206", "15232", "8625", "9596", "633", "2743", "24713", "3705", "23464", "25290", "3387", "23360", "15406", "19631", "18285", "17240", "1950", "13963", "21304", "3937", "14801", "12517", "11523", "780", "13725", "10486", "22364", "24603", "11830", "18781", "19879", "16419", "22872", "12463", "19140", "25850", "6435", "9970", "7959", "11606", "12314", "23941", "13263", "5905", "16813", "10128", "4347", "13560", "5937", "5412", "20884", "23725", "20830", "18734", "9423", "24737", "2177", "19751", "15526", "19768", "6752", "6441", "3769", "17667", "8046", "17923", "19396", "2863", "15430", "4703", "3032", "12166", "9633", "19965", "323", "17253", "3806", "11766", "8101", "10001", "6059", "1698", "4887", "3131", "20177", "499", "18379", "5290", "16043", "17741", "24360", "3490", "21936", "18282", "9810", "9606", "17592", "6529", "4885", "378", "12367", "8616", "8425", "4326", "6145", "9511", "17807", "9821", "8277", "5661", "4139", "7489", "25974", "22317", "25306", "19389", "16088", "5049", "251", "19607", "5461", "21878", "3177", "2636", "7641", "24268", "14985", "9751", "21896", "3285", "19060", "1163", "21254", "16443", "4331", "1096", "8499", "4499", "11043", "10027", "1580", "9136", "20116", "21871", "22572", "10073", "5606", "12873", "716", "12712", "15657", "6730", "2607", "11047", "16210", "16203", "25733", "14866", "24450", "22676", "7853", "20225", "291", "129", "20862", "4467", "25130", "13188", "18314", "13025", "22328", "7758", "22859", "14199", "19534", "8049", "2229", "1730", "8648", "14995", "5666", "16309", "9239", "21203", "14706", "9140", "18797", "25699", "11980", "18006", "25844", "21796", "16561", "12461", "10895", "1355", "9800", "6378", "15207", "21252", "21169", "5897", "22689", "11599", "21582", "5408", "20895", "20755", "19123", "7395", "673", "5888", "24522", "25221", "25866", "7127", "20594", "17532", "15432", "12522", "19585", "6540", "2873", "17349", "8842", "25868", "2786", "24309", "20462", "2511", "7912", "8161", "3740", "3819", "4053", "12534", "8536", "1693", "25404", "15864", "16218", "19391", "1383", "13829", "17528", "21959", "13012", "9830", "24666", "21481", "17563", "7179", "25849", "17791", "6814", "23657", "3797", "12755", "25428", "20289", "429", "2665", "1227", "21744", "7720", "15973", "19894", "22969", "11196", "11461", "21377", "25396", "7793", "22549", "1077", "14115", "8843", "7364", "7463", "2320", "20196", "14582", "6782", "25902", "2054", "11209", "15049", "18994", "13450", "39", "22559", "24975", "12081", "3304", "5425", "18478", "7704", "9943", "20614", "8370", "4252", "2583", "6198", "23891", "4465", "5947", "16658", "14912", "11089", "20001", "16701", "11903", "20356", "8495", "9197", "15107", "8984", "7706", "4812", "13536", "17922", "14756", "10380", "11173", "1956", "24390", "24908", "14250", "16872", "12570", "19128", "14588", "11806", "7921", "7838", "17265", "2615", "12186", "20849", "9274", "23427", "11595", "25943", "5877", "11312", "3242", "9628", "13211", "20566", "15117", "21490", "12880", "1235", "24922", "11708", "14061", "52", "75", "3969", "24671", "18809", "16974", "13713", "23521", "12468", "19432", "21119", "23821", "20073", "10909", "20229", "1600", "20973", "399", "8243", "9269", "18183", "866", "12617", "15471", "23757", "23507", "24054", "14970", "13603", "13672", "24860", "10076", "6007", "13609", "7991", "13304", "23918", "17374", "11558", "25233", "4764", "1493", "20890", "25742", "6627", "1452", "6778", "20886", "2850", "1964", "20195", "19595", "2563", "24669", "17629", "11576", "1135", "19715", "13332", "1472", "6742", "12212", "4486", "6781", "4909", "15224", "14555", "24311", "19596", "4747", "23534", "1208", "24414", "23629", "1673", "785", "10476", "17264", "26025", "12424", "13396", "8000", "15594", "20814", "10141", "24153", "20618", "14549", "7729", "3660", "12464", "10893", "5266", "19750", "17282", "479", "21634", "7061", "775", "24189", "5135", "12078", "33", "14429", "12612", "15826", "23243", "21846", "15283", "3906", "2404", "19654", "15303", "3547", "10451", "462", "11744", "24745", "4111", "21536", "14270", "17694", "1384", "22308", "25790", "8056", "25573", "11409", "4923", "10712", "17157", "18155", "11235", "14244", "15439", "17700", "3381", "24240", "4030", "7007", "18698", "20408", "3006", "4415", "25764", "6963", "15014", "1791", "23401", "12558", "23927", "14491", "12199", "15547", "5954", "9477", "5413", "7218", "2525", "22185", "23880", "14763", "9178", "13371", "8629", "24584", "13067", "3337", "7428", "8656", "7102", "10722", "13173", "9789", "10164", "11393", "17875", "22318", "24231", "19583", "10907", "11134", "17071", "22297", "14260", "21875", "19652", "3003", "7678", "1659", "19922", "21598", "5611", "18638", "22462", "7312", "11525", "19197", "7977", "12639", "25672", "20142", "22150", "24077", "2953", "8728", "23824", "22157", "8014", "5126", "4932", "2454", "6177", "12796", "8837", "99", "4553", "6249", "5604", "8504", "3049", "8530", "25087", "16526", "24543", "19025", "24875", "17498", "3957", "6020", "7970", "4826", "14240", "8341", "12737", "10535", "9437", "19149", "23902", "20416", "25495", "1694", "25434", "4468", "6701", "5871", "22038", "2633", "20850", "11122", "6311", "21703", "5827", "12711", "15331", "16467", "13177", "21556", "17200", "9853", "20794", "15707", "7471", "1609", "12653", "11140", "10470", "7461", "8003", "10258", "16895", "24598", "924", "9860", "4554", "10790", "10490", "10304", "18484", "6309", "25711", "15977", "3736", "4821", "14161", "17746", "450", "19801", "6032", "18943", "11507", "23588", "5214", "17195", "16962", "467", "17541", "25169", "23319", "14207", "7479", "20335", "15307", "8544", "14307", "16715", "10541", "22941", "23231", "8561", "3867", "9284", "351", "20681", "17597", "17278", "11887", "8298", "2298", "17191", "309", "24343", "16168", "17090", "24459", "2117", "2259", "8833", "14660", "22961", "19442", "11212", "13892", "21670", "6563", "9157", "25983", "16914", "19541", "12152", "10480", "19111", "12499", "24397", "15260", "20480", "10069", "4440", "24530", "1283", "9753", "3737", "269", "12111", "8205", "15246", "12569", "4011", "12449", "10334", "8157", "8940", "18831", "10134", "6036", "13292", "14854", "3344", "11109", "3239", "1534", "17126", "2464", "18315", "22819", "2271", "3156", "9651", "6034", "15876", "18655", "13745", "6390", "6244", "15234", "7005", "1679", "10096", "14452", "23447", "24715", "10600", "5505", "21725", "379", "24023", "15016", "10068", "25947", "21017", "654", "20207", "15702", "25368", "4128", "8111", "9980", "18617", "17117", "273", "3709", "17322", "19522", "3563", "6647", "20368", "18796", "4016", "2131", "16315", "5305", "2114", "3747", "10275", "8731", "6492", "15494", "9503", "20162", "21810", "24870", "14594", "12480", "2718", "2847", "4935", "3711", "6376", "11259", "22145", "11548", "19856", "2033", "13223", "1373", "17202", "1761", "1242", "6785", "17648", "17744", "10365", "13248", "22606", "5535", "24452", "11840", "12268", "1598", "21414", "3882", "22084", "6245", "24110", "12029", "87", "7047", "22681", "23722", "8423", "20259", "25417", "17857", "19094", "13196", "11817", "21149", "3223", "23264", "25120", "13000", "10762", "19947", "3041", "8151", "1463", "23389", "2940", "23035", "10244", "6724", "1103", "11070", "7700", "2358", "21567", "15464", "8008", "14483", "9805", "5908", "20540", "13628", "9405", "13445", "17552", "19935", "14208", "2067", "11626", "23333", "2341", "18060", "4516", "25006", "20939", "15217", "16607", "9808", "13602", "21296", "10961", "6488", "6681", "25303", "5213", "14136", "14519", "21157", "14126", "6065", "7352", "19958", "9452", "11449", "7616", "20771", "18255", "2392", "23958", "2949", "21131", "2676", "7347", "20645", "4175", "11411", "17111", "4751", "22627", "19151", "9298", "742", "14375", "2864", "17196", "3375", "20760", "21659", "17203", "14708", "15840", "6800", "23156", "1521", "9002", "2491", "24517", "23553", "20080", "15870", "11668", "22537", "5654", "19321", "24015", "6521", "13818", "12688", "6085", "8615", "8034", "2005", "10278", "20635", "22656", "15709", "11232", "10837", "14764", "19289", "1872", "16936", "686", "22088", "5729", "13527", "23252", "6479", "13427", "25270", "7172", "14178", "15728", "15962", "25639", "12208", "6205", "23659", "14832", "21596", "18724", "15504", "6674", "9535", "3286", "24082", "21362", "9750", "5941", "13028", "22329", "18798", "4870", "24229", "25340", "20460", "17536", "8053", "7269", "3146", "5011", "18295", "13824", "13572", "9216", "2812", "15627", "2810", "13033", "3018", "9936", "21656", "1573", "4012", "8288", "25959", "19717", "7076", "8923", "4103", "22300", "5324", "10064", "13692", "5147", "15899", "20742", "14245", "21906", "2656", "4168", "25585", "3632", "22795", "36", "10488", "21046", "13303", "20908", "2460", "14106", "1970", "15425", "4299", "24281", "22495", "16163", "22418", "17135", "24227", "22567", "25466", "10171", "4018", "5523", "14316", "10618", "5714", "10952", "20321", "7200", "12221", "10573", "19422", "6463", "13921", "24252", "8198", "9889", "20744", "8969", "25636", "17886", "11424", "7320", "14092", "1122", "4784", "24934", "17213", "1320", "3316", "19053", "21938", "9240", "17096", "18765", "16762", "252", "19012", "9698", "22011", "15844", "6267", "9072", "15829", "25127", "12663", "3622", "16883", "9802", "9999", "3194", "796", "5044", "21569", "16866", "5361", "17436", "21159", "10129", "701", "23418", "8118", "1060", "20040", "19195", "2699", "6620", "24708", "20087", "19110", "20964", "2482", "21948", "8655", "15438", "13008", "23563", "5424", "20584", "16940", "18390", "2370", "4694", "19170", "5618", "3240", "7067", "5978", "9918", "23204", "239", "19489", "12716", "25728", "7740", "18642", "11088", "15400", "2349", "22734", "4975", "25531", "14608", "10368", "24", "21385", "5840", "19450", "24778", "14094", "18447", "18389", "22877", "23110", "18329", "5004", "20768", "987", "6415", "23217", "4198", "12676", "13335", "6288", "25706", "6961", "4348", "13321", "1508", "3118", "14152", "18081", "19810", "21278", "4948", "22644", "21840", "3368", "3096", "733", "6275", "15497", "4963", "24536", "25399", "8189", "3339", "945", "3202", "13830", "16466", "25200", "16064", "18316", "5393", "8872", "5108", "12799", "10312", "22825", "3115", "9124", "8582", "3761", "1928", "25762", "22974", "3205", "8383", "22110", "11072", "18893", "12038", "12430", "20845", "22378", "9258", "23522", "520", "5385", "23735", "20604", "16022", "2449", "24331", "6926", "9920", "14239", "17302", "16179", "6511", "15783", "8244", "18740", "20432", "21898", "22298", "19618", "196", "13918", "16295", "14050", "20819", "11131", "5588", "19010", "19903", "10550", "10289", "24852", "14187", "5159", "4260", "23055", "1159", "10374", "2641", "20441", "6592", "3523", "12815", "20711", "15173", "18403", "6750", "13939", "8145", "20852", "2134", "15296", "5776", "16497", "8647", "1795", "21544", "11168", "2655", "8158", "22079", "13640", "2000", "18044", "12823", "14690", "18000", "15669", "13397", "3884", "3347", "15923", "12528", "12184", "13298", "17205", "13598", "5040", "25246", "10635", "22036", "8813", "2509", "6783", "13125", "6919", "18662", "15047", "10696", "19620", "11164", "2150", "6995", "4098", "19227", "23060", "18761", "19849", "22816", "14624", "9896", "14938", "12977", "4491", "3477", "6711", "21088", "11822", "20425", "21179", "19143", "15426", "7324", "9230", "4965", "23277", "12352", "4368", "25249", "10342", "22719", "15968", "4543", "21025", "4312", "4578", "10195", "18550", "1196", "14829", "21227", "25114", "2412", "9385", "16124", "1074", "7962", "16521", "13209", "4813", "19853", "2524", "18007", "7149", "14541", "14751", "23678", "1249", "23513", "23961", "25313", "13381", "7186", "17630", "5138", "11944", "7247", "9734", "6412", "16789", "17754", "91", "10005", "23968", "15276", "5885", "17085", "24046", "1509", "12371", "2838", "8790", "15596", "973", "10540", "23440", "4228", "20835", "11864", "12109", "2558", "17444", "17722", "12700", "11247", "10804", "419", "21113", "2616", "25632", "7543", "5289", "16752", "14267", "24182", "1692", "1299", "16882", "25851", "2567", "24800", "25236", "2882", "9767", "4205", "23708", "12738", "16849", "13708", "218", "13040", "22904", "9303", "18951", "25765", "9160", "22968", "17708", "13331", "14008", "3147", "7350", "9895", "24617", "15744", "19794", "18284", "11716", "5951", "9677", "382", "7448", "25891", "7976", "2004", "9963", "16205", "10140", "4303", "3163", "14987", "23045", "17589", "24529", "6325", "3318", "18393", "4705", "3896", "26020", "2943", "16695", "5932", "18580", "12903", "25076", "16870", "1781", "5018", "1396", "1530", "3940", "11014", "4227", "21315", "25784", "2997", "6937", "14343", "17764", "22916", "15370", "8946", "9417", "22357", "14533", "17130", "20654", "1808", "14614", "13027", "8488", "4192", "655", "9840", "25885", "12272", "24841", "3341", "2172", "17305", "10413", "21975", "7910", "4416", "23712", "24758", "23095", "10536", "2273", "10276", "15058", "22978", "16040", "6661", "20867", "8327", "18774", "1543", "25893", "4071", "16616", "21448", "1310", "17844", "22521", "13497", "13771", "178", "17544", "15250", "17852", "25184", "13226", "13023", "19138", "9325", "18987", "12297", "11009", "17422", "20496", "3650", "12912", "25464", "2030", "17690", "17348", "10577", "18493", "24724", "2834", "11658", "9499", "2291", "24732", "25175", "19816", "11032", "1097", "22880", "18840", "22379", "19230", "24297", "16821", "25813", "23765", "5595", "18839", "2125", "3453", "23780", "16848", "13246", "17873", "19689", "21116", "8888", "16150", "11296", "10530", "4656", "19756", "9558", "13595", "483", "8690", "11063", "7750", "20345", "14576", "21609", "8878", "11454", "20564", "14233", "16665", "7680", "22160", "15556", "19065", "24184", "629", "4880", "24823", "21466", "19597", "17054", "8966", "12381", "7927", "4057", "23736", "8896", "25096", "24704", "13417", "14870", "11990", "20365", "21187", "23325", "5716", "6139", "318", "404", "5758", "14779", "9957", "16756", "1620", "11324", "16405", "7667", "22871", "25154", "7034", "19564", "11459", "22283", "12483", "3310", "9393", "7409", "23128", "18126", "7791", "5133", "14977", "14853", "5180", "25025", "14043", "7060", "20525", "625", "3662", "10565", "2708", "24460", "16141", "9690", "1233", "16611", "4188", "17035", "19154", "9611", "2206", "2981", "20200", "12249", "9685", "9227", "7079", "7272", "13825", "14986", "8856", "13257", "6240", "18989", "6726", "24677", "24256", "11391", "10526", "20726", "14364", "5197", "1443", "17961", "6261", "18602", "20445", "2352", "15156", "21623", "14234", "20634", "21331", "9333", "15486", "20127", "17192", "17649", "23387", "20543", "7924", "19283", "21677", "2843", "15201", "13970", "843", "8955", "17606", "18420", "25183", "17044", "17076", "25688", "17877", "19229", "1089", "24370", "21989", "14083", "1566", "18659", "12439", "14345", "8039", "20931", "14781", "9993", "7217", "819", "16783", "6936", "23030", "10290", "4698", "24586", "11343", "5565", "15747", "18279", "4752", "21721", "16028", "5005", "14508", "14529", "5320", "4795", "19221", "13678", "9467", "18625", "21332", "20346", "23994", "10642", "22395", "6425", "21211", "12769", "21011", "21669", "7276", "8707", "6023", "6962", "23419", "20944", "11584", "17812", "7815", "17084", "16897", "19402", "2239", "23046", "5944", "12529", "3616", "12500", "7036", "16648", "6228", "1516", "11216", "20901", "20093", "6278", "15433", "10613", "8577", "6349", "17394", "18288", "1811", "10075", "3175", "18303", "760", "21619", "5681", "5753", "6635", "9383", "23705", "1050", "15743", "18387", "16721", "13922", "22456", "10023", "11638", "19719", "638", "16207", "25367", "14022", "7897", "24775", "10104", "4065", "3421", "5899", "14578", "25373", "717", "19510", "25400", "11375", "18161", "197", "5996", "9540", "15128", "6695", "18290", "19671", "17647", "17855", "17378", "21339", "20287", "20114", "7105", "17343", "11573", "12898", "2057", "4817", "15529", "24868", "15421", "23608", "25442", "5234", "20505", "19942", "1197", "11719", "16661", "5727", "16413", "13845", "24918", "19256", "25205", "20146", "24859", "13429", "2470", "6483", "15222", "725", "18762", "12020", "25597", "4004", "6487", "25613", "2527", "12543", "12035", "21740", "19104", "10637", "6542", "24646", "17116", "21595", "8485", "5732", "9259", "14189", "7953", "20580", "24068", "23249", "12306", "14399", "16867", "1261", "14954", "2841", "3933", "6282", "22443", "17559", "253", "7231", "8927", "19037", "7902", "8060", "13288", "6154", "8729", "10429", "23614", "6745", "12544", "19605", "12023", "16182", "18646", "25027", "15819", "13657", "10866", "20826", "14201", "10016", "17138", "18552", "1646", "25041", "10818", "3415", "11110", "17475", "8882", "19950", "22299", "24813", "24333", "23265", "21140", "17250", "2103", "10026", "17106", "16918", "25238", "23955", "4913", "4404", "16121", "16586", "25309", "9944", "12158", "2582", "8525", "7188", "17772", "17268", "5422", "25625", "6423", "21138", "18728", "10194", "10593", "5152", "17775", "8375", "16162", "2409", "18275", "13256", "21868", "20713", "6735", "21352", "280", "3389", "986", "1331", "9244", "11631", "3483", "10898", "13525", "20128", "14500", "14327", "3850", "25447", "16285", "10592", "8750", "12040", "14613", "15218", "7009", "1732", "20164", "23716", "15749", "14467", "11871", "19043", "4731", "15318", "1425", "8376", "19228", "13978", "16578", "1538", "115", "19979", "9852", "9367", "15435", "25387", "20296", "22420", "21512", "14220", "16319", "12577", "20975", "1553", "25315", "4629", "5076", "24198", "14953", "20261", "21389", "20185", "24206", "21527", "21818", "14744", "6777", "11975", "24471", "16888", "8716", "20533", "24976", "24356", "12088", "21833", "2854", "13069", "1555", "2925", "4987", "6530", "18805", "1130", "13072", "24219", "4495", "14322", "20137", "4164", "20508", "4264", "17171", "10229", "10081", "11437", "25589", "21660", "20620", "14101", "12207", "9740", "19455", "20498", "20078", "9464", "24375", "20328", "12025", "5592", "14799", "9376", "2966", "11277", "11055", "6546", "22687", "5772", "6990", "14313", "19087", "23540", "4244", "22638", "22042", "6419", "4899", "25333", "7916", "20806", "5963", "21800", "2771", "21841", "19332", "6929", "13806", "19431", "10115", "23916", "12845", "9426", "20131", "2924", "24374", "11749", "16736", "25619", "13795", "6881", "8611", "14864", "24342", "19694", "18154", "12161", "1274", "23801", "2455", "11021", "16898", "19048", "23118", "23679", "15291", "18180", "9475", "5285", "12816", "18268", "4311", "20124", "12751", "25513", "24368", "12093", "4123", "14495", "13985", "9595", "22319", "18401", "21624", "3643", "25278", "22117", "20311", "2693", "3441", "22158", "16353", "4360", "20090", "21647", "3656", "4167", "8220", "21470", "11448", "10534", "6570", "1400", "9891", "20897", "24814", "7486", "2837", "18935", "11130", "2468", "20738", "6857", "8604", "6828", "12216", "20825", "18543", "6762", "11025", "19206", "14738", "14457", "11805", "13529", "15580", "1594", "11788", "9787", "23732", "24662", "12356", "7402", "9569", "13537", "14618", "14807", "5636", "22695", "23501", "15830", "11059", "7519", "3699", "5591", "15673", "3766", "8005", "25429", "8769", "20758", "8137", "17339", "6692", "16928", "24058", "18876", "2329", "5696", "10671", "23005", "8371", "24672", "5148", "8374", "18604", "25502", "18834", "22894", "21310", "8013", "744", "5242", "11724", "21899", "25550", "9493", "7125", "4261", "7030", "1178", "11878", "20268", "7948", "17978", "19946", "2589", "14835", "6609", "6265", "17712", "7401", "1412", "4080", "15638", "3511", "915", "7679", "2348", "2082", "10339", "5292", "16473", "11892", "5158", "23529", "12613", "12373", "16395", "4462", "22685", "9964", "23421", "2018", "13385", "3499", "256", "7882", "6186", "13660", "16069", "2611", "24101", "23928", "4041", "2309", "14458", "25907", "23358", "10216", "2403", "10351", "11828", "12794", "13894", "12172", "18129", "13591", "25827", "16039", "10858", "19650", "17312", "7209", "21043", "15563", "23463", "21621", "6490", "5369", "20889", "16615", "24611", "24123", "9647", "15266", "17167", "11905", "24896", "24963", "6220", "4867", "17925", "5095", "5492", "11598", "5169", "16072", "9962", "22471", "20378", "2568", "8660", "20400", "15489", "21822", "17801", "15583", "17291", "17109", "21724", "2210", "5116", "21562", "3145", "17347", "22619", "443", "14214", "7724", "653", "25938", "17428", "6815", "15738", "20106", "23330", "10316", "21006", "6258", "525", "17686", "13933", "18660", "14685", "6725", "15589", "19386", "4401", "8222", "23809", "10000", "18756", "18386", "9000", "6", "21", "12252", "5087", "13734", "24035", "23669", "1438", "15508", "20951", "24090", "748", "18336", "3772", "21707", "20947", "14717", "19970", "9247", "18832", "24627", "25734", "13175", "17561", "19767", "10666", "6025", "12575", "9603", "25202", "11941", "20386", "6132", "10097", "20909", "7703", "8733", "6191", "4371", "10874", "17154", "2162", "20836", "334", "119", "24264", "5141", "9730", "1832", "8908", "18347", "16095", "15949", "2706", "21176", "24138", "13931", "23201", "8002", "12661", "5485", "4170", "22397", "22692", "12605", "1420", "22153", "2217", "6010", "11619", "13148", "20428", "6604", "18232", "8893", "16614", "9211", "21819", "1675", "17773", "12138", "23377", "16077", "4169", "4990", "10426", "20257", "9181", "14957", "14173", "3729", "12765", "24230", "23528", "21576", "395", "3014", "7153", "19770", "21425", "22484", "23943", "9205", "20044", "25042", "2697", "14563", "13558", "17466", "12995", "8951", "15946", "2434", "8751", "1773", "23707", "11415", "12342", "6406", "3464", "14586", "21344", "11622", "11973", "15598", "13365", "18054", "19194", "8130", "4664", "21743", "9780", "7259", "16492", "6455", "12749", "19937", "4628", "17858", "8702", "10647", "199", "21106", "877", "8721", "9733", "20497", "7296", "9561", "15167", "92", "5284", "20431", "7063", "20273", "5303", "5356", "21683", "22651", "15877", "14865", "7374", "22793", "9154", "14287", "2637", "17745", "19394", "5889", "23520", "18923", "9283", "4649", "6757", "24698", "1025", "23922", "6382", "17121", "21144", "3312", "18153", "5314", "19078", "6151", "661", "24173", "21098", "5462", "14289", "14728", "15908", "8953", "2593", "15910", "5256", "12473", "19366", "23344", "6115", "25509", "15197", "22571", "4392", "8163", "3757", "14574", "9358", "791", "11483", "15343", "24620", "17932", "17179", "4149", "1068", "4028", "6637", "14455", "13721", "143", "20255", "8042", "16956", "8913", "16333", "5688", "18709", "20219", "5484", "15543", "25007", "6329", "21387", "3688", "7221", "11700", "23145", "11685", "18246", "5237", "24166", "15861", "3754", "14054", "3587", "23587", "8412", "6063", "12121", "20099", "13958", "25360", "22919", "17047", "25043", "1993", "6103", "25797", "16129", "9011", "15403", "6149", "7777", "22108", "4719", "16340", "2282", "2106", "7446", "10463", "14070", "19889", "23479", "16140", "6196", "1762", "12920", "14793", "19150", "14471", "6243", "21404", "6274", "21145", "16529", "24026", "24909", "21034", "23874", "6144", "7425", "1073", "18800", "6048", "1563", "10177", "16283", "6057", "17462", "22212", "25211", "10168", "20228", "10311", "16100", "10775", "23086", "13954", "4753", "18814", "8614", "1442", "12167", "16688", "10058", "15699", "641", "2293", "25886", "23661", "13951", "19059", "21405", "6072", "23612", "22141", "677", "23989", "20283", "2337", "7779", "9625", "2876", "7682", "18735", "10091", "5955", "6978", "20271", "24457", "14048", "2540", "19592", "12643", "12547", "6589", "19785", "13084", "15300", "24919", "19515", "16980", "12817", "6874", "18456", "12598", "18418", "19813", "19838", "5624", "2086", "20222", "1991", "24366", "8086", "24102", "4757", "2088", "18702", "18032", "1041", "9632", "17874", "17189", "401", "2625", "21966", "13437", "12045", "3440", "1336", "20060", "1128", "14562", "22909", "24640", "25272", "17356", "10294", "22021", "20810", "21454", "21333", "19165", "2561", "8737", "3909", "21100", "6260", "12474", "1406", "11184", "22671", "10376", "19624", "21155", "22241", "4573", "8810", "17736", "7666", "15356", "226", "6060", "15273", "2194", "13913", "25946", "13582", "23399", "14611", "12319", "11880", "21792", "17174", "15742", "927", "5919", "4846", "13319", "18189", "6820", "15337", "7120", "3826", "17560", "19452", "18648", "22350", "13510", "6176", "21632", "1888", "15590", "25364", "19670", "20390", "10367", "6496", "18885", "2021", "21346", "536", "8072", "18138", "4032", "20641", "16530", "650", "2076", "16060", "9215", "21125", "12047", "24440", "3002", "15024", "11287", "24067", "22346", "2181", "6601", "23589", "25634", "435", "25591", "15853", "16096", "24005", "2223", "25092", "15160", "14221", "25655", "5154", "12355", "2992", "17876", "13943", "22598", "13128", "7199", "1260", "22922", "21164", "16728", "22387", "6077", "20371", "11469", "22086", "1683", "8440", "8036", "15802", "288", "9857", "22728", "16391", "18038", "17272", "15368", "753", "12883", "25732", "15229", "4338", "20934", "20247", "22143", "16001", "15642", "2495", "18075", "23246", "25792", "19573", "23855", "19474", "8233", "16494", "235", "15726", "1316", "1957", "1951", "7052", "11511", "10798", "20534", "9557", "13910", "13813", "2889", "23251", "10439", "6608", "22005", "62", "21844", "20066", "3037", "4881", "21003", "12334", "16106", "21133", "15889", "22780", "11002", "8987", "478", "12533", "6953", "2252", "10709", "3655", "20770", "14558", "19677", "13543", "11661", "991", "11533", "25801", "15667", "24969", "14976", "4160", "2272", "2152", "1219", "12964", "3103", "3467", "10438", "9063", "23437", "25931", "23784", "12365", "8108", "23266", "10917", "20305", "7639", "6566", "13624", "14084", "25481", "20184", "20593", "17344", "25988", "18385", "9642", "15001", "7774", "9409", "10820", "2085", "6848", "17372", "899", "16278", "20197", "13050", "19511", "14204", "7801", "23225", "11725", "18852", "1771", "6323", "11117", "18170", "20013", "11215", "25512", "5167", "5757", "22085", "2410", "11124", "14944", "11445", "18324", "17525", "12250", "25162", "11438", "2500", "8764", "9035", "19416", "13377", "17185", "904", "8775", "12999", "7768", "699", "25046", "21438", "25051", "17919", "15029", "5078", "25090", "24047", "15831", "1636", "22738", "5120", "6646", "16970", "22180", "17031", "3881", "24736", "12621", "18919", "24700", "6227", "13637", "17125", "14305", "6359", "20477", "15396", "19350", "11521", "17900", "13577", "17806", "2705", "1309", "18365", "24085", "7420", "13653", "22429", "20418", "7606", "818", "11453", "11561", "9107", "1971", "17239", "20008", "6395", "23314", "1478", "153", "23820", "8368", "9112", "6870", "14934", "8044", "8998", "2866", "5117", "11239", "11934", "8399", "22900", "11636", "1147", "21455", "16811", "2235", "16172", "15415", "8506", "10822", "18783", "15804", "9292", "20133", "10087", "9495", "23134", "7473", "24111", "18079", "12259", "12183", "6334", "3939", "7626", "5343", "8116", "2827", "21449", "19961", "3880", "15028", "15719", "19365", "21226", "795", "21998", "14722", "9404", "16469", "16913", "10838", "12620", "7738", "11885", "12797", "23212", "14190", "5551", "23981", "939", "7145", "19676", "12039", "23375", "23904", "13942", "4381", "9260", "19437", "6930", "22927", "12084", "18927", "16817", "16778", "15138", "21551", "6970", "15042", "3633", "8292", "4953", "19891", "15030", "18248", "22112", "21893", "15693", "2890", "1155", "11597", "12814", "20295", "294", "24289", "25599", "16535", "3518", "21256", "287", "8624", "6640", "3188", "9179", "7548", "22730", "13600", "4720", "3657", "1652", "18649", "22892", "21739", "8493", "16015", "22652", "1616", "861", "9097", "17557", "21390", "10088", "4070", "8418", "13373", "5990", "17718", "13502", "2301", "22895", "820", "7772", "23523", "23804", "24349", "21552", "20412", "20544", "8840", "10482", "21222", "21627", "10308", "5602", "15261", "4484", "4350", "603", "20481", "2909", "13231", "22617", "4844", "7719", "9009", "1489", "19776", "11522", "23147", "14715", "13364", "6893", "11629", "24806", "22586", "13662", "4051", "19784", "1011", "3406", "26004", "10551", "8431", "16331", "2853", "9321", "23361", "9855", "13131", "9875", "3998", "9220", "19044", "18201", "17077", "25499", "11938", "2251", "5094", "18077", "6958", "2923", "9786", "11028", "20276", "13843", "10815", "20083", "14523", "19265", "4822", "15560", "12671", "1631", "4430", "17797", "18535", "2590", "21643", "24080", "347", "14679", "19276", "19351", "5697", "14654", "19237", "20658", "5801", "2965", "6517", "10400", "7365", "9697", "886", "19653", "2003", "7744", "11516", "17414", "4234", "16655", "12709", "16785", "6147", "19531", "17949", "16452", "12414", "3361", "25900", "23215", "23833", "24727", "13024", "12456", "9318", "1256", "3295", "21298", "17180", "20140", "11865", "17881", "3384", "17652", "8585", "23054", "6610", "10084", "17530", "10190", "7672", "3612", "14349", "9600", "13496", "12524", "21031", "2748", "16946", "21937", "25990", "20104", "5803", "18496", "23992", "13654", "6172", "4067", "19516", "433", "14076", "1395", "13051", "20265", "11376", "14116", "6547", "25578", "5071", "3172", "15384", "21942", "19185", "18844", "17725", "1715", "7903", "11784", "4286", "23281", "6080", "21648", "10266", "16254", "20417", "20756", "6017", "2052", "11243", "1912", "13610", "20966", "4549", "7652", "8262", "18411", "21016", "7677", "53", "3821", "9815", "25796", "20899", "8588", "14465", "6033", "24403", "12073", "7457", "13220", "7187", "19224", "22866", "15392", "25346", "18755", "5208", "18240", "23152", "18716", "1548", "15428", "10650", "4254", "23101", "14297", "21644", "22980", "15221", "17921", "8300", "16037", "3260", "16845", "20375", "24453", "7920", "402", "17757", "6387", "13097", "25344", "24196", "13251", "6528", "2686", "16786", "20571", "10654", "18100", "13124", "17657", "23701", "19745", "7971", "24143", "4501", "4024", "3714", "11408", "25815", "9090", "12150", "10625", "21694", "54", "2091", "8652", "14078", "2610", "8305", "6410", "22460", "23570", "7863", "23087", "17214", "3332", "5848", "5028", "2857", "5608", "15680", "23013", "23051", "13696", "9434", "9804", "21189", "1115", "22250", "23102", "25759", "9906", "11971", "9551", "11761", "23680", "15233", "19773", "5842", "16188", "18156", "453", "15740", "14166", "10158", "9123", "14320", "5770", "14213", "15970", "21491", "6093", "22845", "25160", "16706", "20358", "19744", "22492", "23240", "18843", "8583", "25462", "22477", "3270", "15095", "14073", "6454", "17212", "2813", "5137", "18371", "20894", "11930", "22374", "11863", "6319", "20436", "1333", "19867", "8289", "21509", "16388", "9723", "15807", "19740", "24815", "17527", "17226", "7898", "19125", "20900", "4607", "15338", "15676", "4362", "18471", "13984", "6883", "14358", "24177", "10160", "3794", "9842", "9565", "107", "16084", "10965", "5163", "6509", "7601", "16463", "6102", "19949", "18945", "14940", "7663", "474", "11966", "7363", "23111", "609", "17460", "4332", "18024", "12153", "4226", "24560", "18856", "5142", "10514", "14998", "24059", "805", "7884", "19825", "19369", "14550", "25157", "14897", "10198", "15887", "953", "23498", "10911", "7437", "22016", "12714", "20829", "10604", "23116", "17043", "5693", "5902", "20495", "19120", "25507", "25443", "10750", "6946", "24888", "13212", "10418", "25603", "5400", "24298", "23486", "3813", "20611", "17469", "26014", "449", "19705", "7931", "17868", "6636", "2760", "15100", "7210", "23604", "18328", "25916", "23032", "2380", "25957", "5035", "6181", "25751", "20522", "24428", "2596", "22640", "4174", "25624", "20476", "15578", "11246", "16574", "14056", "7508", "6213", "1174", "13010", "8691", "3878", "14988", "22073", "13495", "17220", "25494", "17033", "18541", "4289", "912", "13868", "854", "6591", "17217", "24549", "18837", "5670", "6533", "17006", "6787", "15062", "8378", "20720", "4224", "18583", "1267", "20423", "2350", "17697", "6537", "14749", "24169", "11315", "14734", "15760", "24688", "6302", "3222", "4636", "6914", "25452", "22777", "10139", "25253", "17404", "3763", "5490", "15282", "4351", "19960", "24795", "15701", "21247", "177", "5820", "2994", "19569", "2886", "19178", "3796", "19176", "22111", "14159", "15293", "22930", "5747", "9045", "6832", "2961", "8718", "20326", "9799", "24039", "3504", "24443", "1591", "23114", "7367", "19272", "21028", "21684", "7004", "7171", "21085", "19492", "13336", "1351", "16433", "22682", "19805", "19481", "10359", "6613", "10338", "3825", "21355", "18125", "13431", "13291", "20694", "6026", "2316", "19330", "12312", "13190", "3510", "12096", "21446", "19200", "5645", "20636", "9183", "25803", "8778", "21680", "11688", "19656", "10608", "25800", "2638", "8805", "18148", "11849", "19933", "20176", "23131", "910", "23490", "25012", "18884", "4094", "9610", "10603", "9028", "26006", "4422", "14639", "914", "17866", "11386", "1352", "11545", "9576", "11221", "5852", "17789", "3181", "2761", "6684", "22144", "17853", "11278", "24788", "4161", "21985", "7872", "9823", "16906", "19413", "9580", "22282", "5104", "15913", "2532", "16171", "14206", "3514", "22822", "13891", "22773", "23107", "21666", "24185", "10392", "19002", "11816", "7861", "16922", "20053", "3602", "17346", "11006", "8417", "12278", "17484", "9775", "23040", "23864", "6440", "11035", "16437", "6379", "8985", "24120", "20115", "7195", "18434", "16803", "7514", "9451", "15723", "15453", "1841", "9568", "16101", "15732", "19735", "13216", "4888", "23037", "14261", "1013", "6128", "22353", "12416", "19543", "21143", "9759", "21558", "5786", "19082", "7901", "22465", "9462", "995", "13235", "8216", "3553", "10723", "3887", "21097", "14147", "8204", "15888", "3139", "25048", "583", "3244", "15904", "17951", "8345", "16698", "18633", "2202", "24777", "656", "16960", "1280", "7886", "6106", "2436", "16973", "166", "3689", "21933", "20851", "15064", "21813", "13999", "21051", "24398", "17246", "20147", "6472", "18343", "8786", "5297", "13819", "21393", "15308", "12007", "22975", "14468", "18151", "21353", "18472", "23567", "20696", "22893", "25952", "8793", "19399", "18504", "15963", "10607", "18325", "16128", "4647", "11750", "22743", "902", "9438", "22172", "15774", "7180", "4908", "19114", "25736", "6216", "10208", "25666", "23853", "3363", "20761", "1597", "25351", "21391", "14181", "25890", "14", "15280", "579", "6826", "23254", "16989", "7878", "22995", "21920", "20591", "20994", "8976", "22997", "5390", "12820", "12884", "13289", "8543", "8400", "25726", "8124", "2372", "20210", "22511", "6011", "2187", "11890", "24346", "12085", "6723", "9356", "8904", "4901", "3550", "17916", "19796", "2644", "15689", "8532", "17827", "25182", "9531", "15764", "21342", "20729", "16875", "25777", "13414", "16073", "1519", "1546", "6624", "6746", "18317", "15328", "24644", "22539", "17524", "9927", "13490", "23346", "4156", "6046", "24263", "8841", "22643", "12497", "13136", "19633", "16348", "2111", "14542", "7834", "23579", "5707", "22921", "293", "22393", "913", "4108", "16059", "1264", "1668", "9917", "19799", "22151", "14107", "4814", "20387", "12106", "24618", "3611", "4411", "5271", "9351", "23861", "5355", "13204", "3369", "2284", "694", "22058", "7806", "2758", "6829", "17669", "2074", "595", "24999", "25106", "15289", "13722", "6596", "18744", "23379", "9869", "19270", "15991", "10862", "3760", "18830", "20630", "5516", "18778", "13174", "1798", "11926", "16564", "10348", "8693", "23200", "7467", "6790", "18477", "23169", "20924", "17595", "23505", "3309", "14637", "24431", "21374", "6082", "19005", "18367", "19449", "24359", "2083", "21633", "21062", "14154", "8944", "13349", "1015", "18256", "3567", "2862", "19509", "10853", "8905", "9760", "14450", "23793", "25107", "2533", "1082", "24914", "10113", "12641", "14124", "20325", "967", "5936", "12802", "3541", "19542", "10328", "7715", "24838", "23336", "4441", "24616", "14081", "25987", "3545", "24498", "24482", "9150", "18267", "8410", "25760", "11659", "9418", "22060", "11889", "5916", "22485", "12669", "3039", "369", "14650", "12060", "22633", "11143", "15076", "16485", "14822", "4864", "4463", "21525", "7585", "21786", "11094", "19706", "10888", "19248", "5430", "385", "16236", "247", "7655", "2535", "10226", "12341", "14828", "17518", "5904", "2552", "13839", "736", "21765", "8035", "14905", "23090", "18859", "13586", "13243", "25683", "5281", "11771", "908", "5399", "19673", "22396", "23435", "23753", "10459", "24055", "1586", "14026", "25776", "12521", "15851", "18172", "21297", "1460", "15798", "7609", "3710", "22083", "15075", "3846", "2537", "3502", "13441", "11970", "779", "20531", "15717", "16919", "1063", "21001", "18008", "10434", "17668", "11518", "1653", "22296", "13893", "8099", "3454", "7396", "17662", "25342", "18906", "21360", "15009", "23402", "20551", "9788", "18995", "2785", "10513", "1506", "12406", "17324", "20291", "23892", "24083", "477", "17355", "2822", "23004", "20801", "10935", "5414", "17913", "19478", "17479", "23819", "12980", "1102", "21280", "15038", "5048", "15758", "3730", "8560", "6600", "13171", "9366", "7285", "11914", "12389", "23273", "20679", "11412", "25284", "16450", "21790", "20869", "5184", "6081", "5869", "22600", "16055", "24625", "5067", "22500", "11314", "3097", "25141", "5746", "24518", "14315", "11138", "11352", "15897", "21722", "16496", "13311", "5441", "17516", "1474", "4474", "19713", "7549", "3685", "2808", "20272", "13612", "21474", "12835", "16510", "16447", "5748", "10188", "581", "1014", "18743", "6759", "121", "4444", "13145", "23258", "18729", "8402", "12311", "12080", "9986", "25286", "21037", "16592", "18904", "612", "22428", "4616", "13667", "9089", "17870", "18687", "8109", "14357", "22863", "9854", "1764", "10869", "6658", "20866", "2642", "19644", "23493", "6900", "14691", "15334", "286", "1533", "12066", "24725", "22937", "22408", "20552", "17273", "11116", "2669", "9068", "15538", "5744", "25232", "13584", "5352", "11271", "1273", "7032", "10828", "21384", "11813", "25563", "24362", "24856", "10072", "18506", "11731", "14396", "6328", "6840", "24441", "9267", "22855", "13804", "3726", "18970", "22755", "19045", "14788", "25588", "16438", "17388", "5928", "8353", "3661", "22029", "24364", "22690", "15815", "15712", "8715", "3215", "22585", "25297", "17284", "21129", "18745", "21605", "14218", "24836", "25772", "10560", "906", "9884", "16442", "5260", "10506", "528", "23769", "4521", "10926", "9515", "23823", "4769", "10293", "18112", "18738", "23155", "6466", "22966", "18811", "12202", "6045", "3124", "17421", "13262", "19789", "2764", "12070", "9314", "18113", "4294", "21070", "7327", "20493", "7883", "7310", "5080", "3168", "23878", "6168", "15255", "14165", "801", "929", "18882", "9790", "1036", "2163", "7042", "2799", "8608", "15934", "24267", "4301", "14361", "5047", "23010", "1275", "7184", "24513", "21823", "10833", "9537", "163", "23859", "21224", "22466", "17542", "10946", "4534", "9231", "23178", "3020", "21063", "15010", "8818", "2242", "2334", "17969", "6184", "10976", "5754", "24148", "22722", "8119", "16070", "21395", "2995", "14824", "24588", "3377", "16023", "14112", "4977", "9747", "13744", "14117", "19135", "3839", "3282", "11706", "15190", "13361", "15238", "7492", "8045", "15154", "9225", "9638", "23751", "18511", "6740", "7909", "23847", "15139", "24947", "17351", "15793", "16947", "18929", "16105", "2338", "16329", "22588", "18946", "22065", "10992", "8858", "4369", "25971", "3697", "22072", "67", "5107", "9567", "16742", "24475", "869", "21439", "10824", "23932", "23078", "11844", "7261", "11405", "1000", "5006", "5220", "23734", "802", "23365", "1372", "24657", "12198", "18209", "9285", "10302", "21518", "19660", "20313", "1417", "12411", "9864", "18677", "1891", "6188", "4193", "18203", "21997", "19375", "297", "7018", "23811", "18462", "18626", "8122", "10189", "16345", "23688", "8677", "11654", "6040", "18523", "8993", "11388", "23233", "4225", "8217", "5836", "21215", "1661", "2927", "1241", "12756", "20888", "9329", "2351", "23306", "6868", "23154", "9277", "13750", "20646", "11099", "21489", "23869", "10655", "19235", "1392", "5945", "21639", "22171", "8527", "2322", "25206", "18683", "24145", "19169", "12813", "21282", "24894", "10854", "25515", "13264", "9593", "18068", "16793", "12721", "14895", "4833", "4823", "4937", "7133", "15674", "5391", "1997", "21546", "23963", "11346", "15683", "1054", "19576", "8866", "22679", "15479", "14796", "12013", "22089", "6836", "17093", "3965", "20576", "8240", "4672", "21005", "18635", "6812", "10954", "24175", "15302", "7983", "5483", "24619", "12028", "10972", "9171", "838", "13240", "8063", "21110", "1061", "16689", "26001", "22782", "8502", "18216", "7638", "10699", "21530", "6355", "21706", "14920", "18737", "6375", "12761", "6625", "14896", "5013", "220", "1936", "1736", "2788", "4734", "16724", "11911", "5709", "10092", "17485", "4561", "20450", "24446", "20405", "12236", "3861", "9052", "17594", "17015", "10240", "4197", "16147", "6979", "6352", "17871", "2679", "24682", "10516", "1137", "14688", "8571", "10649", "12072", "21773", "11876", "12768", "5470", "7697", "9115", "8150", "2842", "21002", "12061", "25928", "16303", "741", "15091", "6885", "12665", "38", "416", "2407", "11537", "22225", "25123", "23814", "10496", "911", "20175", "25371", "8444", "1168", "20004", "18070", "23193", "17292", "6289", "18043", "5177", "25810", "6984", "23703", "22232", "17784", "11318", "11292", "23666", "24121", "10852", "12540", "6326", "3527", "13500", "4061", "23882", "10329", "19716", "1203", "9897", "18322", "15751", "1446", "18414", "16533", "25059", "11442", "4641", "5480", "11305", "23132", "5386", "23998", "20818", "23282", "551", "20015", "873", "21205", "7151", "18715", "17981", "20678", "23158", "8619", "10902", "22748", "932", "1643", "9093", "6536", "17490", "10945", "25144", "18229", "17994", "5832", "771", "1132", "18694", "7049", "3586", "11242", "24690", "6719", "6780", "7524", "22260", "20979", "2818", "13455", "21917", "21865", "4916", "20430", "17285", "9973", "11829", "9119", "9320", "20538", "17290", "400", "3802", "12518", "8696", "22574", "12224", "24499", "15484", "11559", "1138", "25691", "25704", "8541", "17141", "2553", "5257", "25992", "4585", "6543", "5090", "10327", "16221", "1330", "7400", "8570", "4998", "2070", "23849", "17082", "6219", "23620", "1525", "17867", "21829", "12459", "12702", "9887", "11274", "24988", "21236", "5255", "22002", "25748", "8250", "20563", "16002", "9476", "14024", "19611", "23366", "11176", "15312", "18629", "6813", "22013", "25055", "7313", "5907", "4397", "1728", "18474", "21482", "6167", "14992", "10333", "22589", "4615", "4130", "20638", "25716", "16011", "1852", "14389", "16193", "12589", "4596", "23630", "17198", "5468", "4105", "22271", "21770", "18185", "8480", "5181", "23575", "20847", "24474", "10401", "4778", "20466", "6116", "2986", "18983", "7867", "21160", "8816", "8272", "17133", "21480", "22509", "25775", "10310", "10807", "17781", "23192", "11741", "11392", "20399", "15352", "22044", "23670", "20946", "6895", "22765", "8968", "15304", "9502", "980", "2214", "15059", "17946", "9560", "24409", "19580", "6576", "18903", "6298", "14935", "5100", "1250", "4470", "7600", "24124", "8938", "7163", "19635", "1288", "17621", "25319", "13372", "19131", "15754", "21805", "6281", "13270", "12350", "14225", "11426", "1496", "14678", "22448", "4295", "7646", "9618", "16090", "9075", "23912", "7849", "12395", "18147", "3085", "3299", "22050", "18486", "21065", "25549", "23082", "6694", "10074", "24948", "1091", "3849", "13799", "24768", "4349", "6717", "2627", "11323", "3956", "776", "9237", "8059", "6534", "24130", "14128", "14398", "1449", "11114", "20079", "2042", "3109", "16223", "21745", "12877", "4711", "2011", "11978", "6804", "1005", "25524", "23298", "4446", "4972", "18176", "22320", "12423", "9888", "16767", "9939", "19761", "10708", "8330", "24168", "10251", "23813", "14472", "19913", "10848", "21210", "13597", "11011", "6842", "19602", "3245", "162", "25996", "20570", "7423", "20215", "23475", "11326", "11466", "25713", "15469", "444", "22459", "5882", "25887", "25300", "8487", "4984", "16606", "6583", "25766", "25173", "17083", "2974", "20822", "17818", "18270", "12863", "13974", "8321", "22464", "25518", "3346", "24225", "21373", "9322", "20949", "2232", "1742", "22754", "25156", "18722", "1744", "25921", "13453", "1996", "5935", "3256", "2806", "25825", "2503", "13472", "24878", "8279", "4879", "13782", "16375", "21094", "19714", "3636", "5926", "19193", "17474", "23544", "7974", "13285", "22284", "3298", "9394", "2710", "24705", "11809", "11747", "7695", "9309", "25779", "6290", "13034", "6312", "2071", "24696", "7517", "7391", "24819", "12048", "15198", "6088", "17221", "6236", "22349", "14042", "2888", "15898", "10440", "7760", "23689", "11791", "24100", "15510", "2508", "20453", "20731", "18766", "10355", "1810", "24962", "14967", "14146", "1887", "23652", "8556", "24021", "2956", "4469", "8231", "10916", "16755", "20018", "5106", "493", "19642", "24816", "16799", "11363", "20569", "19568", "523", "2526", "14339", "19790", "2130", "17599", "13457", "4653", "22023", "13736", "19476", "11160", "25883", "15795", "16080", "3579", "20778", "3810", "3496", "16192", "18776", "1550", "12140", "7387", "18391", "20170", "15491", "23285", "19622", "16127", "14758", "11592", "5557", "7572", "6545", "14304", "12678", "4884", "2581", "21345", "22707", "5800", "24746", "2072", "18035", "23021", "2020", "22211", "9670", "11846", "6944", "21925", "130", "9755", "7417", "23409", "15928", "13031", "24537", "19144", "21255", "6629", "17790", "19242", "24380", "4544", "14278", "1123", "18815", "23860", "6477", "18911", "21263", "10028", "22303", "24527", "21180", "5358", "24721", "1729", "10373", "4803", "21306", "5774", "4563", "4621", "6358", "18407", "13968", "10879", "21837", "5572", "9965", "9942", "8779", "3975", "4329", "3720", "23398", "10372", "2496", "21059", "5081", "14635", "18846", "22090", "19473", "20108", "23067", "5232", "14052", "4201", "22753", "174", "7249", "8106", "24691", "2648", "22576", "24384", "15634", "1125", "1949", "21450", "16599", "6448", "13140", "22536", "16617", "4599", "12457", "15926", "24851", "10162", "2238", "21044", "25977", "11783", "18862", "2903", "25363", "3955", "25374", "2554", "17223", "597", "9689", "10545", "18721", "5610", "1035", "21086", "1308", "10539", "3486", "24109", "24589", "7383", "23704", "22301", "6150", "7914", "18424", "19100", "23108", "18591", "14948", "10319", "4802", "8610", "23077", "20962", "11564", "2201", "210", "19506", "3561", "21962", "16910", "10582", "22463", "15284", "12740", "25602", "10959", "7444", "7416", "5189", "3228", "21600", "10130", "5647", "5350", "7617", "14735", "17100", "3325", "14138", "4296", "16361", "2221", "16146", "14292", "6925", "1766", "19655", "25198", "23470", "6866", "1583", "18868", "8857", "23412", "17136", "21901", "19299", "3792", "18327", "11873", "5958", "11132", "25637", "23656", "21924", "7139", "3578", "7182", "23350", "12942", "3488", "7735", "22883", "22827", "1958", "1896", "13315", "23452", "18442", "25235", "24250", "25168", "22505", "5278", "24626", "10685", "17635", "17098", "14255", "19828", "10915", "24949", "17080", "18564", "16386", "9049", "1628", "20927", "11883", "9190", "10460", "14167", "26002", "4933", "21541", "7287", "421", "13872", "21880", "12229", "16113", "24098", "19103", "7814", "24638", "18512", "12235", "4056", "25237", "11958", "25325", "2757", "17242", "22189", "11680", "23500", "9046", "14780", "25554", "14891", "13113", "18594", "11457", "16458", "11406", "10978", "16557", "22716", "4836", "3556", "24822", "16262", "17335", "20256", "6954", "9977", "25954", "10237", "19349", "25115", "10832", "4874", "24097", "7730", "24232", "21335", "12466", "22862", "20902", "24150", "6751", "15327", "9387", "12032", "7329", "21897", "14698", "1213", "14983", "23671", "20643", "17986", "11104", "16999", "21750", "2049", "23009", "13091", "9571", "9163", "16568", "6617", "22806", "13509", "7569", "5864", "16551", "14184", "13707", "147", "3106", "11610", "23049", "5174", "23038", "16628", "24078", "19061", "14917", "23729", "22875", "24114", "8022", "19566", "7112", "12019", "20277", "142", "24064", "18208", "665", "5321", "14016", "9933", "24421", "25592", "11037", "8191", "9490", "9461", "23817", "3908", "7741", "7054", "13011", "22609", "15033", "3458", "11981", "23020", "14724", "23451", "18959", "17020", "24733", "8777", "3807", "1731", "21599", "9352", "15316", "9639", "25013", "3647", "17206", "23457", "25241", "19555", "18458", "5634", "19726", "6835", "6203", "11265", "521", "24106", "10967", "24435", "21513", "3598", "179", "1632", "4985", "12283", "19348", "17917", "4947", "23466", "9384", "22198", "15690", "23618", "6841", "11722", "11078", "18861", "11123", "23807", "1657", "4689", "17905", "7332", "18029", "8919", "25717", "8508", "5042", "315", "7104", "15812", "13308", "23979", "10578", "18306", "14156", "23396", "10896", "1124", "5494", "9390", "10827", "3208", "3392", "4216", "761", "10431", "6120", "18791", "14737", "12302", "9981", "587", "22352", "13671", "997", "18047", "13036", "12842", "18206", "17219", "25526", "9203", "9065", "5316", "11421", "8689", "14169", "13063", "10111", "4089", "2809", "2577", "11837", "16461", "1759", "4178", "25673", "4306", "13971", "2241", "22498", "9534", "10211", "23726", "3717", "374", "6071", "21716", "18656", "14257", "20977", "17701", "1686", "18190", "11422", "3161", "10931", "19755", "333", "7612", "16996", "13187", "13268", "5054", "3274", "3155", "7998", "8755", "12396", "25468", "5127", "1529", "21842", "11434", "16824", "1081", "12760", "15610", "21410", "9801", "7645", "6901", "12376", "2016", "21650", "8364", "15595", "5212", "14861", "4506", "10908", "16878", "16988", "1084", "813", "12563", "18174", "2762", "17645", "4681", "4062", "25288", "7394", "831", "8589", "9725", "15161", "13553", "17366", "5190", "6271", "18984", "8770", "19382", "2852", "24709", "2502", "22242", "5835", "3007", "6793", "711", "15127", "1892", "16894", "15187", "15803", "19079", "1062", "2646", "10924", "21682", "8146", "23064", "7539", "15247", "5521", "16925", "15842", "23832", "2560", "7717", "9720", "687", "19480", "10056", "17750", "12397", "8395", "825", "10054", "18059", "19512", "3613", "13575", "8082", "14085", "9273", "5651", "9542", "9984", "15506", "21545", "13053", "15880", "13689", "23326", "2984", "25857", "8021", "22946", "555", "11477", "16151", "10799", "1149", "2304", "21430", "12969", "16977", "2824", "508", "24550", "14095", "14997", "11720", "4630", "23164", "21673", "13919", "22262", "22506", "17982", "16165", "21228", "6178", "9411", "25449", "9626", "24769", "7124", "20357", "7399", "4590", "24218", "2266", "20737", "16570", "18426", "1090", "3987", "13924", "19862", "7660", "2169", "12723", "3594", "23444", "17369", "13121", "9726", "7918", "12832", "4365", "10826", "6630", "6173", "2617", "10997", "25116", "11195", "11252", "5014", "13705", "22046", "2129", "19700", "13029", "3799", "3949", "249", "15772", "16373", "5056", "18643", "6685", "23776", "15105", "20482", "5267", "6773", "22677", "6669", "18175", "19723", "13467", "15866", "22634", "23356", "7584", "25544", "9497", "13695", "13198", "10126", "13208", "17408", "4115", "25406", "181", "24559", "3193", "23782", "25358", "15336", "11649", "561", "8038", "17924", "16787", "10555", "1279", "21657", "17656", "3355", "19334", "15202", "3104", "9975", "3022", "24062", "5812", "7267", "13037", "13700", "798", "25424", "9605", "20627", "18652", "19593", "19634", "5659", "26012", "930", "17627", "12772", "13688", "2089", "13767", "17625", "17611", "17303", "20297", "14017", "12989", "4513", "13166", "2619", "17957", "24234", "15868", "21940", "2660", "11191", "3895", "25061", "3054", "21038", "2255", "10170", "17522", "17839", "16138", "7305", "8290", "1388", "4044", "5443", "14831", "18087", "24034", "4849", "17056", "21789", "7950", "6807", "12893", "20786", "16562", "476", "20082", "8069", "23805", "13876", "2983", "25231", "9514", "22237", "10299", "11378", "22616", "3817", "3313", "23015", "7992", "23728", "8253", "5635", "4420", "5216", "25158", "12810", "8235", "14877", "8373", "13753", "14703", "22813", "22384", "637", "18590", "6182", "15299", "9619", "25104", "21053", "20981", "19681", "5616", "2609", "20306", "11414", "135", "1557", "9302", "19637", "6089", "21877", "17005", "1995", "16877", "20631", "15896", "18139", "2327", "3248", "15566", "7843", "7107", "20260", "13768", "3187", "19015", "14174", "8037", "21883", "10010", "10829", "21757", "3921", "16289", "13403", "4514", "9212", "22915", "2529", "21024", "25045", "22590", "7480", "11897", "15070", "11620", "24063", "19421", "16024", "25937", "17884", "6474", "8181", "12374", "18182", "22370", "23286", "6918", "25459", "13483", "22179", "25028", "15155", "19779", "19318", "15988", "3386", "11475", "7642", "9763", "18292", "21460", "18003", "17513", "681", "20362", "17127", "20812", "25433", "15737", "2725", "17809", "2113", "14696", "3666", "3816", "12610", "8424", "16703", "12975", "14569", "16305", "19985", "20084", "14681", "16322", "12594", "4203", "7597", "9131", "21336", "23353", "16066", "6400", "3184", "6421", "12962", "9166", "25339", "25807", "7698", "10176", "17711", "3434", "1461", "2022", "5938", "18858", "7906", "18972", "20740", "18037", "19127", "18941", "2050", "25314", "25167", "5837", "5895", "2330", "5782", "9144", "22808", "18775", "2551", "6932", "6389", "8516", "17168", "5360", "628", "16243", "11888", "21999", "8989", "11542", "18640", "21238", "10254", "893", "8602", "18607", "4189", "7339", "19209", "19309", "15248", "15540", "24675", "15264", "24301", "408", "12104", "12213", "16455", "11142", "13947", "1202", "24445", "19038", "24249", "16214", "12357", "10025", "4831", "23785", "23304", "24307", "25694", "17759", "2115", "16776", "1817", "5704", "9241", "3932", "14028", "516", "12659", "10154", "6680", "7737", "1672", "19383", "7191", "16861", "9226", "20173", "8889", "8339", "26024", "15377", "16975", "22006", "11623", "24524", "11617", "18121", "18588", "138", "20843", "20558", "17329", "23674", "2613", "8090", "19722", "14993", "9952", "13172", "7771", "3401", "7242", "10658", "1502", "11467", "16448", "7300", "23914", "13284", "18163", "9665", "7939", "16148", "22009", "18842", "23622", "10689", "13871", "21427", "3100", "25546", "12011", "4200", "7232", "6810", "10031", "15169", "266", "671", "10102", "22979", "17258", "14900", "25919", "13391", "17073", "9360", "25020", "7275", "25164", "21120", "20278", "7116", "20398", "2945", "10437", "8943", "8824", "1305", "16381", "10737", "14442", "10228", "2550", "7289", "12914", "12858", "1099", "12774", "11587", "25065", "10180", "1726", "12957", "10131", "23607", "8554", "1092", "864", "21082", "6012", "19883", "17786", "24310", "20323", "17118", "19320", "17094", "1972", "7733", "12399", "175", "10398", "2326", "2504", "18964", "11153", "21630", "17578", "15944", "22310", "17022", "6585", "11208", "10805", "25858", "16507", "21689", "594", "24573", "1881", "12141", "19080", "18515", "23559", "22371", "19040", "11811", "11260", "25746", "25768", "6792", "6330", "23933", "22206", "24332", "12853", "19869", "19507", "17067", "12705", "6148", "3422", "18448", "19843", "25840", "25068", "666", "14784", "8979", "20183", "25577", "13476", "3485", "20653", "6231", "11789", "25163", "15811", "18220", "13566", "15490", "8594", "14937", "8791", "9620", "16", "21641", "1527", "8974", "3074", "17162", "24314", "7876", "9070", "487", "10697", "11504", "24714", "806", "14383", "19465", "7284", "3801", "15056", "550", "10528", "19423", "9415", "14003", "11280", "11936", "20385", "12440", "7373", "13119", "17850", "18231", "13851", "6898", "21748", "3102", "16500", "4423", "441", "8465", "8007", "13552", "20844", "10305", "1855", "10561", "25581", "16109", "12971", "4524", "22163", "7233", "13448", "22309", "25702", "19499", "18015", "3048", "12545", "22778", "7058", "4547", "21581", "2659", "984", "18339", "22929", "8285", "2376", "20301", "22169", "17161", "168", "7168", "17384", "11404", "6397", "11693", "16440", "18790", "16198", "3179", "15474", "11249", "7328", "17122", "17523", "23159", "22240", "13330", "2158", "18276", "8079", "10145", "5175", "20796", "20815", "7544", "4265", "15131", "21914", "22501", "9170", "8524", "12425", "12571", "2599", "4324", "10843", "6001", "13701", "15214", "10340", "8248", "7997", "24036", "24790", "19155", "18693", "21775", "15954", "11652", "13167", "23774", "14392", "3150", "7360", "4785", "16265", "12606", "1154", "7714", "12496", "25186", "10570", "1979", "4276", "16612", "20903", "24091", "6308", "18597", "7132", "14969", "889", "4433", "22720", "14989", "25598", "5686", "9792", "3327", "11479", "12576", "18769", "7643", "814", "14741", "12717", "2892", "13194", "20089", "3640", "6639", "6707", "24244", "24857", "5625", "24462", "24647", "4413", "4808", "4624", "176", "5248", "13327", "1313", "14314", "2424", "8994", "7121", "20842", "21593", "24202", "17060", "13800", "6239", "292", "13896", "9506", "4702", "14411", "5062", "21932", "25542", "10055", "18373", "5161", "15472", "2178", "25831", "6049", "19336", "1433", "3237", "1999", "3095", "20923", "25814", "19462", "728", "4202", "16932", "8687", "22195", "13834", "7604", "7281", "16853", "17698", "23879", "12527", "23334", "1587", "17277", "16376", "5372", "9200", "10958", "2755", "16638", "2267", "11874", "22680", "1800", "1607", "7464", "5517", "14317", "12361", "21444", "16390", "15692", "6224", "11149", "14581", "7621", "4941", "3113", "2566", "5033", "17586", "15601", "7487", "5845", "7860", "25196", "186", "13255", "1332", "24155", "15254", "25418", "16394", "6598", "19551", "8507", "8347", "25771", "10371", "19279", "7607", "23397", "19414", "1410", "6858", "9104", "11733", "3868", "3862", "4069", "17429", "15417", "18071", "24904", "11756", "21853", "10706", "14118", "14015", "9120", "23291", "3370", "10956", "1511", "5883", "25191", "14379", "19826", "24939", "19832", "11578", "18289", "19438", "14918", "357", "18460", "20583", "25302", "1254", "727", "20355", "14680", "18186", "10153", "1947", "13606", "25276", "21561", "18965", "6110", "2417", "2768", "4172", "20579", "6051", "264", "8644", "6117", "9570", "9261", "2024", "14231", "5660", "8403", "12143", "11674", "1763", "12960", "24151", "8983", "24233", "19984", "20103", "18833", "2355", "15323", "11712", "8599", "13715", "11672", "25445", "16857", "21198", "709", "17029", "24755", "18363", "10332", "8107", "11948", "2437", "13475", "7509", "22878", "4199", "10903", "16107", "6090", "21679", "24418", "19246", "1329", "23643", "24540", "24635", "21162", "24477", "22401", "19132", "1180", "3834", "14819", "18720", "22493", "18383", "19905", "13761", "23305", "5867", "2497", "10769", "18264", "14674", "13774", "14355", "3213", "3917", "7564", "5779", "25659", "1774", "18925", "683", "8855", "12058", "12378", "15678", "15801", "10581", "8274", "11283", "9546", "25607", "7410", "3122", "9378", "10353", "21039", "12405", "22898", "24065", "24607", "853", "16784", "13161", "13794", "22107", "24935", "7662", "9491", "16155", "25533", "10971", "3290", "15150", "20433", "7845", "11884", "5149", "3292", "7081", "23898", "8586", "24685", "12041", "20723", "553", "7603", "2902", "4323", "10615", "7098", "13337", "21185", "17375", "875", "24719", "9935", "8140", "11302", "25719", "13580", "20446", "1984", "12159", "6324", "13180", "4859", "11086", "18921", "24347", "2188", "12748", "6044", "7705", "24074", "21672", "16679", "21109", "2002", "10920", "3480", "8192", "12069", "4313", "12223", "11133", "12691", "22454", "11148", "2486", "16807", "20676", "20918", "20828", "10970", "15890", "16560", "6927", "25062", "10562", "13697", "3672", "12393", "2145", "12625", "23818", "11473", "339", "13614", "1959", "10941", "10206", "13507", "20837", "11737", "21090", "21463", "7392", "12209", "5567", "14577", "2798", "10232", "1032", "7419", "18731", "6880", "21084", "24271", "2704", "13439", "17732", "12003", "6350", "12217", "8926", "17463", "24895", "9673", "25551", "13972", "5741", "9186", "4461", "19298", "2225", "14524", "16992", "3644", "25265", "15375", "1716", "4530", "1867", "18877", "16556", "22194", "15998", "4566", "4266", "22991", "1184", "8928", "3741", "8545", "11285", "9578", "24339", "22546", "25484", "12368", "16029", "20684", "22625", "23786", "10702", "25942", "2778", "19917", "16403", "6586", "5698", "9098", "25898", "4152", "10420", "8548", "14528", "2186", "12168", "11875", "8724", "22957", "9766", "5274", "11841", "19679", "1567", "25586", "5089", "2363", "4007", "18297", "4993", "14837", "4457", "10752", "9708", "10937", "4755", "974", "18254", "1079", "24139", "11797", "21983", "19304", "14974", "20135", "3151", "9224", "9774", "5695", "9848", "16240", "17770", "3554", "9038", "24358", "22344", "23721", "20674", "23549", "8792", "1494", "4921", "19373", "8207", "12967", "23747", "7725", "1847", "24843", "9782", "3036", "15920", "5075", "20859", "3997", "2080", "3525", "4845", "13808", "8500", "8873", "11022", "9271", "25250", "21982", "25899", "1799", "18247", "8533", "19446", "12146", "22294", "13685", "20556", "13642", "6507", "13548", "2654", "9764", "5136", "8267", "1929", "1391", "389", "11214", "19056", "10988", "3664", "11470", "18881", "238", "12269", "5073", "390", "22746", "12896", "22703", "12635", "2766", "13217", "21597", "14018", "5514", "23646", "17562", "3508", "11535", "25864", "4277", "15367", "2406", "15567", "7657", "12287", "23906", "10086", "5946", "25423", "8579", "3837", "4704", "745", "8113", "22263", "10499", "17987", "6015", "6942", "25888", "22674", "6430", "21986", "10375", "2367", "2935", "22792", "2062", "496", "5541", "8894", "19953", "12294", "466", "20574", "21692", "10241", "8903", "7379", "16234", "22873", "21128", "14643", "21093", "9195", "4379", "14772", "12244", "20000", "10800", "21681", "6718", "6158", "20710", "8672", "11946", "17247", "19429", "15516", "4685", "17424", "20857", "4586", "25834", "8874", "21437", "19090", "21735", "23806", "1199", "24889", "11039", "16111", "17883", "2559", "922", "4715", "3328", "20632", "6256", "8753", "25320", "8977", "7583", "11571", "16892", "6122", "11045", "16465", "3092", "24304", "4180", "11355", "19553", "4374", "534", "23915", "7050", "11931", "8555", "18914", "11869", "3788", "1212", "22133", "5335", "22414", "19500", "23096", "25606", "18699", "16490", "19584", "2807", "19959", "5511", "24917", "10700", "8478", "16944", "17776", "23647", "22124", "6621", "1980", "14235", "2711", "15974", "6161", "15265", "1193", "7431", "4008", "7819", "10", "3431", "24490", "5510", "3220", "16634", "16476", "19241", "1626", "1284", "23297", "19167", "19231", "12710", "18897", "24566", "10050", "6753", "20621", "11229", "10066", "21421", "16347", "146", "1593", "4125", "2220", "6683", "7502", "24970", "2043", "21408", "15810", "2007", "18622", "25981", "22351", "21073", "12800", "12811", "20042", "15373", "5802", "968", "24729", "12137", "17468", "1471", "16008", "15313", "15571", "11181", "19166", "3195", "9990", "1357", "19969", "13967", "23684", "14609", "19544", "11982", "23482", "21781", "2695", "770", "14700", "21458", "3789", "10987", "22706", "2112", "11579", "25536", "8884", "15123", "1870", "9925", "2160", "800", "9204", "10255", "12876", "10957", "11108", "13645", "10271", "21607", "231", "19971", "4643", "11773", "10109", "8071", "22520", "20651", "721", "13981", "15568", "10468", "5298", "22389", "8569", "8811", "4816", "18685", "12320", "2989", "10043", "5496", "7366", "10802", "3192", "17622", "23800", "5046", "3590", "11112", "15902", "3546", "20823", "2391", "12370", "10037", "17244", "423", "9666", "24211", "6222", "6986", "7177", "17571", "14273", "13478", "10527", "12165", "12428", "22718", "9982", "18137", "14377", "5566", "11721", "1207", "20051", "15873", "18084", "1401", "9067", "3582", "13249", "8780", "17321", "6908", "20750", "11100", "12338", "15394", "23438", "946", "9878", "15369", "9811", "19738", "11557", "1026", "18993", "19840", "7057", "22784", "2720", "22787", "11016", "11774", "2675", "17765", "14393", "19967", "24681", "15324", "20559", "4096", "2097", "18976", "5613", "23165", "2340", "10458", "4581", "4181", "15771", "19472", "5957", "15317", "5515", "21191", "1164", "6297", "4777", "22789", "3068", "4839", "18962", "11498", "7087", "1118", "13447", "12918", "18408", "4085", "19023", "13593", "7629", "2166", "20360", "5288", "21601", "8188", "12006", "16966", "4075", "4568", "12689", "23796", "5370", "7093", "6649", "14352", "2811", "1869", "16991", "18376", "25837", "23768", "3981", "5495", "8126", "6915", "14899", "21715", "1787", "23227", "25304", "10936", "691", "2357", "7091", "25999", "881", "2512", "11358", "10118", "9914", "23137", "25579", "19074", "9039", "20309", "4952", "22861", "11336", "8455", "345", "6255", "19854", "5979", "11717", "11770", "459", "12557", "7157", "14504", "23106", "5329", "6884", "20749", "1251", "10962", "4114", "11367", "18092", "5254", "15055", "24141", "18600", "563", "25741", "10283", "22234", "8626", "14065", "17823", "7773", "8917", "11575", "15600", "22482", "11373", "15269", "24140", "20119", "4259", "19968", "20824", "23315", "7436", "22553", "19525", "4592", "3597", "17501", "14927", "5784", "16889", "13096", "16031", "9945", "11480", "8854", "23887", "16248", "23686", "14790", "10836", "14604", "22610", "16879", "8512", "12001", "2845", "25269", "25035", "9040", "6269", "3199", "20549", "13358", "9837", "15918", "17751", "18213", "14962", "10606", "19434", "21188", "10022", "17959", "24867", "11262", "13224", "21259", "23450", "5119", "22055", "3311", "12498", "9664", "17760", "17872", "14449", "21445", "9141", "24982", "9916", "4118", "13114", "15736", "20863", "12549", "1403", "21442", "4889", "5432", "2480", "19264", "782", "11814", "9465", "22762", "21121", "19207", "23119", "5063", "16791", "8821", "7292", "4378", "9077", "10980", "8401", "18253", "13997", "24877", "16271", "24989", "283", "18922", "23572", "17762", "12042", "8170", "10934", "18104", "15220", "7535", "14168", "4388", "11925", "4066", "1697", "9903", "14766", "11293", "322", "10624", "9735", "1990", "17139", "18753", "3733", "25695", "7355", "11493", "15985", "22388", "23993", "20587", "5436", "7701", "24329", "3413", "1237", "7155", "184", "3423", "19001", "22024", "23905", "4762", "23300", "3132", "7709", "17705", "7989", "11077", "8519", "8", "9879", "23798", "24412", "2490", "10257", "7297", "7043", "1394", "13674", "7142", "18760", "11780", "2781", "21112", "11902", "15087", "13668", "12433", "940", "3926", "3913", "2584", "19900", "24526", "9976", "8398", "8446", "22540", "23468", "6038", "23937", "10901", "4185", "18863", "7746", "8637", "25326", "5816", "15452", "14338", "16732", "14975", "3706", "4866", "10605", "15824", "7348", "17929", "22243", "16346", "11785", "12862", "24327", "20338", "25610", "232", "21478", "267", "16093", "9304", "20575", "13694", "24552", "14804", "24483", "6485", "7059", "10331", "19469", "97", "13060", "20907", "12869", "20", "8449", "9938", "23511", "5814", "12998", "15093", "10007", "8452", "17780", "14599", "1367", "23413", "8936", "285", "8963", "25687", "20010", "6531", "4267", "9727", "8742", "24574", "718", "24500", "21528", "4425", "22034", "6142", "5805", "17066", "25847", "22035", "298", "1689", "10787", "17547", "12022", "25362", "1816", "2919", "16418", "7255", "1243", "14941", "21497", "6043", "24931", "5868", "9389", "16834", "19220", "10863", "1917", "10981", "15277", "14505", "14906", "13739", "10814", "26018", "16943", "2793", "21909", "16775", "2219", "16781", "15978", "18340", "6614", "9001", "3869", "25011", "21253", "13917", "15521", "17575", "14423", "18647", "22405", "8234", "6442", "10172", "8303", "8156", "12214", "24335", "16990", "10801", "23975", "16694", "1722", "23488", "2136", "20201", "23459", "23182", "24302", "16280", "362", "10122", "8518", "8992", "22981", "5540", "3771", "25117", "3624", "15253", "8201", "11331", "11821", "16699", "11067", "9824", "24469", "21635", "12535", "24641", "11364", "19251", "9374", "11007", "7553", "883", "295", "6234", "12082", "6862", "11901", "24345", "14787", "12739", "15005", "7808", "4700", "11565", "5487", "17107", "4190", "12822", "1565", "16734", "21465", "18979", "18342", "5692", "5854", "2182", "14172", "25474", "22490", "21629", "23727", "9529", "24061", "17478", "14506", "15140", "22249", "2971", "7637", "19841", "1595", "12952", "25525", "4612", "3296", "25411", "18978", "8745", "20329", "21704", "15839", "18236", "18689", "6876", "10046", "10150", "22935", "15860", "16241", "8257", "24797", "19708", "23710", "24944", "12492", "12870", "1740", "14488", "9793", "12736", "14414", "18149", "905", "7726", "3947", "8706", "25021", "11092", "21563", "25969", "9202", "20370", "6541", "1501", "10303", "7913", "8796", "17017", "8709", "20048", "7874", "10725", "6099", "20763", "7881", "22499", "11859", "16921", "6461", "25140", "21539", "7803", "22513", "20840", "15082", "212", "12000", "12351", "24071", "4497", "17101", "3395", "23036", "19081", "16833", "14707", "2672", "10865", "13752", "19454", "7846", "5381", "23954", "24400", "20389", "18801", "9562", "5427", "17464", "19439", "22228", "865", "20340", "4373", "187", "2418", "6226", "3120", "1120", "10314", "2519", "9737", "11569", "762", "17488", "8929", "16735", "7021", "1915", "19345", "20171", "23515", "21882", "3859", "18335", "6092", "6886", "14701", "9585", "19398", "7770", "14631", "18090", "1108", "3681", "618", "17576", "6443", "24286", "16377", "17317", "20043", "15379", "22076", "14160", "20123", "10387", "8454", "21534", "1725", "13394", "20922", "22686", "1720", "19890", "2328", "21136", "12532", "25583", "24134", "24547", "3291", "9134", "14516", "7520", "19267", "217", "3319", "15320", "11436", "1919", "14006", "12315", "24582", "25377", "16631", "20688", "24849", "16298", "7787", "1579", "13477", "13718", "15153", "23682", "13523", "14469", "21972", "1225", "23952", "12696", "16750", "12431", "14030", "24007", "19342", "4134", "14755", "786", "21758", "13341", "22530", "21475", "3585", "12301", "1051", "1475", "21193", "13352", "1835", "13517", "15792", "4100", "15703", "19662", "25153", "11919", "22093", "3268", "1085", "12230", "7755", "18352", "24968", "2307", "11311", "11560", "1760", "21064", "15525", "9156", "2118", "15937", "6552", "22434", "11205", "3615", "578", "12788", "13578", "20479", "20783", "6351", "1536", "21653", "17618", "4693", "7023", "3708", "21687", "3062", "5153", "15706", "19083", "2546", "7456", "13154", "11596", "18357", "19535", "316", "11893", "7115", "4049", "21403", "8365", "12955", "18064", "1680", "13837", "8886", "25535", "15048", "15805", "17842", "24247", "25658", "765", "10219", "290", "12886", "17878", "14258", "9349", "25735", "5017", "20904", "23355", "2400", "1605", "24381", "25929", "10558", "9457", "3712", "4218", "9893", "19538", "15262", "5818", "3219", "3993", "6803", "17768", "16896", "4619", "10676", "15130", "12169", "4366", "25743", "17323", "4562", "12052", "7106", "6910", "6246", "22359", "17068", "1599", "9693", "21501", "10225", "4666", "4435", "888", "1017", "17615", "19", "15809", "5442", "24038", "10149", "16453", "25910", "17", "10235", "16838", "17236", "18592", "5863", "23838", "16000", "19666", "25439", "11572", "16235", "14517", "2620", "6875", "13810", "13245", "13758", "7900", "17811", "23843", "4153", "5339", "14636", "12050", "12387", "245", "2247", "1737", "25724", "24630", "15483", "24187", "11836", "8939", "9306", "20562", "21343", "16434", "2401", "1191", "21484", "6100", "4679", "8141", "9717", "24341", "14759", "10519", "9798", "11127", "3402", "21572", "21080", "24546", "19991", "23542", "981", "21286", "4983", "7385", "23947", "10125", "15142", "25425", "18918", "5851", "19724", "12646", "14269", "137", "11377", "13946", "25788", "7368", "22545", "25488", "2319", "4236", "19727", "15626", "11441", "18614", "20055", "6465", "9765", "24395", "1702", "24886", "15628", "5156", "23162", "24456", "24464", "18546", "20250", "25470", "16632", "4419", "14892", "20096", "20154", "24901", "9777", "24396", "16718", "19184", "19020", "14739", "3694", "3765", "20354", "9128", "25865", "18619", "15762", "15386", "21628", "26013", "11201", "9368", "23653", "23966", "9582", "6965", "5539", "16404", "2313", "20191", "2602", "14656", "18466", "7235", "23229", "4483", "23079", "17441", "16489", "7303", "3197", "9121", "993", "22087", "14522", "3259", "13213", "1604", "10751", "16335", "18022", "22382", "4509", "994", "20034", "6599", "11714", "25547", "2435", "7340", "13570", "6955", "25940", "18252", "2015", "13055", "16814", "2395", "23605", "7097", "24391", "13059", "14230", "1842", "2429", "19288", "13123", "2662", "24820", "18855", "12098", "10210", "12841", "24129", "79", "1389", "14802", "574", "22860", "22714", "9956", "14849", "1078", "15644", "9442", "12489", "7341", "1113", "22419", "18686", "4847", "14192", "13290", "19582", "10366", "23568", "23221", "25679", "15126", "112", "714", "24484", "25737", "8379", "20109", "2280", "17458", "21918", "6514", "18676", "12075", "13482", "16230", "22829", "23429", "19039", "17262", "25923", "4533", "18319", "21746", "13221", "2814", "24735", "3174", "1669", "198", "13534", "12908", "1989", "16629", "5291", "13254", "413", "16771", "18419", "23424", "25329", "3809", "19218", "9007", "13142", "17049", "18866", "835", "11663", "10963", "6795", "4060", "20223", "11417", "25281", "24915", "24372", "19405", "3439", "19275", "14557", "21166", "19613", "20151", "17001", "5880", "23088", "5383", "17828", "15843", "16523", "116", "3385", "3890", "7822", "1983", "2265", "1992", "21902", "22601", "6462", "17727", "22515", "20396", "25187", "5533", "14961", "11530", "2803", "12586", "22406", "23538", "9905", "16559", "7754", "24144", "3970", "19293", "24519", "7460", "934", "10223", "12191", "15088", "10478", "9564", "1963", "12587", "144", "5715", "7126", "25669", "5069", "25566", "15882", "2598", "3445", "7521", "7972", "19669", "21486", "24491", "21420", "6659", "16307", "24656", "14671", "20586", "20772", "5239", "17758", "18107", "13890", "2433", "4249", "2868", "12865", "17972", "16047", "18879", "7085", "16195", "15635", "17778", "21195", "20149", "5677", "7291", "1666", "12237", "3348", "6445", "25074", "23865", "18098", "21502", "3639", "18803", "16604", "9448", "17912", "21953", "6301", "24958", "10845", "16459", "17838", "24079", "18265", "21658", "11898", "1656", "5679", "21753", "5796", "5726", "10729", "7742", "6391", "3649", "24204", "4247", "5878", "13717", "2767", "6959", "8653", "20916", "13710", "1743", "14590", "15645", "19675", "4912", "14585", "15650", "3734", "18002", "25145", "16900", "7789", "1758", "1624", "22100", "21777", "7338", "18178", "1655", "6891", "11961", "2317", "12400", "10175", "5987", "2211", "26009", "18824", "19311", "25594", "19870", "14623", "20443", "10864", "5571", "214", "14695", "22392", "9029", "24699", "12303", "12130", "18167", "10718", "463", "13683", "22796", "22268", "9481", "19226", "19951", "10052", "1640", "2299", "5279", "7665", "20094", "20795", "22667", "12554", "16997", "4658", "5974", "2312", "1623", "14719", "20457", "21825", "20285", "23654", "19915", "24081", "11206", "13152", "11060", "4285", "7122", "13884", "10424", "23198", "2290", "14440", "3630", "2734", "9233", "21691", "25036", "8952", "12280", "7857", "10391", "12759", "9617", "10701", "8901", "26", "10922", "23752", "19268", "18102", "12444", "10557", "19709", "2058", "14412", "21262", "14462", "23400", "23822", "5031", "12317", "11263", "21372", "6805", "19146", "16856", "21939", "903", "25663", "5569", "22534", "19068", "1311", "7611", "4485", "12906", "10747", "14416", "11726", "619", "8820", "11186", "7510", "12783", "13847", "8782", "23207", "982", "9021", "8433", "16982", "202", "2661", "22976", "12699", "19175", "1277", "8271", "23097", "6921", "420", "23320", "6809", "9646", "18263", "12162", "6830", "2356", "18821", "3691", "18969", "3073", "12132", "19636", "14271", "22274", "1765", "4333", "24509", "20353", "10529", "25982", "14296", "23655", "10406", "10212", "17492", "23376", "2805", "8131", "19731", "4781", "9773", "5053", "4713", "15993", "690", "11546", "8676", "19430", "7752", "11380", "5459", "23332", "18854", "1423", "5282", "9778", "16850", "21718", "6200", "6002", "2881", "18718", "15515", "23381", "22068", "3864", "6285", "25769", "18668", "19098", "25086", "22650", "1358", "7877", "8391", "13348", "23839", "6550", "8258", "23063", "3668", "160", "22347", "2402", "6851", "14659", "10085", "5609", "22805", "4255", "1748", "11319", "4278", "23863", "20014", "20974", "11769", "9071", "15518", "18381", "25408", "5687", "8622", "16601", "10588", "13561", "8741", "22658", "11264", "18058", "15759", "13143", "15347", "11370", "3189", "19990", "13085", "10298", "11389", "14777", "6860", "6279", "1782", "18657", "15177", "2240", "9323", "10080", "1319", "12539", "13463", "20167", "7314", "12447", "1385", "18258", "7727", "6758", "7812", "7650", "16681", "24410", "9427", "464", "11632", "2691", "15226", "13503", "17295", "1042", "10691", "23830", "5793", "15502", "23473", "23289", "15060", "13675", "25532", "3722", "2557", "21426", "1859", "7670", "11361", "3562", "1895", "3831", "21863", "3001", "23744", "7173", "1696", "24927", "25985", "24553", "13601", "15930", "16137", "15499", "560", "15006", "16920", "13199", "7161", "7315", "1052", "25245", "22547", "23292", "18971", "12777", "10677", "10443", "18733", "3320", "25099", "15555", "22354", "17692", "23639", "14424", "21905", "8932", "15539", "9769", "22984", "6899", "17410", "11984", "21299", "2573", "14964", "20324", "19378", "5750", "14867", "23957", "4804", "16089", "24955", "11129", "302", "18099", "7743", "3473", "243", "15387", "11778", "8868", "19331", "24667", "1152", "25455", "21477", "7946", "13328", "18853", "7077", "8193", "3718", "827", "5109", "6130", "8169", "18482", "4676", "22739", "428", "21434", "8935", "18299", "25331", "24712", "20577", "18181", "20537", "6743", "17333", "47", "311", "10505", "25378", "14547", "19975", "23208", "21316", "12786", "12812", "16777", "8598", "11526", "12074", "14247", "6938", "20249", "4068", "6879", "25482", "1482", "2707", "21324", "16654", "23074", "2215", "9256", "9843", "15166", "7219", "21380", "9702", "17829", "25039", "25651", "12789", "8921", "18046", "18494", "8673", "5419", "6894", "25681", "17885", "22931", "15528", "18451", "20429", "9950", "11362", "19688", "349", "24623", "15716", "321", "18751", "70", "24844", "15548", "24780", "19322", "9926", "4632", "20363", "1602", "10378", "15532", "4143", "9369", "9278", "15393", "22276", "18936", "14077", "7661", "14596", "14888", "19093", "18089", "5196", "25584", "13456", "4654", "9791", "12383", "3297", "7778", "23840", "20464", "8315", "18450", "19407", "17286", "1065", "1415", "1738", "5956", "3107", "25192", "14509", "15967", "5251", "12391", "19520", "2371", "2578", "10167", "21099", "3396", "1840", "7708", "25064", "22064", "5068", "24907", "1622", "10859", "7973", "19627", "9876", "14019", "1752", "11145", "22843", "23791", "7451", "10682", "3833", "12102", "17590", "11169", "18421", "11489", "17846", "18572", "9713", "24176", "24679", "11268", "11211", "13343", "5405", "12744", "23773", "12507", "26023", "10344", "25454", "5192", "16154", "21368", "20244", "6469", "22928", "14444", "8685", "23150", "17190", "13749", "14527", "23759", "13508", "22254", "16826", "22043", "12782", "23407", "11020", "13720", "15023", "12640", "24133", "6850", "12477", "16917", "13652", "19250", "17574", "1469", "20252", "25880", "784", "18611", "8646", "17496", "14437", "25569", "16360", "24394", "5447", "23100", "3716", "22314", "3429", "11359", "6314", "5247", "4319", "14196", "10695", "14959", "19277", "7504", "3968", "5445", "8999", "23176", "24743", "9794", "1862", "15072", "14597", "7376", "17376", "5268", "1231", "8132", "15329", "14552", "4897", "23616", "24003", "16662", "41", "10670", "14089", "1634", "19051", "3750", "4505", "1126", "20553", "24782", "7092", "20698", "3367", "25365", "10684", "19948", "9108", "4124", "22529", "4732", "5030", "6068", "8838", "22713", "10549", "4097", "21087", "20532", "18673", "140", "20735", "7455", "11808", "15080", "25740", "12117", "8890", "7183", "24648", "24610", "24228", "12629", "8291", "21850", "8754", "8083", "17580", "4974", "15735", "14840", "17803", "12479", "15845", "13117", "11279", "7304", "25054", "19939", "16972", "9967", "6424", "25485", "8634", "15925", "24902", "15035", "11151", "1181", "9424", "17186", "2731", "16760", "16643", "13165", "6672", "17717", "3916", "3218", "5860", "17457", "6141", "25", "3349", "2576", "25354", "14025", "11782", "3621", "5444", "10772", "12536", "10584", "7356", "12485", "16907", "13957", "3521", "9459", "22445", "3046", "8477", "14540", "24325", "17327", "11076", "22973", "13786", "18222", "15012", "8885", "6101", "6434", "1111", "16324", "5193", "12885", "13350", "4637", "12938", "2473", "11831", "6811", "14405", "25018", "25121", "15986", "1662", "7206", "16639", "18152", "17558", "4003", "25014", "8688", "2773", "9099", "4173", "15670", "25975", "1257", "5831", "11494", "23149", "25823", "12009", "14045", "7749", "4356", "15616", "14222", "13156", "2531", "7354", "8171", "3343", "17747", "10201", "9784", "7941", "7270", "23385", "19876", "9559", "5526", "8390", "18228", "25460", "11855", "7608", "23487", "16212", "14858", "17027", "9392", "15200", "15620", "17360", "25538", "3531", "2279", "3024", "9486", "5026", "5364", "22201", "20721", "23430", "15359", "24534", "20799", "8862", "9608", "4606", "15581", "12438", "21492", "11503", "25979", "23547", "18988", "21081", "5146", "14814", "11580", "1769", "8892", "4009", "25219", "20410", "810", "13511", "14760", "14783", "4414", "20246", "14108", "19035", "24028", "15148", "3829", "15002", "13399", "20992", "2110", "13210", "16646", "12318", "11491", "13058", "25201", "4436", "24615", "13987", "5449", "23552", "10282", "6721", "12567", "1349", "24308", "19124", "7001", "17437", "5981", "22400", "25956", "3780", "7945", "13873", "16580", "10781", "23141", "15420", "10419", "6527", "12505", "14382", "10766", "21814", "20882", "15186", "19308", "25571", "12735", "12002", "15346", "4520", "18136", "10889", "2714", "22305", "8366", "21077", "25410", "9004", "17070", "1886", "12927", "17036", "6957", "15040", "21843", "13310", "13002", "13043", "1253", "2147", "7769", "11798", "24426", "24105", "7811", "23456", "15416", "14638", "13238", "16184", "9749", "1856", "3436", "19698", "19295", "13687", "5456", "2027", "24259", "12958", "17864", "11427", "4297", "14955", "24689", "3576", "1413", "1368", "3238", "21839", "17149", "18498", "4399", "190", "22630", "8211", "14747", "9446", "20730", "7748", "16432", "22517", "6833", "20157", "23613", "94", "15463", "4830", "22543", "9776", "2530", "1354", "8828", "549", "150", "9831", "22562", "5050", "16825", "3905", "20085", "8746", "11012", "16018", "24417", "16208", "13740", "643", "21674", "19294", "1710", "9265", "25060", "5788", "8421", "301", "18105", "3629", "13993", "24707", "13906", "7685", "692", "9612", "2447", "17158", "24045", "9577", "5113", "1112", "17102", "19508", "4163", "22906", "4204", "6693", "21018", "9337", "11515", "12375", "17389", "10369", "7307", "14193", "22943", "10653", "13904", "17752", "6975", "8744", "1185", "2667", "24320", "5914", "25643", "233", "10019", "3249", "13077", "23143", "15731", "10362", "19374", "12177", "13232", "14818", "24826", "6520", "17706", "15079", "25352", "17270", "5909", "25038", "3989", "1343", "13885", "1038", "19558", "7854", "2772", "13018", "4683", "22737", "5207", "10746", "10735", "21855", "17935", "16166", "2176", "23446", "22114", "3108", "25821", "6994", "6096", "15591", "22732", "872", "22074", "3459", "15482", "21168", "10197", "24585", "21457", "11767", "22518", "20192", "18204", "7170", "15593", "10665", "20968", "9594", "6207", "1116", "9758", "7573", "12239", "611", "608", "7246", "171", "14485", "23181", "20032", "25026", "19612", "19897", "4835", "12349", "6343", "25791", "12246", "16264", "16685", "20046", "6644", "208", "3227", "275", "794", "11517", "14925", "12624", "19158", "8865", "13021", "13184", "21625", "23295", "19163", "455", "24883", "4610", "4940", "14203", "8798", "20005", "17358", "18538", "5198", "25674", "19623", "1239", "25859", "18261", "10336", "11497", "620", "25836", "11713", "10855", "25079", "13934", "8661", "3500", "13923", "20705", "501", "6676", "1441", "22015", "21981", "21943", "18711", "20364", "22641", "9135", "24521", "9335", "21294", "20141", "12451", "12394", "15536", "10152", "7403", "17097", "8125", "6201", "21993", "25327", "1466", "14198", "7295", "23827", "5862", "10200", "21419", "7226", "17598", "5985", "17310", "21041", "22655", "26016", "4086", "10619", "24930", "4914", "18218", "14087", "8714", "14039", "21571", "13996", "7430", "24033", "21741", "8743", "20777", "16092", "6947", "8164", "4427", "12542", "24734", "1156", "7273", "3200", "12565", "18219", "15125", "8538", "4588", "10878", "7201", "21172", "9960", "10472", "24739", "12429", "9345", "21371", "918", "1922", "6394", "6333", "25294", "12175", "25295", "6263", "5479", "21667", "14848", "18528", "21519", "22876", "5396", "6016", "19187", "20248", "22731", "1684", "22637", "7458", "23956", "25958", "6924", "20293", "4036", "3474", "900", "4037", "10741", "16190", "18558", "21921", "8024", "15467", "25978", "11347", "5317", "16483", "10474", "24785", "24592", "23406", "5463", "23851", "18730", "23122", "25098", "18023", "22593", "19519", "4240", "11400", "23496", "8947", "9985", "9545", "2738", "25876", "16795", "9331", "18542", "17018", "7632", "11908", "5623", "11366", "24621", "24258", "23760", "25872", "4345", "457", "20379", "9826", "14433", "8490", "1690", "25808", "23949", "2988", "22326", "1039", "19875", "2515", "25930", "25514", "22129", "7334", "9637", "21225", "15893", "4717", "1988", "5178", "22146", "23719", "1871", "9543", "4873", "25323", "11046", "4776", "885", "16891", "16579", "10510", "18308", "19552", "9194", "10546", "8800", "21506", "3534", "22776", "8212", "4891", "21170", "10120", "7541", "25989", "25661", "13643", "15931", "2432", "15212", "25147", "13469", "19934", "2750", "15604", "14125", "7840", "305", "3425", "24661", "21874", "22287", "2063", "10583", "6587", "21851", "22544", "16449", "14743", "7938", "22453", "14127", "14714", "599", "8384", "15210", "8970", "15203", "15480", "9176", "19011", "23983", "6457", "21945", "21867", "3191", "17661", "14640", "626", "15213", "13370", "17385", "491", "25397", "24094", "5656", "257", "12561", "11621", "19024", "18938", "19066", "5876", "20834", "18847", "2203", "12064", "8701", "2294", "10504", "25642", "14617", "4293", "8481", "2677", "11735", "8249", "20194", "15786", "11746", "17869", "5984", "10846", "18792", "17696", "16801", "9165", "3468", "6854", "25677", "10912", "5501", "16302", "3856", "23758", "6377", "13186", "7478", "25955", "17840", "18613", "14720", "11976", "20704", "12658", "24444", "1364", "20474", "21359", "9051", "4288", "73", "6673", "4101", "5641", "5767", "2702", "18707", "7316", "17798", "20150", "19245", "14388", "12118", "21592", "5007", "2339", "3455", "8420", "1613", "23854", "24628", "3745", "15410", "21313", "19445", "25964", "12631", "12010", "21495", "4472", "1479", "23057", "4794", "23224", "25773", "18641", "17928", "22818", "18133", "971", "6698", "3855", "10711", "15106", "10995", "7835", "20126", "9640", "13524", "7858", "9803", "8183", "9403", "10318", "6523", "13260", "8990", "13346", "7481", "24154", "21364", "8016", "9361", "2099", "7627", "17733", "3489", "16663", "11819", "25749", "1777", "327", "2345", "136", "24567", "20606", "8462", "6595", "24156", "7065", "7055", "22852", "26027", "2230", "17064", "12831", "5627", "5378", "6156", "8534", "19141", "24149", "14486", "3548", "5934", "12241", "22253", "8922", "7967", "10435", "22336", "7490", "18773", "7795", "8479", "23808", "13875", "16985", "23144", "8196", "1944", "24321", "12910", "14554", "14144", "15984", "5411", "11787", "1141", "24794", "10325", "4842", "12298", "2647", "5455", "11068", "25057", "538", "23146", "15183", "16788", "10543", "2196", "7537", "19034", "9941", "14371", "24162", "2874", "5345", "14219", "14105", "11040", "18750", "16470", "17676", "3021", "16640", "16397", "3152", "17461", "19122", "5098", "20227", "10173", "13883", "2954", "24115", "17019", "22390", "7766", "9357", "13956", "23896", "9724", "18513", "21279", "23621", "19057", "20725", "5173", "4539", "25053", "4597", "8814", "20753", "23924", "11682", "1190", "10323", "4343", "24379", "1031", "22797", "11000", "12120", "9456", "14000", "10576", "23378", "13324", "16730", "16238", "15401", "3667", "12541", "22040", "22192", "11534", "20599", "3136", "24070", "846", "22248", "16246", "3045", "24277", "24353", "14130", "21091", "4241", "4116", "6786", "10622", "9544", "21538", "5915", "18901", "18665", "16159", "18975", "18018", "14251", "2579", "13664", "19368", "16532", "2908", "16081", "3121", "22911", "4604", "7949", "18320", "14847", "6972", "14672", "11655", "6638", "3087", "8112", "12257", "10326", "17439", "8075", "23531", "7673", "22119", "21577", "16183", "24084", "3596", "19954", "3915", "15586", "4336", "2124", "15894", "24282", "13900", "22424", "139", "23124", "7075", "10404", "4896", "19426", "11365", "25500", "10817", "19570", "6004", "8941", "11256", "3972", "12993", "8897", "11683", "17550", "6655", "24472", "13367", "24040", "7500", "20898", "4302", "10542", "9900", "8756", "15053", "7690", "6615", "23219", "5603", "23845", "24493", "9185", "8094", "25285", "336", "24726", "8458", "18872", "14149", "17902", "20049", "9583", "9762", "12973", "21386", "21900", "16491", "18770", "25119", "1303", "5509", "15292", "8430", "19842", "23940", "20029", "8041", "21328", "6476", "13449", "24057", "2932", "1585", "5219", "22824", "9326", "5598", "16407", "5322", "3058", "20754", "14727", "22147", "14536", "18675", "11091", "20961", "20214", "7298", "15184", "23263", "15757", "11485", "2740", "22570", "12719", "20775", "23301", "10274", "12135", "10871", "25135", "20716", "23369", "8286", "13716", "1098", "1874", "8331", "12915", "20342", "15818", "10523", "7762", "20470", "9949", "3874", "4633", "22507", "12864", "16290", "19836", "10281", "4855", "2035", "25185", "24578", "18584", "1750", "8605", "22996", "10849", "15614", "25128", "16253", "20347", "2343", "25004", "12369", "1775", "5409", "17450", "17500", "6616", "22807", "25271", "7495", "10768", "13205", "22342", "24786", "7022", "2955", "13852", "14266", "6380", "6665", "17721", "13379", "1601", "19420", "22649", "13555", "1664", "13106", "20708", "12729", "20344", "25432", "8581", "3306", "4883", "10238", "4355", "20766", "1287", "13841", "8960", "19302", "833", "7688", "14709", "12274", "23023", "12195", "3433", "8973", "13563", "6896", "15613", "10890", "4150", "24565", "2479", "123", "2485", "9937", "11024", "7361", "5099", "1615", "14062", "17820", "10381", "2248", "21964", "1584", "2605", "7529", "20642", "20821", "14058", "11928", "5685", "23485", "3727", "4405", "13690", "10388", "113", "20373", "16726", "9293", "23001", "18748", "14583", "21194", "24388", "25493", "20665", "2037", "1901", "25376", "19685", "13181", "22142", "1574", "14021", "13075", "19925", "1687", "22581", "2856", "5036", "24179", "11913", "2198", "23537", "12590", "14629", "18878", "7213", "16173", "16216", "24658", "17755", "2459", "22341", "8850", "1374", "6108", "6489", "24280", "15906", "1071", "7780", "15068", "13484", "3684", "17382", "1008", "6671", "15286", "9732", "23148", "3051", "21012", "12970", "11896", "9510", "10469", "10757", "23829", "12600", "3090", "3752", "3481", "13858", "5706", "3522", "19572", "20745", "10770", "5892", "325", "23803", "17173", "1056", "13471", "25100", "4040", "5160", "11552", "10182", "11609", "23469", "14981", "22421", "851", "3086", "9168", "3988", "17887", "11476", "19164", "20776", "18531", "17483", "1863", "21401", "16765", "18234", "9217", "566", "6190", "2025", "2790", "15579", "22554", "11093", "20302", "16186", "12551", "5166", "20388", "2759", "1134", "20677", "16540", "21354", "19930", "8178", "8177", "5828", "19211", "258", "25665", "21432", "25945", "4865", "1745", "21559", "898", "24318", "25393", "6367", "24757", "14902", "84", "17328", "25596", "15817", "1966", "11456", "23535", "25282", "15424", "17810", "5841", "1421", "16378", "12190", "23393", "15326", "23687", "19704", "9192", "2212", "18980", "14408", "4407", "17675", "13261", "19581", "10597", "10939", "23003", "12304", "8784", "5428", "15981", "13648", "18565", "1194", "17152", "10823", "24622", "12077", "17209", "19259", "14281", "23031", "4171", "23907", "12446", "21307", "22831", "20499", "8466", "13522", "16197", "5789", "6709", "2440", "1379", "12243", "18661", "21973", "2269", "3996", "7094", "7353", "58", "2652", "6452", "20861", "7588", "11957", "15102", "4945", "22870", "12324", "5736", "15039", "12926", "10471", "1100", "21772", "24978", "10973", "21866", "2441", "9110", "849", "18540", "12293", "5475", "13742", "15063", "3983", "72", "8683", "15223", "5756", "20607", "22591", "6357", "2681", "9509", "25676", "24569", "7776", "25138", "24104", "8012", "13201", "2517", "4384", "314", "15475", "6941", "19944", "2077", "22187", "5948", "4322", "15948", "6974", "19004", "23585", "9305", "22426", "22132", "20494", "7033", "17289", "8392", "21305", "12652", "13576", "23930", "14669", "20590", "3210", "23835", "0", "3954", "7322", "14921", "14286", "17670", "19788", "6902", "13702", "7824", "13499", "12347", "17601", "21930", "6133", "5968", "13259", "13743", "15652", "2543", "15727", "712", "10612", "24880", "23166", "1072", "18082", "22182", "366", "18825", "16536", "21756", "18140", "8806", "14812", "24744", "24702", "22954", "2716", "117", "4551", "919", "22216", "19712", "8356", "21347", "25953", "9262", "573", "6579", "17666", "2777", "11647", "565", "332", "11959", "2595", "6338", "10020", "25383", "211", "19428", "12568", "2444", "4627", "1751", "12412", "4005", "20269", "14463", "25142", "23730", "23720", "8415", "9687", "12115", "6292", "2680", "18551", "1334", "10803", "3196", "23586", "16454", "10493", "22010", "13353", "21568", "18120", "7952", "1380", "22856", "22698", "16751", "10975", "5712", "7", "15776", "10209", "13103", "10951", "23592", "3724", "3448", "15448", "10683", "16149", "18476", "21381", "10819", "10875", "9881", "15533", "18056", "6053", "22669", "24213", "4772", "11444", "12460", "16539", "3214", "19560", "24408", "4525", "4083", "12360", "16828", "13807", "3209", "13432", "15881", "25463", "22899", "20990", "15073", "25251", "11670", "10147", "24913", "9355", "10674", "20458", "2253", "13487", "22583", "16851", "9340", "23142", "2364", "10868", "17256", "10719", "22538", "9655", "22711", "3974", "15495", "6675", "14386", "2920", "10136", "12346", "7579", "6437", "8981", "4110", "24476", "6660", "18719", "867", "10279", "10116", "388", "9413", "3178", "1018", "2894", "371", "18586", "12113", "19804", "12372", "11443", "3930", "17796", "22193", "10643", "20088", "5859", "24761", "22839", "19364", "2369", "16585", "6138", "8711", "17585", "11199", "7377", "22077", "25774", "19626", "15550", "12245", "22102", "21968", "22631", "19834", "9683", "16051", "11608", "4962", "9169", "17058", "19588", "3678", "9785", "12747", "10705", "23294", "15995", "16903", "16294", "16995", "6923", "14464", "1572", "25277", "12418", "270", "3743", "21732", "18080", "16690", "4459", "8226", "1436", "15357", "19358", "11188", "25394", "9095", "3487", "20625", "10733", "2146", "3901", "7450", "25593", "3000", "12390", "6481", "10993", "16291", "25698", "469", "21092", "10999", "3872", "19736", "18118", "11447", "20597", "9371", "15351", "8934", "14595", "19297", "4475", "3533", "6280", "22404", "12934", "1224", "20361", "14665", "11033", "10246", "9866", "23631", "8337", "22577", "24147", "19009", "11642", "11419", "16350", "14886", "21075", "6643", "3980", "18364", "8537", "12142", "7947", "25587", "14600", "19488", "4208", "19192", "23792", "4122", "4595", "63", "3577", "19599", "10782", "7103", "25166", "21686", "615", "16780", "10767", "1849", "19786", "16321", "6594", "13092", "209", "16676", "2006", "22030", "23673", "1327", "14265", "3593", "1151", "5694", "14876", "1480", "14645", "937", "9196", "25356", "23467", "23815", "19183", "14627", "4432", "15081", "14418", "20572", "15796", "16657", "12251", "4243", "1578", "6041", "16597", "5725", "3140", "13017", "149", "12336", "9044", "1526", "1338", "22302", "10998", "5810", "21958", "5921", "12645", "7371", "13877", "9249", "4697", "10758", "11496", "14991", "24429", "12651", "25473", "12491", "4339", "9158", "6728", "23746", "18530", "22857", "6009", "21737", "13712", "18019", "25132", "19054", "1020", "12784", "2465", "7485", "9709", "21955", "15848", "1162", "2094", "6551", "3929", "23484", "3110", "8058", "3331", "18076", "7406", "23583", "191", "3800", "15143", "6503", "1033", "12227", "6427", "21806", "7644", "12422", "25384", "5201", "25503", "14628", "6237", "24052", "9721", "18341", "7591", "5942", "9315", "110", "6909", "6872", "5601", "16366", "13835", "15577", "13929", "15746", "17330", "24214", "10940", "9324", "23130", "6821", "9524", "24473", "17815", "6056", "9885", "7764", "19444", "7635", "7240", "1596", "531", "23797", "7128", "14904", "21117", "21802", "19517", "9614", "417", "16768", "20982", "18802", "22584", "23117", "13726", "9483", "6524", "19107", "13927", "20948", "1271", "16181", "13228", "24987", "4792", "5394", "25786", "695", "9100", "10230", "15980", "4090", "15228", "7763", "23066", "5971", "3258", "1304", "9431", "18321", "24678", "18534", "2393", "6342", "15227", "26029", "18953", "16610", "2673", "12585", "2122", "6878", "11499", "12247", "12992", "12174", "23842", "15378", "17113", "9649", "5143", "21303", "12103", "657", "13004", "3679", "17430", "16284", "10187", "17281", "3812", "19482", "13699", "13528", "3403", "14351", "17898", "3278", "22168", "11940", "17491", "14889", "2594", "330", "14534", "272", "3857", "17306", "7302", "19340", "12991", "1114", "9908", "5507", "17703", "10948", "13617", "242", "11527", "22288", "10166", "394", "20212"]], "xtest": ["int", ["14770", "6565", "15089", "2846", "5662", "22261", "24405", "23543", "24322", "4535", "11276", "15204", "1228", "20941", "4029", "3170", "12941", "5262", "14816", "16041", "2165", "9382", "25884", "17540", "13309", "5760", "16281", "11539", "21338", "16772", "1647", "17620", "1812", "18654", "6365", "22799", "24468", "14973", "4442", "5528", "16693", "24924", "22416", "25630", "25350", "9422", "20413", "2467", "20402", "5241", "19453", "2128", "16332", "12556", "18651", "4930", "10721", "15315", "9768", "19829", "22402", "20582", "4120", "8028", "7582", "5070", "8442", "15350", "22708", "14859", "20619", "21617", "24014", "22025", "9525", "720", "8335", "4907", "13952", "9149", "2941", "9482", "15045", "8552", "10142", "14821", "2167", "9091", "18763", "8065", "13966", "3786", "1953", "8899", "8435", "18413", "20304", "5060", "1660", "22441", "22183", "19806", "21890", "25927", "13793", "13244", "25682", "10464", "659", "20967", "6845", "16766", "19962", "8870", "21264", "598", "11177", "5717", "17103", "3342", "4104", "3610", "21447", "21049", "6392", "17682", "12909", "14999", "14931", "16534", "8128", "7570", "16705", "25040", "25498", "6582", "25381", "1512", "15663", "8807", "5739", "16727", "15137", "8826", "11369", "13512", "22235", "22694", "1270", "19734", "21517", "15199", "4002", "14153", "22801", "16924", "6486", "16758", "317", "19802", "2040", "17680", "7716", "3203", "17740", "16790", "18410", "689", "13778", "10791", "14348", "7053", "1868", "14012", "18808", "8759", "17151", "24031", "5590", "8419", "2612", "17472", "10124", "1001", "1558", "13815", "20266", "16550", "1226", "22881", "19341", "19477", "9827", "2618", "11862", "23284", "15942", "16251", "6202", "8907", "9301", "10632", "25575", "15797", "14133", "18067", "9517", "7831", "19882", "19387", "14745", "4691", "8670", "9839", "24181", "11645", "9172", "9494", "9339", "1880", "2999", "2151", "20706", "7765", "22494", "6087", "16167", "15136", "18691", "2390", "9143", "98", "20282", "7522", "4315", "12897", "18749", "13853", "14632", "13866", "13462", "19018", "9635", "8849", "3886", "19338", "15097", "14490", "5823", "1328", "23279", "1575", "3059", "1232", "22761", "244", "1230", "2090", "17009", "9056", "24767", "12063", "10133", "3279", "10301", "12262", "12407", "15249", "20624", "15808", "15520", "19379", "11755", "20136", "16097", "10687", "17658", "12099", "3953", "21543", "3713", "25555", "12988", "21979", "21529", "18955", "6504", "12868", "9400", "10630", "6416", "18073", "24575", "7512", "61", "15051", "25407", "5959", "22956", "8460", "1876", "15686", "3201", "8202", "9350", "5277", "15752", "8678", "6765", "2053", "21848", "5559", "20039", "23311", "11998", "4359", "8708", "6949", "15152", "17445", "22316", "372", "23632", "13150", "11601", "6951", "22948", "17533", "22910", "25935", "15411", "20536", "18344", "23799", "25471", "16794", "20885", "16620", "3651", "5491", "2996", "8029", "7674", "23043", "18768", "20937", "11738", "15145", "14857", "156", "24539", "1913", "19762", "19177", "12836", "20853", "20998", "19008", "17365", "14670", "2730", "23837", "6817", "21325", "960", "12192", "14860", "1255", "2466", "18438", "1920", "7558", "24811", "24624", "18483", "19692", "24865", "4639", "23327", "17975", "2858", "13532", "5626", "11924", "2342", "9330", "6218", "14372", "20242", "18078", "22491", "10224", "4006", "16033", "4989", "10673", "8043", "21451", "22645", "1058", "5596", "25744", "4300", "18836", "7203", "7343", "5675", "15129", "6838", "22849", "9630", "6098", "674", "19354", "18039", "11851", "9979", "8942", "17165", "2421", "13147", "3183", "19680", "10256", "23199", "8954", "25075", "24995", "12981", "17639", "7254", "12976", "22105", "12968", "16774", "12728", "15020", "13827", "259", "11175", "8603", "185", "22532", "340", "7084", "11284", "3540", "21701", "17406", "9023", "20535", "19316", "9681", "21780", "14841", "13016", "1012", "21876", "16211", "3053", "12947", "17074", "19863", "25961", "21329", "11301", "1437", "18103", "20555", "22472", "2946", "7424", "11054", "2651", "11008", "4021", "23115", "22846", "6370", "21646", "23771", "16277", "6111", "14432", "22307", "23558", "8464", "11739", "19664", "13632", "25501", "8081", "392", "25437", "18812", "2286", "24511", "7659", "4250", "23395", "6283", "2821", "13052", "7542", "3061", "20199", "23499", "14284", "1914", "6677", "23275", "1143", "23136", "16984", "2010", "21712", "18457", "23383", "1348", "21913", "589", "14067", "12830", "22306", "20656", "16832", "18055", "12520", "25805", "14459", "18101", "16852", "13146", "11139", "3135", "7917", "18568", "22733", "18313", "18045", "1315", "17664", "25003", "20929", "11074", "5052", "7969", "9907", "15681", "13797", "22221", "13888", "11154", "5203", "13747", "6118", "18415", "3701", "22833", "23772", "20690", "25715", "1879", "8408", "13405", "11090", "3515", "21036", "18883", "17927", "21123", "12264", "20237", "5210", "7176", "1285", "25648", "4082", "25730", "1826", "12538", "10070", "12555", "15257", "2046", "23938", "7411", "8812", "19329", "8557", "19281", "19529", "2915", "5891", "11119", "10477", "15000", "17208", "154", "16103", "17804", "12310", "16122", "8781", "6560", "19457", "7398", "10039", "19301", "5795", "3141", "10044", "4460", "12916", "24048", "3732", "4251", "14676", "16517", "18198", "18539", "5764", "14088", "21785", "19287", "14120", "5873", "25540", "8087", "6666", "14394", "1981", "126", "14882", "8358", "19887", "81", "14762", "16369", "29", "13538", "9234", "20337", "25177", "23969", "21532", "16504", "5240", "8197", "7252", "13443", "7412", "14434", "7531", "17980", "16004", "925", "5103", "9667", "4386", "8340", "15916", "21487", "18915", "1894", "10199", "15921", "6008", "1911", "5700", "23946", "16286", "8213", "4652", "77", "19312", "1362", "14470", "17398", "13629", "871", "1831", "4238", "25181", "5244", "1873", "18695", "22890", "3703", "25171", "2934", "11308", "18348", "22365", "11956", "10713", "630", "11394", "21127", "6211", "14274", "928", "1440", "17413", "18443", "14612", "13061", "12", "18952", "18615", "23541", "15008", "5367", "11204", "15662", "13520", "4650", "18510", "22095", "13732", "6553", "17091", "18767", "21078", "20741", "4409", "11671", "19073", "5728", "6662", "3410", "3452", "55", "6859", "10774", "2608", "23872", "21904", "685", "5458", "11015", "5194", "4957", "9458", "23274", "5973", "769", "16400", "13856", "21949", "20224", "3682", "826", "7767", "12949", "1918", "11842", "23288", "9856", "20530", "4954", "8809", "194", "18892", "5752", "13039", "20578", "1306", "19077", "6187", "8517", "23754", "13408", "2249", "25420", "3476", "9828", "10205", "1263", "18913", "6223", "19086", "21524", "8224", "4271", "2621", "3911", "808", "1699", "22339", "25197", "25209", "22992", "25291", "5043", "21951", "3759", "17211", "13886", "3159", "11843", "10448", "20803", "13438", "9512", "7378", "6526", "12727", "8822", "9092", "7739", "21754", "14929", "18706", "25521", "2792", "13949", "7017", "24941", "10416", "23564", "9492", "23391", "5088", "12607", "18128", "20006", "4165", "85", "19728", "25402", "19766", "20198", "22914", "24407", "9479", "20732", "6652", "18502", "535", "5488", "4117", "20243", "6414", "6769", "21610", "23352", "25427", "3011", "23919", "9820", "13780", "24717", "5663", "12514", "10107", "3591", "13066", "20241", "12674", "7136", "10788", "18703", "5783", "6396", "8587", "13193", "10217", "17941", "4895", "14420", "21661", "3625", "20991", "12204", "12791", "4187", "2900", "3101", "7090", "17308", "11291", "22560", "19099", "6404", "20589", "23239", "5964", "14312", "8682", "19486", "23624", "25727", "17991", "24636", "7747", "6611", "8736", "10063", "6086", "16215", "24041", "3356", "522", "23033", "17368", "6042", "17510", "15468", "14276", "2726", "11036", "6869", "4058", "5273", "159", "13850", "14616", "25056", "23766", "18610", "133", "15537", "2039", "14873", "8093", "16673", "23909", "9648", "21137", "1458", "319", "24113", "20751", "14310", "23206", "755", "12562", "17679", "19346", "19711", "15582", "21555", "13080", "13094", "1793", "13207", "24049", "17175", "21032", "7452", "17514", "14752", "9872", "4556", "18051", "10715", "15175", "2585", "8695", "11923", "4394", "852", "11272", "6999", "10703", "3979", "15972", "17996", "13275", "3891", "1756", "12763", "3116", "25763", "14134", "25321", "21594", "23489", "3565", "23836", "18368", "2244", "7986", "12844", "8098", "1724", "16501", "1075", "24827", "4493", "3695", "17607", "17021", "6911", "8182", "24593", "14684", "18193", "4464", "12163", "3865", "25254", "23186", "3675", "7691", "23261", "18346", "15061", "22097", "2797", "25190", "8727", "14619", "3081", "19858", "901", "13491", "21511", "1276", "15969", "7707", "10247", "11618", "6109", "1784", "947", "24793", "24096", "12504", "13045", "15862", "8437", "14980", "8451", "1927", "2450", "22811", "25949", "16809", "2095", "16656", "22868", "4077", "25438", "4047", "25638", "13141", "3470", "1903", "18404", "23737", "10572", "1322", "15151", "13342", "22469", "13195", "5306", "16587", "22602", "60", "9748", "24203", "8261", "22373", "24568", "7957", "11694", "24002", "15306", "21852", "4437", "23903", "13899", "2565", "14293", "11463", "22879", "17252", "13267", "18571", "12743", "21626", "20748", "19764", "14477", "5644", "14794", "25369", "10032", "14572", "24415", "703", "20461", "8550", "24273", "18636", "20871", "13390", "3478", "3505", "18366", "19016", "23022", "14815", "9505", "20316", "12095", "15838", "1211", "22721", "18974", "8501", "11627", "20521", "6475", "9148", "11018", "20880", "14282", "15588", "8640", "18887", "89", "16737", "12170", "21547", "14520", "24847", "983", "9890", "15115", "4744", "23828", "15086", "20456", "4779", "5917", "17115", "13164", "3067", "8852", "23011", "14598", "7693", "16844", "13110", "20178", "13755", "2717", "21566", "4402", "19752", "6948", "10364", "18917", "6678", "24747", "11999", "17918", "16160", "16194", "6362", "5853", "22510", "21895", "25842", "688", "18789", "2353", "4661", "11135", "13959", "3725", "17999", "11023", "2747", "19064", "9752", "24245", "17614", "8980", "16911", "5749", "17787", "13297", "165", "14328", "20284", "11665", "19440", "7523", "111", "16952", "13838", "1704", "21651", "25914", "5164", "20334", "19355", "24807", "21516", "956", "11335", "8703", "8730", "23626", "11662", "18688", "5021", "438", "17334", "14713", "25692", "24238", "4088", "22835", "8278", "1481", "14834", "14589", "25298", "4428", "2478", "2060", "23715", "24137", "3739", "15440", "815", "17693", "713", "23942", "506", "13976", "22579", "21500", "10601", "25646", "11669", "24834", "9162", "14093", "3198", "6229", "21270", "21441", "14438", "12323", "1439", "23974", "7434", "9466", "13727", "2826", "25709", "3994", "16722", "12892", "12935", "24892", "18049", "13305", "6834", "4135", "8080", "992", "1814", "23211", "2116", "9711", "23610", "14209", "2751", "7610", "2387", "12882", "22398", "188", "21861", "11", "22847", "1976", "1095", "5379", "11679", "13822", "15971", "8851", "18333", "8492", "21309", "25580", "13821", "21540", "3078", "5032", "7014", "10732", "8134", "7465", "2179", "23495", "19885", "9114", "23565", "19406", "14395", "23232", "15512", "7346", "436", "12603", "1520", "4926", "10773", "16078", "3243", "7864", "4771", "16328", "20675", "22356", "18061", "16374", "9460", "15658", "9521", "14211", "18645", "719", "21994", "23635", "16481", "3951", "5775", "6194", "11860", "21108", "10870", "13618", "12945", "22864", "15997", "22604", "18242", "3493", "19540", "5678", "5998", "5165", "1265", "25628", "7968", "22292", "16520", "19049", "6688", "16514", "7501", "21317", "5328", "16816", "6407", "17160", "19191", "20155", "25933", "4674", "24159", "25793", "15481", "17643", "13882", "24996", "15342", "25627", "11282", "20987", "19848", "23602", "22999", "19730", "22896", "4791", "25818", "19658", "15553", "25131", "23856", "4507", "11807", "13088", "11180", "12295", "21838", "23184", "1021", "14072", "9132", "16074", "1405", "17583", "16336", "12279", "1045", "16729", "13848", "13356", "22566", "17631", "9242", "5261", "24246", "4257", "268", "16935", "17041", "3075", "6126", "12476", "24467", "19815", "25690", "9969", "15718", "19823", "23554", "11508", "19720", "11490", "20878", "8070", "16087", "16761", "20965", "7547", "4806", "6295", "9074", "15462", "9294", "25721", "11049", "1298", "12670", "21675", "7825", "12944", "19859", "1318", "20058", "15782", "8988", "18627", "15901", "5966", "3707", "5617", "21727", "20189", "5471", "9931", "15734", "10495", "1674", "4918", "8995", "25005", "8679", "20779", "11413", "11327", "1093", "556", "14875", "8606", "20452", "752", "4476", "25203", "20234", "11349", "24496", "17326", "8436", "17147", "15361", "4421", "11150", "25701", "6634", "19921", "10370", "6491", "18997", "7580", "3079", "21907", "16306", "24279", "23738", "18712", "9971", "11082", "23514", "16908", "21130", "19754", "5188", "8239", "12940", "24419", "21494", "2899", "2931", "11648", "14702", "11095", "1465", "12410", "11163", "21589", "13064", "6556", "4158", "19994", "17459", "13596", "14104", "17440", "13189", "25617", "11538", "15124", "12633", "23420", "6510", "11446", "10627", "18495", "22256", "14268", "18202", "24275", "1192", "21244", "4768", "6055", "11554", "17401", "18727", "12672", "21124", "19548", "11615", "8632", "12548", "14946", "8089", "23557", "14731", "10932", "10892", "7788", "5318", "10918", "16317", "16493", "18924", "679", "20935", "15355", "10930", "19377", "7349", "9892", "21363", "25510", "3236", "13316", "4741", "24997", "11402", "7011", "7019", "8846", "11240", "11861", "16119", "265", "14515", "14496", "19019", "19771", "12359", "15770", "13087", "7615", "4583", "14040", "11146", "18326", "4706", "4976", "4943", "17671", "452", "11120", "350", "2321", "15816", "14978", "200", "3551", "4917", "23625", "897", "8621", "22963", "12913", "16829", "16044", "7651", "17207", "13412", "342", "352", "2171", "3793", "1153", "25023", "10631", "19941", "13410", "21483", "6199", "13621", "3767", "1324", "1428", "25296", "25215", "20953", "12679", "19314", "14082", "8568", "21349", "14915", "6346", "10989", "18040", "18820", "16670", "5583", "24531", "21055", "10537", "6134", "21956", "3491", "5489", "6277", "20469", "25582", "14387", "15158", "2987", "20100", "141", "11666", "90", "2123", "17307", "20764", "4210", "23335", "10531", "16890", "7342", "19069", "15867", "14587", "7318", "14279", "22917", "13564", "10117", "6495", "44", "11759", "22279", "1297", "24840", "24350", "17779", "1802", "16075", "1048", "6064", "20950", "16808", "2635", "14129", "12656", "15542", "23868", "23571", "25528", "16745", "11307", "5057", "15541", "10876", "13237", "10738", "12546", "2830", "13772", "22217", "6525", "6433", "17028", "12059", "9988", "8432", "25259", "19533", "8237", "5597", "21503", "17851", "4862", "5903", "13057", "2575", "13120", "17937", "1119", "8765", "14380", "13540", "19678", "3604", "24508", "8067", "13301", "10991", "12321", "13962", "17377", "10159", "17425", "20652", "7483", "1559", "23562", "23781", "20331", "19763", "88", "5224", "9783", "14163", "5857", "20216", "3027", "9584", "6066", "17993", "12984", "310", "6436", "18360", "24501", "16744", "10349", "4308", "13865", "20132", "9022", "5886", "10407", "1210", "3417", "24470", "24541", "25448", "18544", "21719", "21029", "15242", "9498", "24174", "10148", "18928", "6291", "25097", "15940", "24842", "9779", "8411", "2209", "24642", "5702", "6538", "17880", "4033", "17958", "17819", "4766", "20415", "2059", "7559", "5522", "6388", "1087", "11200", "10616", "7988", "9338", "7753", "9048", "25889", "19403", "11340", "7930", "13250", "840", "10048", "17729", "25366", "14482", "22238", "21246", "19795", "11383", "8739", "19319", "10873", "22775", "13252", "2048", "24637", "8105", "18902", "14362", "3559", "25069", "17888", "16027", "7118", "24818", "24324", "7382", "3383", "13550", "6798", "2645", "7620", "20914", "6331", "9308", "8227", "14397", "4072", "21705", "11678", "22452", "1939", "7031", "13314", "22945", "19846", "3985", "20515", "7372", "21565", "25146", "2928", "13607", "22769", "20963", "23359", "24427", "25621", "12990", "14502", "20339", "5558", "11042", "19448", "18561", "4591", "10784", "24073", "23328", "1022", "6131", "14439", "18908", "23984", "49", "25894", "9706", "25071", "20997", "11709", "2939", "10454", "23019", "15947", "5872", "23237", "14621", "13203", "23591", "11732", "22345", "11574", "1376", "24207", "1629", "8068", "1409", "1735", "18093", "25761", "5730", "17288", "8613", "11684", "9714", "14810", "7325", "3028", "8296", "15144", "5940", "19874", "20312", "17477", "12441", "21709", "15272", "14419", "14911", "11329", "16986", "4496", "19244", "16625", "9915", "6816", "16547", "16337", "9819", "20254", "24378", "22266", "18713", "20827", "2869", "19022", "16257", "6447", "14263", "10698", "24594", "3617", "17730", "2396", "13139", "21798", "8283", "2189", "23248", "24336", "3408", "5553", "12097", "16456", "20263", "14363", "24998", "1293", "11203", "17184", "25283", "24808", "7518", "25960", "23991", "14874", "17509", "20047", "14949", "6124", "5121", "22031", "16644", "6313", "4023", "2078", "25629", "17112", "14571", "862", "12157", "5420", "23985", "10483", "19491", "7935", "9341", "18217", "16038", "17822", "2170", "15305", "13754", "3885", "1335", "16341", "14466", "15443", "6225", "15996", "2270", "16247", "7964", "25520", "15631", "13347", "12404", "9017", "6369", "25159", "9312", "6982", "19550", "22338", "7407", "4898", "18378", "8902", "1830", "7082", "17650", "15077", "13418", "5417", "13930", "14325", "10259", "24389", "1805", "24488", "10165", "15437", "6035", "5581", "19914", "18111", "14838", "25568", "1611", "15643", "25266", "22255", "5111", "11126", "21553", "23526", "17487", "5929", "17364", "5223", "16674", "10249", "3572", "19739", "8845", "18905", "24664", "11097", "9859", "10532", "10456", "15599", "11589", "14553", "19855", "21206", "16428", "5874", "10994", "368", "21248", "4939", "944", "18545", "393", "13693", "2008", "1994", "22512", "2867", "21944", "17825", "9113", "18841", "23967", "3211", "1714", "16553", "22430", "20876", "10660", "14280", "18072", "22715", "13757", "12332", "12632", "10641", "16588", "22052", "15846", "14856", "2574", "3637", "23180", "2481", "6522", "5129", "7274", "2833", "16797", "21908", "5755", "9533", "22768", "24266", "3287", "10135", "24764", "19759", "24060", "7560", "24373", "4213", "19910", "19397", "20789", "7358", "19381", "14942", "12861", "17342", "16202", "12210", "21148", "22033", "6192", "25839", "564", "22246", "18875", "3599", "24299", "19317", "14979", "14603", "22932", "5658", "16714", "7614", "660", "19943", "9229", "11300", "8168", "14111", "7996", "17379", "8426", "21076", "14740", "2784", "8342", "6095", "7013", "24753", "2098", "8386", "14385", "23890", "22431", "16680", "13766", "16252", "6393", "2969", "20459", "1865", "25976", "25925", "9817", "24821", "8297", "13498", "11781", "8138", "21733", "18446", "24695", "21126", "19646", "18710", "839", "5771", "4456", "15633", "4454", "20468", "14699", "11852", "22436", "25194", "19408", "8382", "12953", "11295", "12946", "23627", "13684", "23050", "7265", "25754", "12148", "6341", "1500", "7475", "1739", "21422", "16581", "2893", "1176", "20449", "23763", "475", "14176", "8184", "3892", "2632", "9816", "17843", "5503", "2921", "9834", "11870", "3507", "6590", "14071", "20930", "16552", "12282", "5342", "5235", "4136", "7496", "23480", "6383", "12327", "682", "569", "631", "25311", "22531", "19105", "24653", "13355", "5051", "11932", "1076", "750", "4584", "2631", "13505", "19419", "16486", "22907", "777", "8722", "22458", "9575", "11727", "4571", "15501", "20697", "22091", "8422", "18461", "18123", "18225", "15451", "11710", "14288", "20117", "8438", "3961", "13454", "10383", "4026", "5648", "398", "1269", "482", "789", "6305", "1564", "23312", "4582", "4936", "9408", "16318", "1258", "22291", "8565", "24561", "8329", "19121", "17395", "1610", "5545", "10923", "12706", "9718", "17637", "22399", "19269", "2505", "15413", "8789", "19216", "24609", "6217", "2285", "25714", "3252", "17608", "16098", "7734", "12982", "20926", "3330", "24959", "5933", "6897", "21330", "1117", "787", "11810", "11207", "25049", "2472", "25504", "14460", "8835", "12494", "19497", "9137", "25280", "11544", "3373", "15668", "4059", "14451", "24748", "18435", "969", "19743", "9096", "24319", "20838", "14956", "21163", "18860", "24494", "9966", "18752", "19871", "14492", "16764", "10933", "4310", "8719", "18020", "17793", "3466", "7683", "4063", "24960", "18291", "14038", "3345", "25204", "2904", "12904", "13798", "42", "13102", "96", "6069", "8871", "23084", "8031", "12261", "20896", "1047", "13378", "23384", "10143", "1289", "122", "13676", "20153", "18132", "18981", "2141", "12034", "5961", "14318", "10813", "9412", "22891", "19598", "21976", "25622", "1326", "14830", "5549", "25523", "19693", "13282", "18700", "236", "21111", "23343", "23944", "25091", "10231", "16513", "2901", "13022", "25908", "9235", "20609", "3262", "2257", "17140", "21369", "18380", "9601", "7384", "24416", "1589", "24208", "8698", "21749", "19977", "16231", "4014", "8030", "21859", "21165", "17616", "5735", "14406", "4668", "12516", "11748", "12016", "15935", "23187", "15639", "17241", "20230", "12839", "18142", "16399", "5989", "11616", "9536", "9410", "1216", "1136", "19647", "9344", "8218", "11977", "15422", "21300", "5668", "16372", "21190", "14047", "17057", "2728", "12057", "9586", "9429", "13086", "12636", "11328", "25392", "2204", "9849", "6761", "12854", "15886", "8817", "7634", "15721", "2804", "19821", "6572", "9034", "14373", "8247", "6417", "5200", "7449", "9940", "10983", "12189", "16358", "11867", "25752", "16741", "25419", "13278", "7826", "5881", "2671", "11723", "3526", "10743", "21652", "13738", "22056", "13842", "2399", "8830", "1476", "17892", "504", "17977", "16209", "16810", "11633", "20026", "3627", "13895", "17276", "14697", "4609", "19600", "4479", "14885", "375", "3920", "20384", "20351", "16273", "5676", "1568", "24303", "5227", "8241", "25080", "15641", "4229", "11368", "21281", "23976", "11360", "17756", "2012", "8659", "2634", "18616", "9102", "13789", "16519", "14182", "23532", "2263", "5502", "21858", "6555", "12363", "23749", "13214", "700", "12703", "10146", "20068", "1801", "12807", "860", "23743", "24929", "11061", "5967", "13569", "24951", "5913", "9055", "12307", "22524", "23093", "5228", "9086", "7507", "3359", "15654", "11220", "2034", "6070", "16297", "276", "5552", "173", "896", "23345", "8847", "8362", "12296", "6335", "10183", "8674", "17380", "19909", "20475", "4872", "17579", "12482", "13407", "21265", "4439", "8127", "15975", "25631", "10518", "22526", "8937", "9736", "9850", "14493", "19258", "5950", "18813", "4448", "25904", "18865", "7025", "9572", "14478", "14813", "8324", "10269", "25152", "24858", "10004", "5855", "12685", "22101", "559", "6657", "12299", "20613", "2137", "16965", "22836", "6320", "24981", "14326", "1921", "15936", "5734", "12979", "8406", "7731", "16082", "7555", "282", "6273", "8186", "6411", "16934", "21520", "14757", "8114", "15907", "7311", "22970", "7223", "6864", "9145", "9006", "4330", "10410", "18932", "22885", "9902", "1217", "18150", "20647", "12776", "10728", "16282", "23092", "9016", "23257", "23085", "8447", "19116", "18492", "15205", "11512", "20715", "9682", "3340", "731", "8381", "10213", "4262", "18192", "7164", "18985", "9396", "3841", "20011", "23170", "21142", "11742", "20393", "11757", "16288", "1427", "6593", "18119", "14302", "17572", "18212", "13317", "24355", "19204", "11320", "18398", "3114", "1828", "18639", "24831", "25539", "2534", "7003", "13003", "3143", "15569", "18114", "18031", "22361", "9807", "4542", "1829", "18784", "19524", "19089", "15884", "13380", "8804", "13728", "22366", "5353", "25458", "11832", "19401", "22502", "8047", "24243", "12211", "863", "19606", "11238", "5807", "14431", "20785", "25058", "23636", "1019", "254", "13135", "23901", "2746", "3471", "19931", "20805", "12339", "6332", "17783", "24933", "16798", "15933", "20668", "1386", "21367", "23428", "11858", "22191", "20887", "23408", "1603", "20424", "8521", "25487", "1448", "3860", "8916", "11906", "4665", "18747", "4144", "10515", "3034", "12655", "24224", "3494", "7429", "19210", "18992", "164", "9695", "3091", "12240", "3854", "14210", "20182", "7802", "17087", "20617", "8664", "17600", "17251", "11337", "13960", "9094", "2045", "11038", "6849", "16424", "10710", "3289", "4081", "6827", "1577", "512", "729", "18416", "9662", "21522", "23980", "1719", "18977", "11951", "18991", "19212", "17623", "20054", "5649", "4801", "23717", "12382", "12961", "12021", "15649", "19385", "20640", "23848", "3272", "20660", "18986", "18127", "22197", "13867", "10964", "9248", "4372", "1173", "10609", "12866", "21042", "1978", "4354", "56", "22949", "10356", "15855", "12426", "575", "13649", "10835", "1848", "4959", "15066", "16054", "18250", "3628", "14748", "10621", "16692", "4876", "18439", "3479", "9082", "2360", "8859", "15777", "6185", "19159", "20359", "2489", "2507", "13546", "4537", "6024", "5986", "3910", "4364", "18817", "6573", "454", "20484", "12225", "23190", "14903", "1187", "21801", "16026", "18116", "20693", "14259", "23973", "2384", "9291", "23491", "19353", "17003", "13225", "8236", "4868", "5434", "22116", "4046", "12354", "12049", "21894", "8669", "10427", "6696", "20143", "16457", "4245", "16169", "21161", "16431", "25855", "4631", "8528", "13879", "732", "23497", "19333", "7628", "14202", "22277", "12713", "13229", "22925", "23694", "2233", "9206", "5310", "2414", "2926", "20121", "9142", "15496", "2001", "157", "6956", "20439", "9397", "16841", "590", "1717", "4723", "19352", "13112", "22887", "14725", "23372", "8223", "158", "2087", "18525", "1223", "4671", "24186", "9221", "11166", "18179", "13219", "12316", "9279", "10395", "12559", "7100", "11955", "8628", "2829", "13435", "13677", "16468", "16020", "14131", "16596", "4523", "4754", "14059", "12203", "1426", "24092", "16415", "17856", "21728", "16157", "11431", "23745", "18432", "766", "23724", "25334", "19359", "7870", "21521", "17059", "11983", "20995", "9663", "9715", "13419", "4132", "20376", "19238", "15121", "21965", "13724", "5499", "16139", "16753", "25970", "13659", "23432", "9287", "20936", "25804", "13983", "11686", "13604", "6054", "6458", "19973", "24711", "17357", "24296", "772", "8313", "1778", "25287", "14150", "11051", "25386", "17940", "19872", "20545", "21181", "16287", "3426", "19042", "10740", "4142", "22564", "1312", "11231", "9547", "3742", "13413", "1705", "7890", "6251", "8172", "17665", "25874", "2120", "5893", "20856", "6327", "15982", "16868", "13277", "24095", "19306", "23850", "18013", "18518", "5629", "22039", "5529", "17119", "17417", "19906", "13944", "14068", "3083", "2973", "2411", "16213", "5365", "14753", "11228", "320", "17766", "12122", "15841", "5211", "9010", "14693", "11918", "19989", "8260", "11857", "2066", "3442", "2494", "23323", "25172", "7117", "21514", "12684", "23881", "18280", "21468", "8735", "15865", "19787", "10860", "9432", "5879", "20666", "15788", "18171", "13389", "8760", "12681", "11079", "5199", "9573", "13296", "12664", "7445", "5304", "540", "6654", "14145", "20332", "11635", "12929", "12731", "14242", "9822", "11351", "15017", "9738", "9441", "13307", "17296", "19305", "568", "21612", "23660", "6401", "23121", "2544", "22988", "22955", "2038", "5334", "7279", "9425", "16343", "5162", "20958", "17336", "3519", "3063", "15314", "21504", "13567", "7848", "14425", "21815", "19547", "7002", "13916", "4221", "19290", "12348", "15832", "8991", "20667", "13823", "6316", "2216", "4273", "308", "13481", "22222", "11390", "25878", "2968", "3008", "1677", "19852", "16201", "8472", "25963", "19503", "16354", "10808", "14157", "4725", "20033", "24237", "19466", "12107", "7216", "16710", "7839", "5206", "1399", "15215", "5451", "5319", "15390", "18011", "17952", "11219", "15531", "857", "18302", "920", "23103", "15687", "18637", "3777", "8338", "14179", "1247", "7152", "26005", "3971", "11702", "13765", "411", "21970", "1882", "13200", "5884", "4620", "4735", "24164", "23113", "23545", "25819", "20317", "12829", "7782", "12851", "5655", "20883", "23460", "3273", "10444", "15858", "17545", "25361", "2183", "3671", "25572", "17767", "228", "23241", "13083", "15575", "19601", "16079", "274", "5023", "24017", "406", "6153", "5205", "18663", "13698", "7421", "2629", "13909", "9005", "23349", "13518", "2849", "15691", "4001", "2471", "16804", "21287", "25795", "12907", "14593", "4925", "4408", "9454", "21764", "6169", "24557", "4964", "1245", "7515", "1148", "8762", "17383", "14730", "19920", "18425", "5395", "15429", "4091", "14994", "11474", "4690", "5450", "13901", "7264", "821", "5887", "19578", "9421", "18166", "21992", "25037", "25067", "4721", "2950", "13062", "15436", "25802", "7214", "1272", "9188", "21642", "19929", "7817", "5074", "20623", "3044", "13911", "13779", "1127", "9658", "19106", "18934", "2245", "11997", "25601", "20733", "8623", "11758", "2287", "23175", "12255", "11942", "18899", "25968", "24804", "18259", "11505", "24659", "16363", "12110", "10263", "1617", "25689", "23354", "5493", "23185", "3914", "14692", "22647", "10806", "1844", "6755", "1541", "22788", "25832", "1067", "13737", "4109", "24952", "13276", "1129", "23502", "11762", "15859", "6304", "8061", "10610", "21766", "12005", "9657", "13182", "17409", "19850", "25944", "10617", "20294", "3543", "18338", "5015", "2359", "18310", "21249", "17734", "24220", "1671", "7426", "21207", "2815", "25543", "18867", "2622", "9797", "10501", "7775", "18062", "6633", "19496", "12123", "22417", "2910", "403", "1712", "2347", "15165", "9899", "24506", "10379", "17416", "2683", "10494", "13234", "18160", "5969", "11487", "9574", "25559", "7899", "2991", "17777", "17521", "12821", "7192", "25389", "16380", "16981", "14275", "8273", "11603", "7462", "6710", "21947", "9661", "15544", "11418", "17515", "24649", "16158", "15527", "7443", "7511", "8354", "21889", "4248", "14687", "6428", "22061", "14484", "1935", "12837", "19878", "19179", "21763", "16930", "14299", "25335", "23411", "22231", "18960", "20719", "9501", "18851", "19137", "2562", "2026", "1818", "14034", "15730", "18479", "14200", "11165", "15114", "3180", "17194", "11478", "17012", "24001", "64", "24317", "299", "10910", "6861", "13542", "8503", "1908", "25645", "4327", "8950", "21662", "15924", "10423", "8535", "22646", "8144", "25108", "16099", "125", "12828", "16365", "10764", "13387", "24760", "2377", "8710", "23404", "20490", "2268", "17976", "25722", "21960", "5719", "22164", "26026", "6183", "19868", "4602", "24590", "9930", "12956", "5102", "18725", "7894", "8175", "7596", "19982", "8006", "16618", "7844", "8697", "24383", "23826", "9147", "10765", "10841", "16068", "16899", "22867", "4999", "4788", "19255", "18954", "2499", "16085", "20792", "23230", "8877", "18589", "3963", "22265", "3626", "6446", "8895", "17163", "25310", "18069", "8259", "2335", "9313", "19328", "16847", "19095", "13287", "1845", "24663", "3820", "8717", "23884", "1345", "8316", "18412", "284", "19645", "1633", "6997", "2541", "18598", "22592", "4529", "15748", "3023", "2193", "15046", "12525", "18973", "10099", "20978", "24809", "16256", "23283", "10950", "13440", "15915", "7248", "17053", "13730", "18165", "16863", "12601", "2014", "11066", "23008", "19513", "20610", "17799", "14661", "9627", "11334", "21794", "18233", "25032", "18488", "8734", "14570", "6399", "20816", "5453", "4996", "2536", "20369", "676", "14121", "11812", "14498", "10882", "11125", "9607", "24292", "21891", "12634", "23262", "13748", "11057", "15104", "17831", "13855", "2092", "14890", "18453", "1941", "17099", "23256", "4195", "11073", "8819", "13973", "6539", "4035", "23000", "25088", "23948", "25024", "20868", "14044", "25491", "21526", "3513", "13049", "1146", "20695", "12487", "5970", "21542", "19363", "21010", "25242", "17995", "4230", "16436", "3064", "856", "4087", "5439", "24801", "16754", "5025", "14336", "19088", "19460", "4614", "12043", "9030", "6631", "5179", "23634", "11689", "14798", "10779", "12762", "5763", "24964", "24720", "16782", "17300", "6123", "11549", "59", "13986", "25444", "23691", "11833", "18005", "20463", "18278", "5531", "4902", "20251", "1635", "19549", "14197", "14916", "1741", "3040", "13116", "7796", "10049", "21954", "1457", "4246", "909", "20158", "1815", "9548", "1514", "7656", "22159", "8225", "6907", "1378", "18134", "4856", "10095", "12062", "24348", "1188", "22385", "12775", "23471", "11350", "10508", "734", "21398", "6663", "5186", "25151", "3138", "11105", "2787", "5821", "10394", "24221", "14366", "10234", "17863", "5975", "17063", "3894", "23047", "20912", "16666", "430", "21971", "5856", "18806", "6300", "24363", "16704", "7865", "8252", "11321", "2041", "7556", "24885", "6505", "4726", "17143", "9743", "15834", "7618", "17979", "21531", "19236", "414", "4848", "3475", "6843", "11653", "25951", "7288", "19096", "2237", "22670", "8654", "16593", "24382", "2153", "15535", "21969", "4309", "8363", "10352", "3824", "1614", "24315", "15785", "21812", "23073", "7676", "21061", "21440", "12281", "2100", "9280", "11152", "10306", "6960", "16446", "24991", "15671", "12764", "17974", "20235", "18480", "7999", "5476", "3928", "3952", "12787", "10218", "23157", "1943", "24392", "876", "1590", "18452", "19621", "23965", "21033", "1016", "4737", "4618", "15181", "14525", "14334", "25758", "7331", "16630", "23210", "22367", "21319", "8803", "3529", "7687", "3836", "12825", "23953", "14191", "16274", "988", "14350", "4015", "9079", "3948", "25072", "437", "6549", "3105", "2413", "13920", "15101", "9436", "25345", "10659", "15704", "20492", "18559", "5825", "7020", "3773", "1975", "23580", "17879", "25612", "9407", "20800", "8154", "6179", "9624", "23053", "12647", "13513", "21103", "11019", "10350", "20790", "6708", "4231", "12144", "22134", "10098", "1094", "15724", "12139", "22474", "20159", "15209", "17985", "14374", "18739", "3324", "22161", "22497", "4997", "25228", "16591", "5377", "18159", "24928", "14300", "25537", "20377", "17318", "2952", "1700", "8572", "15391", "25063", "14347", "10034", "11988", "17774", "11604", "6690", "25066", "13982", "22960", "23476", "13620", "10669", "20598", "18788", "9587", "22993", "6398", "5708", "2487", "7885", "21196", "15380", "12856", "18682", "5519", "10925", "23007", "5806", "4528", "16003", "18224", "16509", "9088", "14648", "7333", "18445", "19134", "25703", "18245", "22092", "4242", "16577", "12741", "22127", "8312", "3232", "4503", "6873", "12145", "18221", "2998", "19957", "10842", "17039", "17738", "21690", "22176", "19757", "22136", "19672", "19494", "10734", "2601", "24013", "10163", "634", "1733", "6502", "5870", "4824", "19000", "7686", "6513", "22162", "16012", "10442", "10921", "410", "20727", "10062", "15007", "9686", "11776", "8749", "18272", "19109", "16884", "25739", "13631", "22369", "11570", "11904", "17948", "509", "8176", "8397", "17613", "16912", "7071", "11113", "25830", "10761", "22470", "14869", "23767", "25479", "14754", "20275", "25398", "12154", "24833", "2175", "2463", "1144", "21808", "4973", "4659", "14732", "21388", "10403", "4775", "17443", "3176", "25207", "11933", "25817", "4994", "22752", "1900", "13044", "2264", "23996", "8609", "24824", "10904", "21267", "22203", "6988", "18375", "2297", "12288", "25033", "5397", "13626", "15739", "20905", "25915", "24943", "5093", "3964", "11692", "4305", "356", "6939", "4383", "21963", "6368", "23195", "10563", "8387", "20921", "7736", "23386", "7984", "20940", "5898", "20512", "7800", "13474", "2142", "24651", "14647", "15768", "17748", "7438", "4545", "21152", "23191", "8836", "17142", "16083", "767", "16006", "11543", "15951", "15883", "23858", "13158", "23414", "507", "7024", "8965", "4663", "11198", "16862", "23267", "20501", "19818", "3294", "7271", "12939", "15206", "6083", "3435", "23405", "10363", "18033", "14806", "3358", "12401", "14795", "20954", "14742", "4526", "17187", "11289", "23236", "11354", "19339", "13095", "15074", "17899", "20662", "10986", "494", "14850", "4466", "6749", "11628", "24525", "20067", "23388", "19690", "14996", "15252", "7562", "17423", "10421", "8139", "12054", "22131", "9414", "9027", "18251", "24197", "24516", "25785", "20511", "7454", "4284", "16232", "22394", "16939", "5642", "15385", "13814", "451", "17505", "22335", "5055", "9008", "472", "708", "15952", "11541", "16733", "4417", "8645", "17481", "8092", "21462", "19649", "22582", "7169", "3686", "1807", "24903", "12031", "23770", "7035", "7028", "13506", "1049", "16007", "21767", "20513", "8627", "19835", "2628", "23970", "1638", "7554", "4253", "22800", "23259", "22439", "17013", "9554", "19514", "2079", "19032", "4992", "5371", "25861", "13293", "9164", "3992", "21873", "3723", "23597", "13719", "8280", "13599", "6157", "18795", "8726", "22990", "24449", "21606", "15", "21835", "18956", "24051", "4215", "18799", "14430", "19233", "12871", "17743", "4613", "14551", "10929", "6544", "19778", "5718", "13889", "20038", "25787", "11705", "19145", "23649", "7871", "12857", "3407", "11257", "6209", "20650", "3127", "16134", "23339", "10789", "3782", "964", "20996", "17008", "2942", "1820", "19468", "22175", "14726", "24687", "15460", "10614", "20240", "5269", "11226", "8658", "20440", "18028", "1866", "22209", "21824", "16126", "21731", "18431", "4740", "341", "11704", "25279", "12180", "3648", "10753", "20022", "5498", "25570", "4893", "14319", "15362", "12046", "12878", "25480", "10620", "18191", "23651", "3700", "2354", "12151", "12219", "894", "5808", "14625", "3804", "18529", "6679", "20603", "22828", "17222", "5798", "2897", "22020", "13144", "7040", "25920", "19753", "19205", "10969", "8095", "17674", "422", "8511", "22218", "20986", "15021", "3404", "12086", "14852", "14990", "3293", "22461", "4317", "2591", "21260", "21068", "23950", "1411", "15990", "22905", "14712", "21235", "9538", "13706", "10589", "3321", "6214", "12875", "15517", "10720", "23016", "4443", "24839", "24354", "14579", "21040", "15922", "8681", "7923", "17314", "24016", "5513", "4718", "2650", "2914", "5155", "3986", "6030", "23596", "18949", "6720", "20367", "9083", "10544", "20319", "12079", "14256", "21717", "16541", "17709", "16945", "9015", "4434", "10771", "9590", "9450", "17293", "23260", "18570", "19433", "18948", "12746", "3093", "22082", "4739", "15364", "24323", "8597", "5745", "17896", "24722", "12734", "13530", "1681", "17731", "8657", "12757", "3414", "6119", "10192", "19657", "21322", "5110", "2234", "15319", "22340", "7932", "22621", "4565", "6286", "25230", "5631", "20988", "11553", "10727", "16272", "10103", "25275", "24157", "19603", "10227", "5448", "2877", "5034", "446", "702", "13163", "7134", "6580", "14908", "24387", "19133", "13608", "2438", "10277", "13271", "593", "5962", "18262", "5497", "21412", "8578", "24142", "324", "2121", "9276", "13665", "8264", "22628", "13286", "22897", "7089", "5949", "447", "6444", "22051", "11170", "14642", "1418", "8915", "18358", "18723", "23779", "3019", "7215", "5259", "9019", "19614", "16383", "14215", "15695", "6564", "17182", "225", "13731", "12037", "277", "18926", "763", "15646", "8066", "23929", "9922", "6019", "9634", "14097", "19995", "10850", "9182", "11703", "20984", "6847", "4767", "16622", "25150", "9833", "7653", "13155", "19980", "8720", "11075", "22295", "21654", "6519", "18016", "23576", "6164", "21239", "15622", "1323", "11827", "20703", "948", "3973", "12270", "21229", "16948", "19809", "11158", "461", "12232", "7245", "25912", "11085", "1857", "8229", "19873", "17989", "14950", "6252", "15404", "2148", "7794", "14649", "19036", "936", "15096", "22467", "5124", "18469", "18195", "22220", "21402", "18497", "4714", "10386", "6021", "16842", "3549", "22022", "22214", "5930", "8997", "18669", "5738", "19545", "5809", "12638", "5238", "3427", "4307", "12083", "6943", "21736", "14845", "17362", "1505", "18829", "8306", "3492", "19936", "1701", "14602", "8057", "20290", "15606", "21192", "4186", "17467", "19360", "17313", "7879", "12313", "21616", "22178", "18423", "12922", "15374", "3394", "10786", "12750", "22832", "4603", "25476", "11965", "20920", "21461", "12265", "20673", "22247", "2963", "793", "9359", "22756", "18296", "1788", "4986", "10717", "10462", "16542", "20911", "16711", "13709", "7351", "7239", "5252", "24606", "14965", "2258", "22503", "6420", "8473", "17681", "20700", "7236", "14109", "13461", "24723", "17939", "16065", "16370", "2922", "18449", "12682", "11141", "12579", "7237", "18394", "6478", "25590", "6028", "15360", "12649", "8887", "11155", "8162", "5811", "25644", "9615", "16871", "19993", "8450", "10393", "6303", "25522", "22629", "17269", "2851", "20270", "23685", "25395", "17687", "5943", "3936", "353", "15371", "24660", "21230", "18428", "22121", "24882", "21472", "12309", "2887", "22312", "10623", "23422", "22323", "878", "17092", "11316", "10202", "4220", "3457", "21974", "23756", "7842", "9347", "13138", "21350", "16938", "6689", "11064", "18085", "13323", "13574", "6284", "21067", "23027", "15366", "10567", "18818", "11519", "303", "24338", "17716", "4141", "8159", "7648", "11403", "14359", "14733", "591", "24563", "22103", "24330", "10492", "8788", "19234", "25009", "22415", "24287", "18537", "16641", "13338", "13366", "21881", "12470", "21570", "7711", "22184", "11381", "10648", "16769", "48", "19604", "17910", "18567", "25133", "13846", "8010", "14186", "16880", "18574", "22660", "19136", "7880", "5366", "2722", "2274", "14090", "5000", "12847", "8360", "7224", "2733", "17128", "15943", "9886", "16352", "12573", "15939", "23637", "4981", "10405", "10436", "11514", "8209", "12745", "18169", "1353", "17473", "24752", "9761", "8964", "823", "11594", "13215", "3144", "15562", "5105", "3642", "13691", "2295", "17489", "7911", "5426", "9026", "9552", "24846", "10585", "11050", "427", "8961", "9199", "2451", "20009", "18030", "3186", "9631", "23913", "8551", "5532", "7141", "17340", "16747", "24030", "10286", "9579", "24076", "14960", "1532", "10193", "9671", "554", "20179", "7503", "12859", "24158", "24217", "23052", "11460", "13178", "8867", "16094", "4770", "9523", "2415", "12273", "16668", "16652", "23138", "3162", "24613", "2381", "1940", "11193", "24938", "20746", "2624", "2522", "5309", "12919", "1780", "22693", "23718", "11197", "21978", "11233", "16873", "6125", "23870", "11290", "23945", "20427", "18194", "7545", "8523", "14453", "5815", "6361", "21358", "11356", "15725", "23072", "2780", "14797", "617", "19347", "23578", "3557", "11048", "17684", "12329", "12697", "15407", "16475", "21586", "14972", "17095", "23617", "12850", "6597", "24558", "7194", "13333", "15094", "15122", "10385", "86", "5398", "15487", "10291", "381", "9622", "8496", "22019", "5130", "22293", "32", "23997", "17556", "9678", "1789", "24871", "3334", "7625", "10664", "16820", "15632", "24053", "109", "5410", "4552", "25372", "15608", "15189", "11696", "24650", "23080", "16957", "8266", "5270", "18679", "22944", "21688", "19479", "12997", "25176", "18596", "20288", "20648", "15376", "21158", "13046", "1294", "24128", "3066", "1779", "9955", "21122", "11646", "4478", "10905", "10002", "22383", "5534", "20600", "17216", "14884", "17134", "935", "12560", "3017", "16417", "11353", "23740", "9310", "15950", "722", "5202", "13134", "18931", "25174", "43", "23787", "19999", "16320", "4797", "13295", "15365", "5910", "15274", "23692", "4320", "2427", "19161", "13312", "17848", "13773", "9184", "22664", "4183", "10943", "24992", "25965", "12650", "3416", "4840", "24670", "9946", "25892", "19927", "545", "12720", "9485", "1444", "6364", "4570", "14252", "11630", "3607", "11286", "22623", "22699", "9710", "19916", "5402", "25047", "24261", "412", "16567", "2701", "16408", "4869", "21293", "4750", "11001", "3904", "16812", "23900", "9153", "3822", "13099", "664", "14952", "3758", "1009", "11937", "13594", "8078", "18409", "778", "6686", "17123", "6129", "19699", "1209", "23546", "15294", "10891", "25255", "5976", "19071", "6248", "8848", "13299", "15090", "17105", "23714", "12450", "7710", "9018", "13393", "11004", "9882", "11728", "23662", "19261", "24433", "21115", "4678", "10517", "14170", "24579", "18678", "4396", "24312", "3469", "1977", "14538", "20527", "12089", "8294", "6360", "1278", "9223", "10521", "15836", "11920", "16227", "17504", "13733", "5824", "14264", "890", "880", "17965", "15172", "3781", "5787", "7895", "17704", "20081", "11167", "14404", "11137", "22804", "16451", "23268", "23677", "3746", "12486", "23416", "22412", "12185", "15381", "4099", "6980", "1027", "25574", "17363", "21508", "12781", "636", "9375", "17176", "22125", "23960", "19063", "13988", "2013", "5406", "18547", "11495", "25380", "6578", "24781", "17507", "10196", "20782", "23598", "1545", "17554", "458", "2948", "21747", "13516", "8559", "22032", "16933", "14923", "2325", "11341", "16344", "23911", "15185", "18214", "2431", "21019", "17520", "11921", "13395", "6212", "6733", "648", "23764", "17673", "15549", "14630", "14705", "7178", "16929", "20217", "6703", "18337", "15231", "15193", "4445", "21693", "3333", "17052", "17551", "21021", "11185", "21826", "18620", "10679", "11266", "24911", "13781", "19148", "20588", "2310", "7243", "8663", "4112", "22613", "9296", "13047", "17301", "19687", "5646", "114", "25490", "3704", "11922", "10191", "8860", "12630", "11593", "161", "8671", "4644", "15647", "14694", "12704", "13126", "5123", "9246", "17938", "24024", "571", "9947", "1727", "23667", "19668", "23127", "14704", "22281", "11304", "20612", "10003", "2475", "12413", "21505", "15775", "12833", "21696", "4375", "4577", "6734", "13079", "12881", "19223", "16382", "3069", "926", "11734", "24789", "19186", "14894", "14226", "24866", "4580", "4280", "3005", "25122", "25939", "1350", "22285", "25973", "9716", "17129", "20063", "19546", "145", "19217", "19014", "2332", "24596", "15026", "22614", "16296", "21782", "10399", "19501", "2600", "25967", "4796", "25149", "18475", "19495", "647", "17539", "17086", "18372", "7225", "17238", "7418", "22210", "15112", "14615", "21464", "4707", "5020", "24439", "16035", "11342", "21273", "20839", "13907", "9913", "23075", "19565", "1296", "13202", "13787", "2668", "3588", "2105", "13803", "18516", "2586", "1804", "19118", "3674", "23310", "25109", "1571", "19384", "24906", "7064", "2452", "19577", "12238", "19661", "22007", "996", "20752", "970", "16416", "5418", "4162", "21573", "3673", "15054", "1292", "5665", "13926", "2724", "7599", "18603", "2687", "21201", "7293", "11384", "21995", "23458", "21934", "11882", "570", "16261", "14354", "3443", "13406", "3307", "10830", "25052", "2839", "14053", "6052", "16558", "25224", "24455", "18470", "24274", "20451", "14378", "25102", "2937", "20421", "16623", "10382", "21578", "6588", "2800", "7896", "1945", "21678", "255", "22403", "23899", "21202", "4494", "10520", "24632", "7268", "20245", "2721", "675", "3619", "11917", "4020", "229", "1007", "20314", "20318", "14626", "24239", "797", "15288", "18036", "346", "14402", "2630", "13176", "20691", "20414", "3978", "4010", "18794", "22908", "17930", "3814", "344", "4746", "11118", "19562", "5766", "1161", "24042", "17535", "21845", "19908", "25782", "17537", "11325", "11234", "3461", "19812", "19031", "14773", "22632", "3606", "2029", "16651", "13857", "18485", "4353", "6197", "4019", "18034", "10417", "4852", "6865", "14175", "19050", "23706", "9921", "15720", "23028", "14768", "7263", "12200", "24377", "17210", "639", "8104", "4222", "24393", "18845", "3842", "15182", "16016", "5091", "8931", "17723", "22483", "20322", "24011", "12593", "2303", "25685", "9500", "7951", "12572", "2802", "24631", "5612", "8052", "16091", "12218", "9700", "1432", "17565", "25757", "16691", "25862", "13247", "3432", "4194", "24411", "20382", "25649", "22481", "4107", "6073", "19663", "18764", "22054", "2069", "18653", "3212", "2769", "17415", "11953", "6648", "2547", "16316", "5038", "18509", "11881", "2292", "3646", "18199", "24900", "6356", "12707", "25675", "24328", "19808", "25094", "19617", "7294", "3185", "14811", "14924", "20672", "13641", "16112", "23917", "11581", "2", "424", "18826", "17418", "4617", "15919", "13729", "3922", "15766", "23076", "22267", "10093", "16323", "14148", "5799", "16678", "105", "10629", "8531", "10018", "7981", "8978", "2296", "1057", "7538", "2985", "23449", "7202", "15619", "12193", "23524", "3013", "8409", "21104", "22555", "19376", "15267", "17370", "17486", "15769", "25618", "25972", "1387", "14564", "21392", "8834", "19523", "2135", "14185", "19362", "9806", "15505", "11964", "9829", "12285", "20528", "17719", "7563", "5233", "1393", "22363", "9874", "24447", "21761", "25070", "15835", "11834", "23923", "790", "16393", "13089", "25609", "24555", "24759", "20943", "10856", "24581", "14253", "18576", "13817", "9513", "10968", "25022", "662", "23606", "18907", "5589", "10083", "9958", "23750", "7282", "1166", "6739", "5619", "25653", "1300", "15677", "15412", "22156", "8667", "7798", "9770", "8372", "19190", "18334", "25010", "14499", "3129", "12972", "14605", "71", "3844", "25529", "3221", "12932", "20519", "14655", "25576", "15078", "22027", "25941", "21711", "3419", "7114", "10716", "3999", "5846", "4805", "12201", "19940", "13521", "24183", "6381", "3153", "17454", "6561", "18400", "9539", "15237", "19721", "5037", "22202", "14736", "4608", "24977", "23650", "7907", "20471", "11330", "13945", "20972", "15879", "7110", "22208", "15574", "13905", "11298", "961", "3811", "20596", "8666", "14027", "3055", "11765", "7422", "5607", "23042", "22037", "1366", "11217", "24805", "10754", "25926", "3631", "10885", "24854", "21591", "22059", "7380", "10880", "3264", "5415", "11915", "3538", "10432", "15449", "17261", "15434", "24605", "1549", "8497", "23834", "5995", "17626", "24172", "4206", "17944", "16683", "23012", "2556", "7833", "17714", "13619", "9616", "19417", "16635", "4346", "12680", "726", "21366", "1618", "19028", "4922", "3225", "134", "19845", "5653", "25240", "12933", "5819", "11993", "16115", "8540", "12951", "10357", "23390", "1498", "14037", "9139", "18168", "11894", "20959", "25118", "8359", "6137", "20999", "16249", "8317", "6470", "12819", "12160", "18429", "19280", "3935", "19415", "21323", "24898", "16723", "25113", "7751", "24972", "19504", "11435", "4942", "24072", "20209", "16819", "1909", "6140", "3843", "12253", "11952", "17642", "12092", "5711", "22573", "14936", "19041", "15477", "15019", "11967", "14546", "25414", "10927", "13579", "4235", "5550", "18780", "19172", "10913", "23104", "6074", "25558", "15763", "22245", "13501", "11916", "9851", "4431", "12834", "8795", "2461", "9951", "21365", "11591", "3798", "4774", "23415", "5561", "7073", "10533", "13589", "4481", "17953", "24614", "6162", "12662", "1404", "2739", "25456", "892", "20300", "17495", "20130", "11322", "8757", "9122", "17482", "7405", "192", "14035", "14789", "12343", "17834", "16410", "13759", "5586", "4237", "23867", "12722", "15961", "17392", "21473", "4334", "12767", "16402", "21649", "6931", "8076", "17808", "24966", "24812", "4298", "874", "21020", "5993", "13679", "3520", "25877", "7561", "6374", "24796", "6107", "20945", "19171", "23318", "16968", "3056", "15570", "365", "19811", "10509", "22668", "4951", "22569", "18197", "4736", "10547", "8595", "12171", "16576", "5204", "13682", "15192", "15849", "13630", "10408", "21418", "2569", "9363", "9342", "21784", "22014", "8600", "7990", "19357", "20780", "16019", "16118", "25809", "9983", "373", "3446", "16032", "16712", "14409", "21250", "25415", "18487", "25781", "705", "14113", "22449", "25527", "24548", "6336", "16800", "10602", "11803", "17795", "1167", "2228", "13306", "15850", "14119", "22435", "18298", "3072", "16687", "14007", "25838", "11868", "23995", "2791", "21326", "15134", "514", "5002", "25248", "3923", "9060", "25293", "11313", "22468", "12795", "8475", "4489", "24899", "23640", "21013", "2690", "5059", "25078", "3267", "5429", "14646", "21946", "6515", "9953", "360", "15399", "9445", "18177", "10897", "21533", "2744", "19742", "10556", "25370", "22177", "19266", "5380", "21105", "5222", "17970", "5769", "22556", "223", "4304", "4934", "19772", "23235", "25347", "326", "16185", "18164", "300", "23472", "9731", "21787", "25950", "4828", "7859", "19696", "11664", "1023", "337", "20811", "16219", "18063", "22496", "19964", "22017", "17950", "12780", "11509", "16430", "19181", "15298", "13227", "25828", "3942", "23481", "19315", "24300", "24706", "20218", "15784", "1487", "25312", "16864", "1843", "3255", "24891", "11306", "15332", "13339", "20671", "15708", "25336", "22673", "5349", "3838", "706", "15103", "16133", "19896", "40", "7039", "5225", "14143", "18244", "23681", "10680", "21292", "20781", "6439", "16937", "5667", "12234", "16049", "11254", "26010", "730", "22964", "24160", "21383", "11866", "20644", "25103", "25430", "21022", "16177", "17045", "16014", "3784", "13992", "3380", "9003", "20111", "23846", "20560", "5326", "15609", "12101", "25002", "4385", "5731", "1682", "19487", "6293", "4773", "3026", "16239", "5768", "5065", "4611", "16904", "5637", "16792", "23509", "13434", "3532", "4910", "6903", "6264", "15285", "25105", "6796", "17942", "12091", "230", "2382", "3012", "2420", "7694", "2513", "21285", "24262", "1105", "4031", "23202", "24022", "3753", "20349", "1792", "24178", "21240", "14769", "17895", "19139", "26008", "7813", "7185", "11192", "18828", "20232", "17497", "24983", "3338", "22850", "9388", "12420", "8558", "2190", "14436", "20281", "19952", "13549", "6250", "24205", "7576", "17146", "12254", "6347", "6384", "933", "16175", "1419", "9117", "1408", "25652", "25273", "14036", "4716", "6653", "10144", "9669", "16228", "5723", "21575", "24191", "11224", "12263", "15551", "18708", "16308", "17275", "3900", "13006", "13283", "16114", "10512", "11189", "18624", "6716", "9507", "22358", "10415", "8967", "8025", "3166", "16967", "1064", "19733", "2972", "18519", "12626", "24718", "23508", "15485", "20762", "18526", "25917", "23081", "4825", "14080", "6952", "25220", "12181", "5176", "5850", "22830", "25359", "14123", "24401", "18226", "4146", "19463", "9480", "10678", "9909", "23048", "19769", "2132", "14401", "6155", "9129", "25794", "20787", "1934", "124", "21027", "15825", "14254", "22003", "11425", "16978", "23658", "10726", "7038", "17728", "6933", "21026", "22476", "2831", "18205", "25478", "3517", "635", "5861", "23250", "11162", "10220", "7804", "24255", "3570", "20394", "20028", "22745", "17677", "20107", "13480", "1325", "3663", "11985", "22109", "10877", "20447", "8033", "19774", "180", "9919", "3169", "25860", "2075", "20702", "10089", "2184", "5041", "19860", "10511", "1937", "20253", "4790", "1003", "2639", "13651", "10681", "6170", "37", "22691", "4438", "3638", "25966", "2207", "16959", "23338", "1150", "7484", "17506", "9674", "19189", "20820", "6000", "6713", "5438", "1080", "24570", "13359", "20030", "25729", "17865", "15875", "25901", "4680", "14188", "18782", "1642", "24602", "25216", "1688", "24810", "17628", "1561", "13137", "10315", "14567", "3738", "20488", "9014", "261", "658", "10008", "11612", "17225", "19119", "14651", "7260", "348", "25798", "10675", "19783", "12128", "20435", "23309", "24668", "13073", "19981", "24887", "7135", "2346", "15157", "22170", "11775", "11429", "13915", "1875", "11929", "6852", "7190", "14826", "5264", "14786", "19485", "15461", "24503", "20473", "10412", "7581", "17591", "518", "18621", "11241", "21590", "17345", "4798", "24872", "17638", "18746", "2951", "18440", "5022", "21496", "8958", "1189", "10079", "2456", "15618", "23025", "24984", "21000", "22953", "20258", "9553", "23683", "9844", "7893", "21488", "12931", "22001", "6253", "25896", "14447", "15120", "10038", "11909", "21436", "4727", "12805", "1450", "409", "14074", "17610", "4050", "15032", "16739", "9307", "17945", "4182", "1314", "23783", "18345", "951", "8190", "8898", "18870", "14750", "17042", "23474", "22155", "12231", "7037", "9581", "3076", "15624", "17338", "25195", "13756", "15741", "6468", "1588", "9974", "3680", "10445", "8763", "23270", "15576", "6632", "9106", "4214", "19629", "5465", "11963", "17088", "18369", "20045", "11230", "1560", "19498", "3530", "22936", "17593", "25227", "14592", "11372", "13802", "5901", "328", "8088", "15656", "7919", "13493", "1531", "2501", "8869", "1301", "1477", "13633", "18384", "10579", "18857", "2510", "10157", "678", "14365", "490", "17997", "13041", "18406", "19102", "18690", "20524", "83", "359", "8982", "13488", "11690", "4127", "22548", "5215", "12331", "6518", "19587", "23723", "18849", "3828", "25029", "952", "7943", "12874", "3719", "10783", "22113", "6928", "3863", "7820", "24497", "6467", "19857", "11005", "23123", "3246", "24436", "9656", "18491", "14565", "18810", "25089", "17836", "12511", "22653", "23844", "2727", "9", "7891", "1621", "24056", "9222", "11101", "12248", "15132", "13914", "12196", "11428", "20877", "4748", "3305", "1266", "1028", "10475", "6756", "5931", "12155", "24126", "23778", "17860", "25017", "20327", "9174", "22912", "19101", "21214", "12116", "263", "18083", "9057", "17567", "21869", "15476", "4819", "5008", "6628", "5344", "9463", "24577", "11823", "26015", "24953", "21864", "9087", "1341", "9439", "1854", "12267", "3749", "23908", "24774", "11532", "25348", "25657", "7068", "17891", "3495", "1864", "11044", "16571", "8787", "24923", "418", "6459", "9263", "27", "13452", "18556", "23271", "2096", "20035", "24803", "2493", "8959", "19371", "18916", "18793", "3288", "3190", "6354", "4903", "8355", "12027", "16998", "11800", "12044", "18741", "19781", "23895", "15987", "25301", "182", "4749", "24489", "7873", "8238", "7868", "6143", "3692", "9299", "20019", "20409", "16583", "15745", "4517", "4911", "17564", "11213", "21834", "22204", "22803", "15694", "10296", "25789", "12503", "6998", "19075", "7915", "14069", "22741", "23926", "24576", "979", "16398", "16384", "9923", "2794", "503", "13969", "2643", "20017", "23876", "19567", "13460", "19880", "6506", "25477", "19793", "21668", "17771", "5096", "17465", "18644", "15395", "3393", "10132", "16206", "12024", "23364", "22575", "698", "24069", "14919", "7888", "10686", "10569", "7468", "24376", "985", "4", "8265", "21083", "5737", "6473", "4670", "9085", "4281", "21827", "10090", "7326", "19370", "12436", "24716", "12872", "14232", "2361", "14872", "9996", "5229", "12531", "1170", "25436", "15999", "7470", "6451", "22139", "3164", "644", "24837", "9373", "24986", "10347", "13473", "21269", "13539", "9419", "8486", "6846", "8747", "23070", "16942", "23425", "18283", "22047", "7000", "21312", "15083", "2492", "25030", "5283", "24600", "8350", "1749", "1504", "16531", "24337", "16697", "6075", "18548", "24216", "11267", "24485", "1517", "3351", "19067", "10672", "17889", "9487", "19215", "14501", "2277", "3959", "22729", "14164", "1070", "15236", "19213", "6171", "16314", "1140", "9111", "1962", "21295", "7668", "19966", "20113", "24595", "17817", "21768", "20120", "11398", "6571", "13854", "16325", "13129", "23741", "24492", "20118", "2958", "24288", "13831", "15037", "20174", "25226", "20561", "5994", "10831", "18356", "24601", "10906", "24326", "24697", "21453", "11895", "15597", "15552", "19830", "17341", "8326", "6622", "2101", "17224", "20892", "9496", "18593", "14139", "5622", "15211", "16104", "20983", "24116", "21008", "7905", "13816", "16477", "23094", "22278", "5672", "4380", "1834", "4905", "24194", "13656", "3229", "16806", "7566", "13775", "18667", "845", "15885", "23548", "18963", "7675", "15071", "16180", "16385", "24994", "2521", "3748", "24223", "13661", "22675", "20581", "5171", "15710", "604", "1709", "21290", "24507", "16053", "5568", "21102", "6180", "15534", "6741", "22348", "4601", "14291", "16958", "15179", "14939", "8214", "12256", "16135", "20712", "4126", "2545", "24634", "17124", "8328", "3875", "5560", "6413", "8491", "9455", "8439", "10053", "737", "18115", "10793", "3353", "950", "2318", "19833", "11297", "19168", "5585", "10057", "11010", "10060", "12771", "24112", "7308", "489", "7369", "7985", "5437", "2462", "2588", "4919", "22325", "11962", "9488", "19579", "6993", "2205", "15511", "25670", "17619", "24399", "16537", "21318", "3366", "24201", "20757", "9691", "3497", "4512", "23196", "17587", "10960", "15955", "10262", "6618", "20426", "15814", "9754", "18779", "15682", "6577", "2362", "19775", "1286", "2023", "5", "12284", "13398", "10114", "13302", "3805", "18330", "17909", "539", "21762", "7230", "4283", "6983", "3275", "17108", "4811", "21957", "22099", "24771", "12660", "17078", "5527", "22659", "20655", "23690", "20832", "20298", "14435", "6687", "11786", "6991", "7386", "16310", "25993", "12206", "5733", "3903", "14805", "5830", "8073", "23761", "1142", "19749", "18287", "22516", "21769", "22290", "11455", "7474", "18595", "19196", "8526", "2416", "17038", "5327", "17046", "3117", "10465", "15664", "5115", "10834", "20203", "16598", "8344", "13562", "5875", "2957", "8713", "4818", "828", "6967", "10011", "12611", "18520", "22", "8054", "22244", "11845", "2592", "5530", "17782", "21854", "9343", "15034", "2977", "23238", "4522", "4724", "20021", "7447", "20350", "749", "6626", "24680", "18238", "19435", "22251", "25995", "22128", "3352", "24946", "471", "16505", "22986", "13777", "21886", "23962", "24093", "25307", "4979", "7158", "18785", "17455", "10137", "16474", "6058", "13014", "1711", "2848", "6296", "12715", "237", "3405", "16743", "16010", "3787", "12484", "9468", "21199", "24195", "7220", "11838", "6484", "24495", "15611", "21066", "4686", "735", "15979", "17691", "21990", "14765", "24432", "25193", "4728", "12008", "10595", "16902", "17753", "18939", "6548", "15398", "9428", "6853", "9105", "6094", "23044", "22411", "5544", "8369", "7841", "14376", "18581", "18402", "1619", "11258", "21603", "1606", "13938", "25678", "7435", "25686", "5674", "16502", "15330", "15561", "9504", "486", "1946", "4687", "22654", "8732", "23197", "1703", "2765", "6656", "7166", "25289", "11194", "16327", "8564", "20139", "19219", "20626", "16259", "8165", "5382", "6426", "8957", "18609", "20685", "669", "16268", "22663", "13466", "20025", "20320", "380", "8639", "17432", "13611", "20286", "23234", "20105", "15194", "13605", "23825", "13573", "19945", "10810", "23964", "18143", "25101", "24295", "14180", "7119", "20629", "18930", "16720", "24783", "5145", "552", "19559", "13571", "23539", "11760", "18243", "24127", "17353", "16389", "5315", "6966", "18898", "7051", "19729", "4667", "19530", "7026", "16582", "18124", "24505", "17024", "14663", "9719", "11275", "25873", "22853", "25984", "25980", "2796", "15344", "7530", "2311", "19674", "16356", "23555", "17973", "260", "972", "6003", "7829", "23642", "7671", "14497", "14367", "8210", "2138", "5638", "18145", "20163", "2783", "17735", "24294", "14863", "8032", "25712", "22779", "18057", "18522", "19998", "22213", "1024", "25188", "50", "4389", "12512", "1486", "9470", "25316", "23700", "4269", "6315", "4991", "21811", "6754", "13556", "25745", "1783", "5580", "23357", "2719", "1637", "15057", "19918", "9469", "21251", "2161", "3683", "13253", "25125", "2383", "10667", "8160", "4657", "10979", "15043", "22712", "24828", "2775", "100", "10346", "11062", "21996", "24945", "7869", "19335", "1645", "358", "18110", "13581", "21209", "14381", "10253", "5564", "11083", "219", "12911", "21237", "7975", "7234", "14102", "19591", "8255", "8287", "16846", "15013", "1430", "12808", "18405", "25623", "16153", "10872", "19898", "13990", "23789", "17954", "17048", "17555", "4662", "16217", "22565", "19630", "13279", "9809", "22236", "21396", "15011", "22840", "18684", "12860", "22258", "9430", "12495", "11432", "6456", "24352", "22622", "24676", "572", "19252", "22455", "20098", "23163", "11098", "23595", "3234", "21941", "20374", "5064", "3447", "3832", "23461", "1569", "14791", "14518", "3858", "10693", "18074", "18714", "1004", "25225", "2333", "20310", "12708", "20509", "18399", "20568", "20221", "1998", "10821", "16818", "7784", "8332", "22882", "25812", "10287", "542", "12894", "13547", "22751", "11547", "13206", "16653", "20086", "5195", "9871", "22858", "15003", "25605", "14622", "672", "14556", "5705", "12581", "23699", "11763", "22475", "9901", "3251", "10268", "22607", "5082", "6027", "5605", "16472", "6788", "24654", "20736", "12362", "6667", "680", "12308", "20875", "4367", "10503", "21233", "18459", "12386", "8351", "7624", "23109", "5952", "23510", "5554", "18237", "9846", "16009", "6160", "1121", "1172", "5699", "25261", "22697", "20102", "10886", "3620", "17232", "23454", "22802", "6992", "23135", "19337", "19924", "10051", "19518", "25161", "14098", "3498", "16313", "1248", "9207", "11799", "17172", "12469", "11058", "2140", "5640", "25391", "18628", "11637", "20343", "17235", "13098", "17695", "2960", "2870", "1837", "16263", "24423", "19996", "12432", "4527", "15465", "12445", "9609", "25349", "12068", "20148", "11501", "16757", "4780", "9645", "11886", "19992", "15794", "15651", "21926", "1612", "8187", "6682", "5168", "21622", "16462", "23977", "2861", "14051", "17447", "9042", "24802", "415", "4093", "17926", "10322", "10215", "8123", "21699", "25332", "12686", "1040", "2261", "465", "425", "5713", "24152", "18266", "11656", "1809", "13426", "23061", "16423", "21341", "25911", "12801", "22962", "7344", "868", "11736", "5486", "18196", "434", "24146", "14227", "10498", "15655", "20097", "19459", "15750", "8228", "23403", "14559", "13925", "2469", "3528", "6797", "22140", "6257", "22334", "18427", "23290", "645", "24253", "15679", "7476", "4548", "312", "19746", "25641", "6259", "5332", "17570", "519", "3004", "1290", "3990", "20686", "15621", "14843", "4078", "12986", "14677", "12608", "8077", "2139", "8199", "10745", "473", "3451", "15891", "23788", "12921", "949", "8121", "120", "1029", "8510", "884", "4256", "14099", "10587", "5338", "19827", "22844", "21911", "14591", "23316", "7095", "9228", "13269", "9191", "7433", "5751", "8539", "20074", "21289", "8590", "20680", "1885", "11625", "3776", "9959", "20854", "9929", "4851", "24937", "14342", "3982", "12380", "6429", "24920", "5421", "17234", "10611", "10887", "11801", "22488", "17890", "21245", "20709", "13392", "17007", "16805", "262", "24749", "16196", "22926", "16738", "11643", "19741", "9250", "8723", "1910", "4232", "9443", "16512", "23611", "15852", "12983", "11500", "5684", "3271", "17914", "4783", "20657", "7123", "8463", "5785", "13898", "14132", "23603", "500", "8468", "3077", "8319", "13881", "25863", "19240", "16572", "21849", "4042", "4890", "16994", "8945", "19117", "21537", "19447", "1034", "16602", "15230", "3731", "14968", "3762", "12490", "7636", "4949", "24165", "8513", "20172", "24674", "18681", "1483", "18869", "3840", "14606", "8281", "9838", "21219", "2397", "18961", "18001", "9152", "21178", "1851", "10984", "7381", "18109", "809", "632", "9818", "25210", "7699", "34", "2157", "26030", "7980", "9243", "25403", "10248", "4390", "2185", "24480", "15761", "17089", "9835", "24313", "3263", "13326", "18601", "14926", "18353", "5312", "3601", "13019", "23519", "2064", "18736", "19493", "24692", "11399", "8295", "4793", "7922", "14548", "23441", "2389", "13565", "4510", "4929", "10919", "11026", "20500", "5920", "13009", "3171", "14825", "8712", "22440", "8580", "20739", "24835", "6748", "11261", "4209", "222", "19162", "12134", "18758", "24957", "830", "2394", "10335", "17849", "10178", "16425", "1691", "6729", "14675", "5922", "20145", "16445", "9076", "19325", "391", "7623", "8562", "1507", "9353", "25895", "24639", "22710", "2448", "1770", "9911", "17526", "13465", "24587", "8596", "23071", "18623", "3362", "25413", "12770", "24862", "12108", "5407", "18888", "25640", "1954", "102", "8256", "2763", "24404", "7207", "18848", "6905", "13384", "25626", "18269", "11396", "2685", "14243", "5403", "4789", "12291", "1524", "19046", "15584", "19214", "25409", "24251", "7533", "3364", "20807", "11103", "9652", "9281", "17546", "1592", "1930", "23698", "5578", "12065", "14718", "8311", "10568", "14445", "22558", "14561", "5024", "21799", "25260", "25338", "7942", "2776", "2603", "23255", "15957", "22747", "21795", "11777", "24125", "24583", "20406", "20134", "11423", "13300", "16108", "17794", "5231", "3182", "10252", "1321", "24850", "6146", "10078", "20592", "14545", "17683", "18524", "25475", "24965", "12987", "14004", "22322", "1157", "11793", "14933", "6497", "9420", "8748", "14064", "10467", "370", "11107", "3303", "25264", "9214", "4159", "3354", "16589", "22504", "5721", "3350", "15444", "11472", "12164", "15806", "16362", "25126", "21301", "11528", "8638", "11910", "2698", "7961", "2779", "1307", "25871", "17933", "19253", "9621", "12930", "14610", "9478", "21952", "4054", "21096", "3112", "24926", "19471", "23777", "20831", "10575", "4863", "9471", "18697", "9138", "6670", "8876", "21892", "5673", "7718", "12899", "8085", "15450", "8808", "14212", "11171", "18301", "15781", "15636", "23308", "66", "24132", "25461", "7080", "3418", "668", "9744", "22942", "23317", "13711", "10123", "624", "9012", "14842", "93", "6668", "14330", "977", "15989", "21742", "4843", "1893", "23897", "13422", "3976", "9978", "17201", "6584", "4325", "16260", "25165", "17137", "25082", "19285", "6174", "16244", "14031", "11695", "16659", "5703", "11084", "20976", "14512", "9380", "4403", "15524", "22098", "10566", "2308", "9257", "17311", "13382", "18554", "6339", "20204", "24502", "24879", "21793", "18658", "22749", "23478", "16584", "812", "3506", "24910", "23216", "20855", "7131", "4395", "1182", "24936", "13020", "16548", "10425", "9591", "3428", "7850", "13054", "670", "11253", "3231", "11847", "16189", "3308", "15488", "17051", "13169", "21836", "16174", "5710", "18864", "15333", "17297", "15423", "13932", "432", "11174", "7029", "1359", "20919", "8096", "2154", "9867", "19157", "12726", "20489", "16130", "19451", "17055", "22705", "12112", "24269", "22967", "22790", "18563", "17228", "16987", "17002", "14947", "23213", "8986", "15015", "19427", "2324", "9232", "16499", "16749", "17538", "10221", "12619", "3281", "592", "7862", "6714", "25650", "9756", "14426", "9659", "14075", "2980", "3751", "738", "19243", "15965", "19239", "8200", "3035", "5446", "8100", "24979", "2155", "8308", "17391", "2127", "9453", "21872", "517", "2032", "23220", "613", "7805", "17788", "23702", "11182", "9213", "4048", "15119", "15607", "11614", "16660", "14984", "16052", "9050", "21009", "1663", "580", "5657", "25922", "15111", "24787", "1360", "2226", "24285", "7010", "24171", "6494", "11222", "10127", "5615", "17604", "14246", "15688", "19202", "13636", "5630", "6189", "15822", "4518", "14653", "6824", "22611", "22120", "11027", "7809", "12936", "22514", "19174", "24629", "19797", "4121", "25426", "20076", "9692", "10447", "26007", "8662", "19464", "7441", "17026", "16170", "5253", "25879", "12442", "3645", "1139", "20280", "4969", "3687", "17254", "6691", "204", "4546", "15353", "4598", "22012", "3269", "18579", "8367", "9354", "21476", "2159", "3503", "16359", "2107", "20565", "15653", "18468", "16885", "9069", "4151", "15820", "21411", "5139", "16478", "19160", "4684", "2884", "11768", "3848", "16357", "10639", "2770", "15358", "7630", "2967", "4492", "16017", "13056", "9297", "9073", "13769", "5999", "16963", "20264", "4710", "22772", "3775", "22959", "19901", "4971", "5652", "8799", "17992", "8694", "11974", "10591", "9311", "13663", "14887", "21751", "16152", "25308", "17577", "19484", "3581", "9739", "2819", "21147", "15498", "1850", "11269", "8404", "12959", "5132", "5384", "19528", "2231", "15275", "16144", "5481", "4857", "17218", "4877", "18042", "24832", "15162", "5547", "24117", "707", "754", "7469", "21821", "20144", "1252", "22820", "1948", "916", "696", "15500", "17451", "2009", "18549", "12258", "25710", "22888", "2422", "13222", "5435", "21695", "7181", "14441", "17433", "14295", "8017", "1497", "23633", "8142", "5276", "18444", "22000", "20520", "7045", "17617", "13078", "24784", "6935", "11030", "23014", "7866", "11907", "18481", "22219", "17069", "10652", "23179", "20077", "20743", "15911", "17065", "15558", "15256", "9447", "10441", "24193", "8573", "11401", "78", "25600", "14532", "2555", "12388", "16351", "1790", "6005", "10242", "2275", "7689", "23041", "21132", "14966", "8167", "14309", "15672", "14644", "1456", "2133", "24655", "11848", "11711", "18705", "20874", "1088", "13940", "20226", "2572", "23920", "14427", "758", "22786", "10138", "24161", "1158", "14303", "19780", "21467", "3376", "1906", "23609", "12392", "6831", "4290", "9528", "1878", "24291", "2670", "2405", "8019", "16716", "15092", "13070", "10264", "1916", "17350", "22913", "8802", "13192", "8740", "13197", "16326", "11433", "20050", "14002", "6715", "10267", "11994", "10707", "4692", "10847", "11481", "17170", "1055", "931", "16950", "9745", "7070", "279", "10657", "16269", "17931", "530", "10013", "1708", "5009", "14943", "4342", "23623", "16435", "15585", "5953", "23645", "4358", "8920", "6906", "16600", "5742", "23448", "15800", "4363", "24961", "16279", "5236", "23810", "17233", "17821", "24542", "17792", "22595", "7335", "8972", "405", "17359", "12673", "14487", "289", "19208", "8293", "12963", "25604", "5362", "16522", "9722", "23026", "5185", "13400", "22717", "941", "21114", "20925", "213", "11794", "23433", "11752", "18052", "19483", "24027", "7889", "11995", "18361", "24514", "6371", "20188", "1186", "22725", "12337", "14335", "21620", "824", "22901", "19703", "8758", "22965", "24283", "22104", "23089", "9266", "18286", "9832", "15164", "14945", "8467", "10270", "13325", "768", "17503", "8549", "6779", "557", "13805", "6737", "7681", "19222", "6061", "21014", "11779", "20220", "4519", "14346", "10243", "7619", "3539", "5016", "18500", "14194", "8117", "12584", "11675", "6567", "23615", "7150", "8064", "10082", "19198", "3778", "8026", "7317", "3043", "12124", "18318", "15675", "2729", "21671", "22998", "21885", "18489", "6195", "1381", "2964", "5272", "12852", "9994", "10894", "16839", "8641", "18958", "19686", "7048", "19278", "13849", "21057", "16464", "8413", "7851", "15715", "2692", "4429", "12733", "9381", "16613", "12599", "21241", "19881", "13318", "19911", "22884", "3696", "3283", "25725", "14229", "17847", "2737", "17299", "12458", "19748", "9268", "1853", "2944", "4321", "14510", "4482", "11156", "16955", "9696", "8861", "22726", "23477", "12848", "12030", "25614", "25822", "2571", "9064", "13935", "16508", "18692", "1932", "10261", "3728", "5977", "23340", "24773", "15225", "12648", "11935", "22227", "17320", "8148", "21045", "14422", "3984", "19983", "24260", "3094", "5077", "4550", "11989", "3893", "841", "8675", "23431", "14103", "2483", "21665", "6233", "6084", "2898", "12790", "480", "6890", "7979", "16926", "11096", "4760", "584", "3574", "7238", "10061", "23296", "15545", "16025", "19594", "24088", "4538", "4400", "22519", "24290", "10974", "12730", "24278", "25382", "4387", "3317", "21950", "13294", "8684", "17449", "1375", "21637", "12328", "20993", "8635", "2880", "3927", "3071", "12427", "21587", "4738", "14620", "6574", "13415", "25412", "19609", "21050", "19257", "24973", "13989", "2604", "13007", "25179", "7892", "19608", "1485", "7015", "19561", "8443", "25708", "18281", "3226", "18957", "7404", "3609", "306", "20699", "6050", "7440", "18634", "15118", "1083", "25519", "25913", "16304", "23600", "13042", "5218", "14183", "4000", "22834", "21499", "20659", "3374", "21809", "7551", "13504", "13544", "14909", "21378", "13870", "25375", "2031", "12803", "9550", "17434", "9080", "6215", "15036", "10041", "15418", "13994", "18230", "19425", "1435", "20020", "3057", "15069", "20542", "13320", "12513", "14514", "19632", "6076", "23663", "3322", "526", "20605", "16575", "529", "3372", "22508", "18", "11891", "12694", "7012", "6113", "20274", "4155", "16258", "4594", "24371", "12344", "21035", "10694", "9910", "9059", "20769", "3444", "8612", "15773", "14652", "8971", "17568", "25441", "2518", "15903", "14421", "6802", "14448", "4314", "10411", "4274", "21056", "22972", "4834", "21048", "21604", "5376", "8906", "10479", "6299", "3400", "8766", "2855", "24018", "11872", "859", "3702", "7684", "21759", "16524", "22704", "15697", "12902", "21967", "2104", "25139", "2878", "8484", "17508", "19895", "20448", "4559", "7987", "25247", "11183", "21288", "4488", "13770", "4673", "12325", "1175", "3365", "24643", "4531", "17553", "12677", "21710", "20784", "2373", "1238", "7586", "11017", "16983", "21912", "23494", "9555", "5477", "13118", "505", "19409", "9623", "18900", "16664", "22561", "24772", "1445", "22252", "6166", "18632", "7954", "9031", "18465", "14400", "21424", "7722", "12954", "9877", "13635", "19310", "23462", "12475", "14049", "19556", "22423", "16042", "8143", "19725", "22550", "11972", "19864", "24361", "7926", "15340", "22355", "7336", "23492", "9198", "4498", "807", "4730", "343", "21459", "22841", "2366", "407", "17882", "14657", "24344", "5817", "14456", "23062", "18990", "2093", "2044", "21072", "24556", "10944", "4853", "22070", "23455", "6408", "13991", "7283", "24848", "1786", "2653", "1402", "24075", "5323", "19232", "25820", "10317", "7286", "12928", "14914", "25124", "12917", "7006", "13425", "4871", "1858", "7756", "24422", "23203", "16621", "24572", "17967", "24591", "9847", "16516", "2036", "19260", "18501", "17144", "6340", "18454", "17636", "21356", "13646", "10222", "20397", "10300", "1877", "21755", "17830", "7654", "15755", "962", "20616", "10174", "13383", "19886", "842", "11751", "8956", "3931", "13104", "16823", "25997", "2883", "22028", "24956", "377", "25747", "5401", "18998", "9728", "8785", "12582", "9365", "4878", "21177", "16667", "5548", "18650", "4052", "14013", "1201", "22380", "12178", "10161", "2745", "11753", "14568", "24873", "12409", "19249", "12385", "19292", "25239", "8084", "3907", "23453", "4452", "5101", "19225", "5740", "23866", "19058", "6774", "13833", "22337", "16569", "4426", "13360", "13658", "24731", "25421", "8103", "4956", "15912", "25783", "24932", "20714", "25660", "4207", "19254", "13334", "12377", "13109", "22798", "20211", "651", "16300", "25469", "5690", "18293", "8304", "22409", "4477", "6418", "16164", "5650", "24486", "20858", "6642", "25738", "11225", "21327", "6091", "1195", "9416", "18271", "2344", "17470", "15684", "586", "24199", "9928", "19307", "9401", "9474", "497", "14091", "4815", "25465", "21655", "8471", "8254", "10409", "24454", "16633", "1282", "16713", "5965", "20160", "10491", "12289", "5230", "1973", "11056", "7357", "14356", "6266", "5265", "24545", "7856", "23034", "7088", "6405", "17104", "7408", "18772", "13133", "24180", "3654", "12276", "10524", "20112", "22205", "2820", "7491", "9379", "7108", "759", "1407", "25330", "22985", "990", "2227", "5151", "13486", "11729", "22661", "440", "19017", "13941", "18532", "11943", "6471", "25234", "13788", "1447", "20557", "23098", "15509", "22327", "13074", "13470", "6554", "2061", "8650", "8618", "11462", "17493", "23370", "11034", "2712", "16686", "4927", "4574", "6238", "16411", "15592", "5896", "21613", "2538", "15341", "10636", "6722", "15992", "22783", "20125", "19030", "22810", "18467", "15958", "10021", "14710", "12867", "19006", "1302", "3595", "6981", "128", "5826", "19199", "24089", "14241", "16528", "4480", "23886", "17373", "16058", "4017", "13230", "4635", "22413", "329", "7413", "21243", "18874", "4376", "567", "21515", "4759", "13430", "20110", "16057", "10030", "3050", "17832", "21903", "24029", "4564", "22759", "4382", "21469", "21311", "18947", "9173", "15454", "215", "448", "21788", "4646", "16876", "21640", "12465", "4860", "20517", "22018", "21334", "2102", "6039", "23533", "14195", "5140", "4579", "19323", "11826", "6738", "23313", "18573", "10596", "6940", "12943", "640", "16034", "20193", "4708", "25318", "5045", "5582", "3501", "4113", "3944", "23337", "10307", "9707", "20864", "20734", "3583", "16566", "4148", "6744", "17040", "15554", "19344", "20969", "6013", "879", "23733", "24136", "9862", "1654", "14122", "4361", "8194", "9406", "24000", "19443", "8509", "11407", "23442", "14341", "7345", "9675", "7174", "959", "25073", "21052", "7929", "3871", "8136", "8772", "1240", "23228", "17494", "13416", "10045", "15827", "855", "1723", "532", "10214", "20649", "11492", "1821", "23594", "11450", "19610", "24762", "21257", "24779", "17907", "16543", "12018", "8263", "6306", "16110", "10042", "4455", "8494", "9159", "23959", "5858", "2962", "21150", "23139", "9987", "7810", "22785", "14878", "12471", "24012", "9116", "15698", "3119", "25616", "11041", "3360", "1706", "17901", "21394", "17841", "12415", "203", "4567", "14301", "22809", "7309", "13515", "6557", "11531", "227", "5743", "7799", "17405", "16225", "11245", "18144", "4950", "5351", "14686", "24442", "21242", "23731", "3898", "8914", "23058", "17699", "23002", "8269", "17249", "2250", "16700", "12290", "24533", "19303", "21615", "4745", "16627", "10320", "12127", "24430", "19919", "24420", "7493", "20372", "16549", "6825", "24066", "19707", "17237", "25696", "1552", "10265", "25905", "21554", "5221", "2368", "24645", "14205", "15235", "14475", "25835", "20510", "12925", "7993", "22618", "1551", "18890", "15714", "16045", "9541", "20483", "1965", "21631", "20722", "12087", "25467", "13015", "6235", "21139", "13519", "15251", "13363", "17120", "13157", "22938", "12724", "3835", "1987", "1890", "21058", "17034", "15110", "17854", "1753", "24103", "11053", "11395", "3770", "2817", "1317", "12437", "25199", "20169", "16642", "16412", "2906", "21141", "21231", "23795", "10184", "1244", "18742", "21804", "4838", "10067", "18671", "15787", "24025", "9841", "6945", "4233", "11237", "12642", "11439", "14839", "8091", "19639", "13035", "3149", "25567", "942", "11385", "21549", "22372", "12448", "13541", "361", "18010", "3047", "2875", "17014", "21980", "14716", "8776", "8827", "13322", "12687", "13860", "22599", "12515", "376", "1899", "7955", "24434", "17448", "14249", "23711", "1721", "16426", "5537", "2918", "7622", "2929", "14473", "19912", "16859", "11506", "1648", "12692", "439", "17582", "24357", "4391", "4966", "21060", "15956", "17245", "8203", "23242", "21857", "1499", "8461", "5144", "10100", "7781", "17966", "14285", "20957", "17386", "7792", "25008", "65", "9676", "10040", "1513", "1955", "17998", "3142", "11641", "2442", "8322", "10033", "17263", "16969", "9795", "18106", "4055", "8428", "4357", "10586", "2911", "15354", "8911", "5504", "7148", "2017", "7528", "15196", "12187", "22851", "23174", "623", "20419", "7167", "4073", "10047", "4275", "4622", "15044", "13368", "6989", "5593", "13741", "23348", "9328", "19640", "14782", "22626", "4292", "14290", "19411", "2990", "3173", "10795", "5839", "23324", "22391", "6697", "10816", "21074", "7847", "21397", "16292", "19976", "2947", "6403", "10633", "23059", "2398", "16142", "18666", "8620", "5900", "21204", "20279", "740", "108", "19263", "17531", "4129", "11159", "1053", "7138", "23342", "9066", "20152", "19926", "1889", "25780", "25256", "3618", "3584", "8268", "7154", "15823", "281", "7083", "11248", "4154", "6136", "20669", "23299", "2498", "5577", "25516", "24293", "1528", "24967", "16488", "4458", "17859", "18996", "9167", "9519", "22842", "16636", "12530", "1827", "10598", "1218", "5250", "25662", "12849", "18670", "22207", "8336", "3261", "14932", "6844", "21720", "18382", "12004", "11223", "21760", "7189", "14539", "1382", "5182", "1537", "22138", "3552", "26011", "8334", "1746", "10484", "21314", "189", "5296", "5844", "22165", "6018", "11583", "22289", "8725", "8520", "2860", "10121", "18804", "5263", "3462", "18397", "12017", "7592", "25870", "5983", "5368", "8427", "13218", "16226", "12275", "9613", "11605", "13735", "24122", "9398", "6031", "24306", "21585", "5392", "12179", "3484", "207", "8242", "2260", "23644", "21713", "25268", "12985", "11218", "25214", "17955", "22666", "12900", "3847", "18950", "5325", "24798", "3154", "6606", "331", "17920", "21216", "6889", "15929", "10343", "2682", "5911", "167", "3098", "14086", "20910", "3873", "12627", "21548", "7785", "22425", "3785", "20595", "26031", "16923", "2542", "22437", "3677", "24008", "25243", "19112", "12364", "2143", "20180", "25826", "17761", "23205", "6705", "6964", "25093", "2741", "22045", "23056", "20682", "22446", "3899", "23223", "14823", "12277", "4337", "2895", "21919", "21302", "12615", "1960", "22122", "4980", "15447", "4451", "16299", "8692", "22744", "18096", "7550", "21479", "16367", "13902", "7069", "17640", "3335", "25934", "13354", "2428", "19590", "10497", "495", "13666", "6422", "17062", "3276", "12855", "21916", "7414", "2149", "2539", "3535", "9101", "24544", "11482", "2314", "20608", "24652", "20573", "524", "811", "5829", "21697", "24044", "17624", "10881", "485", "16949", "2365", "17908", "8270", "25615", "25654", "25019", "14958", "6516", "7257", "6287", "9813", "14634", "20687", "19313", "11650", "24226", "3669", "1902", "4924", "7472", "6307", "13479", "304", "6645", "23503", "16909", "4341", "7552", "1037", "21182", "20315", "12887", "17663", "16675", "16387", "24163", "24763", "4406", "24863", "13071", "19029", "5556", "8704", "335", "2445", "23883", "3958", "25497", "25562", "2453", "10452", "9873", "4352", "788", "3991", "20486", "14601", "25756", "4490", "16143", "2587", "3160", "10914", "23748", "1658", "9650", "20928", "917", "17968", "6699", "2047", "4695", "25671", "16647", "18909", "5440", "18294", "20333", "23293", "4809", "13937", "4318", "7330", "15637", "5843", "14910", "11069", "22126", "22259", "19574", "8514", "16702", "19814", "1205", "20879", "16927", "17371", "17131", "20330", "11486", "13115", "25620", "155", "80", "7565", "5187", "4515", "22636", "2993", "16444", "6014", "20546", "8575", "3233", "6337", "15321", "21428", "502", "23812", "1295", "7211", "11488", "14032", "17325", "15322", "13776", "22814", "20239", "16338", "25932", "1518", "4177", "2970", "5822", "11912", "6871", "544", "2126", "15602", "25633", "7832", "13001", "14667", "2323", "3165", "21771", "24554", "21574", "12604", "25846", "16013", "13908", "3791", "18066", "387", "15994", "12609", "9863", "6438", "18323", "6819", "3536", "6453", "7337", "15456", "13107", "8348", "15711", "10273", "20793", "20213", "23443", "14368", "19505", "5474", "15178", "14868", "19765", "19274", "25385", "19651", "25557", "13832", "10661", "18441", "15270", "13105", "7875", "11281", "22750", "21095", "24515", "9364", "23426", "11562", "25292", "15116", "1431", "13964", "10204", "10953", "10207", "13160", "12695", "20411", "25560", "1649", "7251", "12666", "18388", "1556", "22597", "7156", "17824", "17156", "10358", "4576", "15789", "1434", "13784", "11510", "9348", "16626", "2164", "14390", "6535", "25705", "13458", "20689", "24890", "3665", "23593", "25401", "11772", "23873", "7159", "17947", "18300", "11172", "7933", "15960", "367", "13686", "9924", "24792", "5546", "1133", "19667", "15279", "18664", "11754", "18569", "8476", "24853", "6712", "15258", "4140", "5992", "22672", "19502", "19273", "25343", "20601", "7397", "5918", "16650", "3329", "783", "17081", "22740", "24604", "12725", "17283", "18880", "484", "2068", "4782", "14476", "12149", "16224", "957", "18999", "6784", "13590", "17435", "8027", "8900", "12333", "6767", "8048", "10883", "18184", "5085", "13809", "16887", "13791", "17315", "1086", "1470", "20366", "25213", "4263", "2199", "20299", "14461", "17309", "9518", "16854", "16840", "11624", "21676", "4316", "19986", "19324", "23742", "19458", "9275", "15966", "9472", "5280", "4557", "21399", "1106", "24703", "14833", "15419", "13111", "9177", "17177", "12622", "2073", "15854", "2917", "14928", "22848", "24528", "10900", "20166", "13242", "15188", "5122", "8015", "6152", "21156", "16837", "25222", "22130", "15174", "15573", "14033", "23668", "10977", "9825", "757", "201", "20955", "3605", "6294", "21714", "18507", "8489", "12340", "2246", "20506", "16538", "23331", "9998", "14100", "17442", "19441", "22620", "9992", "12220", "20952", "17707", "19047", "20041", "20523", "23584", "12994", "19718", "17720", "16061", "1794", "11804", "16145", "18304", "9961", "9058", "582", "1370", "11577", "1796", "4800", "11899", "5482", "11144", "4025", "7823", "2978", "16624", "5761", "22700", "1838", "3430", "14332", "8407", "7978", "23091", "22920", "18065", "1044", "4920", "7631", "7370", "5939", "4837", "14808", "21816", "13340", "9399", "12778", "6175", "6037", "18966", "4034", "5724", "16427", "12100", "2801", "5157", "3876", "23120", "4370", "14029", "25845", "4938", "7546", "5924", "24020", "14020", "10203", "15108", "6768", "13485", "5923", "18188", "12053", "152", "13704", "13820", "975", "22062", "10730", "25770", "7904", "23362", "15572", "9290", "24535", "25355", "13613", "15503", "24043", "7494", "19076", "2782", "25552", "3803", "2825", "25718", "24855", "30", "13551", "8930", "20348", "3721", "17010", "1554", "19142", "15780", "12176", "15617", "22557", "3962", "21977", "3934", "17519", "1904", "13861", "3378", "17178", "24756", "1986", "5433", "13170", "24004", "18787", "20917", "3388", "836", "14871", "5579", "20031", "19286", "11332", "20942", "9563", "25517", "7027", "3853", "18396", "19153", "14898", "15397", "8881", "614", "7506", "9151", "16056", "14413", "11607", "22053", "15522", "24564", "14776", "19737", "11317", "151", "15244", "17646", "1938", "13559", "20802", "5587", "10446", "11567", "468", "10321", "13670", "25767", "7526", "20846", "4540", "16645", "14443", "11288", "16021", "14862", "19884", "816", "13388", "25229", "3558", "11161", "21862", "14507", "22078", "17943", "15163", "17569", "6135", "13880", "8051", "17403", "13714", "10990", "3025", "19247", "22196", "16954", "5628", "24980", "17188", "1825", "9772", "19907", "12614", "15195", "2520", "25044", "14407", "21433", "16355", "6727", "2959", "21277", "15052", "12305", "2458", "3877", "25731", "2614", "22624", "12583", "22817", "5092", "3420", "22760", "23988", "17255", "24864", "943", "20724", "307", "2896", "16770", "22587", "3755", "22386", "19084", "5520", "6760", "21888", "3698", "12905", "23894", "9300", "21271", "11947", "11128", "23287", "4079", "11818", "19203", "1630", "11602", "751", "1342", "2678", "18173", "4722", "23321", "22410", "25305", "2684", "12773", "4166", "16270", "20487", "16555", "9251", "10450", "18891", "25084", "6193", "17132", "12948", "24257", "15473", "24974", "69", "4335", "22826", "12205", "1676", "7613", "76", "20091", "5791", "11853", "2979", "25324", "19978", "6916", "3774", "24087", "4861", "24305", "13903", "313", "8505", "3764", "18463", "12637", "17512", "7934", "16255", "12550", "23982", "1069", "11745", "5794", "19648", "23214", "5287", "2696", "3693", "16116", "2626", "14370", "15085", "8307", "5570", "12228", "10749", "15863", "9317", "21776", "22940", "3379", "25341", "25134", "19625", "25379", "18273", "17655", "24190", "13451", "6353", "21452", "14141", "19115", "19356", "7078", "12076", "533", "10646", "13280", "18235", "21645", "24751", "20075", "11740", "25843", "17534", "22791", "18260", "24912", "21429", "9508", "25077", "15147", "22137", "6345", "13554", "20037", "844", "1839", "7165", "12888", "2408", "23173", "1670", "6322", "17805", "7066", "11430", "6029", "8221", "3744", "5027", "25112", "20661", "7147", "19539", "23394", "5576", "11568", "10966", "23888", "22041", "4882", "5347", "16979", "16901", "5838", "2657", "12197", "2913", "6603", "18717", "24571", "22924", "13013", "14778", "546", "23380", "2457", "1107", "19638", "9757", "9013", "3641", "17248", "2262", "21927", "21611", "8738", "4931", "11707", "20071", "3302", "4589", "5982", "6771", "23177", "15383", "10668", "23601", "4147", "17438", "4045", "22727", "558", "1905", "22568", "19026", "14880", "7966", "10414", "14011", "8469", "1985", "5209", "6348", "1171", "18362", "4191", "22781", "17427", "13796", "9934", "1337", "11502", "1824", "3060", "1467", "22709", "4569", "10688", "16392", "11210", "19899", "10638", "9668", "7994", "1484", "22489", "17689", "5331", "20092", "21370", "1942", "9484", "7712", "3912", "17400", "13093", "16860", "7761", "19758", "9043", "3247", "21991", "18131", "25416", "3038", "17633", "51", "22982", "19923", "3128", "19113", "25388", "15932", "15405", "16422", "5894", "17446", "17304", "15856", "20717", "13329", "24799", "20336", "20808", "8592", "24367", "22758", "22774", "7415", "24954", "17155", "13127", "10466", "4450", "10389", "22457", "2982", "25556", "18392", "16293", "19388", "10360", "10280", "10077", "8074", "11764", "23561", "14883", "4377", "10763", "7828", "23638", "8912", "12974", "8483", "8584", "22542", "978", "12616", "21431", "15905", "131", "11255", "8155", "1262", "20798", "15389", "13468", "22951", "17000", "3827", "19702", "3966", "14057", "25697", "13535", "13236", "21047", "8783", "6204", "10481", "22684", "18130", "15753", "4593", "15363", "14010", "9532", "17037", "3516", "9865", "23794", "11566", "18455", "13265", "4634", "18696", "10156", "19807", "4471", "2222", "6006", "2523", "6047", "5311", "13545", "9219", "10928", "101", "5226", "19027", "10430", "22480", "13436", "22923", "4145", "18807", "1667", "22315", "5249", "6619", "19436", "23987", "3450", "834", "2028", "999", "19470", "17962", "6700", "10825", "5340", "22433", "10736", "10345", "25505", "15909", "10731", "1339", "16480", "24580", "16525", "15219", "2425", "22377", "1454", "6493", "10186", "15141", "7887", "22188", "14137", "22605", "22918", "8522", "1179", "2374", "13864", "18048", "14479", "15821", "5404", "21308", "12194", "15878", "21910", "8180", "14001", "17502", "22226", "22563", "2713", "19404", "19628", "12564", "14248", "5469", "15022", "20938", "20308", "21218", "7432", "3941", "17659", "1104", "6067", "3883", "19695", "8880", "15507", "12701", "20622", "24019", "10284", "3084", "20774", "2514", "17644", "3125", "20485", "21023", "2736", "2283", "13274", "1907", "6221", "8179", "5387", "21417", "22701", "23504", "20070", "921", "17893", "456", "6575", "16222", "24135", "2173", "3082", "10292", "12335", "2840", "19537", "6888", "21832", "11677", "22381", "3623", "13863", "24971", "3608", "1200", "15427", "2871", "15869", "20392", "22376", "22657", "18088", "7557", "22473", "1397", "7137", "20767", "6623", "21935", "20567", "10840", "15372", "8642", "20554", "7816", "2938", "6968", "585", "23971", "25217", "1214", "18026", "9377", "7041", "20024", "20817", "21071", "23935", "10337", "16836", "9033", "23129", "15348", "22724", "3946", "9078", "13233", "23697", "18894", "11986", "21351", "384", "8018", "1883", "22332", "7299", "23775", "22069", "16267", "764", "22812", "9362", "19456", "5243", "1822", "16719", "7713", "10014", "25534", "23160", "23574", "12330", "14607", "26028", "16546", "12071", "15565", "12402", "22450", "12398", "13681", "22081", "23619", "20307", "15216", "16229", "4961", "224", "23551", "15345", "6732", "7745", "18587", "20639", "19777", "23695", "17685", "10390", "17826", "4555", "17393", "9255", "16076", "17566", "2476", "3080", "17529", "25799", "10288", "11471", "8883", "13273", "15335", "9372", "1833", "17651", "13836", "22123", "7241", "6112", "14223", "25869", "14151", "15339", "11397", "25178", "12552", "16759", "3866", "15050", "2419", "2315", "23172", "5472", "8853", "4258", "9742", "17381", "3575", "18578", "19997", "3676", "13585", "20791", "7807", "10119", "13369", "148", "12467", "8529", "9556", "4928", "10938", "9932", "23921", "24465", "12623", "8515", "18377", "6501", "24921", "9126", "13183", "5452", "3652", "22186", "1576", "13961", "18311", "22648", "16396", "4279", "15149", "6839", "14294", "1468", "13185", "24037", "22004", "22821", "15872", "13266", "10996", "14723", "1823", "16993", "18417", "5536", "1010", "7534", "16515", "4536", "19554", "5621", "3158", "4900", "10245", "14827", "17653", "13357", "15241", "20069", "25994", "24683", "17517", "3564", "17862", "24608", "3779", "25548", "12843", "14340", "12033", "9084", "19892", "5313", "16131", "19532", "20383", "4854", "627", "22257", "9781", "15833", "10112", "16858", "12300", "7319", "22742", "6922", "11382", "10645", "9036", "19963", "15458", "12215", "18895", "17688", "13811", "2302", "6500", "1066", "13977", "19938", "24463", "12806", "15791", "16915", "21275", "8318", "5363", "13950", "7389", "9395", "5599", "18207", "3265", "7937", "8251", "21774", "9741", "2379", "10626", "1515", "20707", "9812", "4892", "2109", "17412", "3603", "11065", "16869", "8195", "24458", "4841", "24730", "6363", "12478", "20809", "5308", "8700", "965", "18942", "22639", "22223", "16048", "17816", "12966", "15278", "24897", "23069", "18009", "15311", "19563", "14041", "14063", "15309", "24215", "21089", "17988", "10780", "7790", "3670", "3088", "2975", "1974", "22635", "9694", "1259", "20407", "6772", "25136", "25218", "8768", "16953", "3089", "16311", "5083", "20915", "6969", "5039", "23696", "3398", "21415", "20970", "6210", "26022", "21382", "5849", "26021", "19972", "4572", "19392", "15445", "4500", "3482", "24248", "3280", "24167", "8230", "17934", "10724", "1220", "22273", "20002", "15084", "8699", "16132", "13673", "10651", "23417", "24612", "3031", "8173", "21870", "15325", "2907", "22331", "15778", "25050", "11697", "9796", "16779", "7827", "7589", "11273", "14851", "15900", "8405", "22200", "9672", "20913", "588", "10473", "12785", "19475", "11071", "18889", "11639", "11410", "14323", "10776", "1414", "2276", "11087", "10785", "12824", "4605", "12242", "13162", "747", "16117", "20637", "21700", "15976", "5473", "25924", "19710", "616", "9894", "25511", "492", "14306", "10324", "2828", "23065", "2570", "23527", "2224", "14331", "16637", "11949", "4502", "16275", "21232", "15895", "1924", "8542", "4858", "16136", "1707", "1424", "19380", "23525", "24512", "17911", "5275", "9701", "17800", "57", "6247", "7960", "21134", "14658", "13723", "4970", "10755", "16649", "22173", "18021", "8153", "25328", "19974", "9245", "19291", "8166", "12891", "15290", "25472", "23978", "118", "1544", "22615", "6310", "2306", "16495", "13489", "14066", "7783", "2439", "3099", "21101", "15263", "10759", "1644", "3472", "23302", "19893", "13132", "7633", "9861", "2281", "8275", "1923", "7227", "25829", "8429", "837", "19284", "746", "24200", "11115", "15466", "481", "7162", "6512", "8394", "2865", "7532", "12754", "6165", "15287", "12732", "17079", "11681", "31", "15133", "14046", "23465", "14729", "3653", "21423", "2823", "20161", "8459", "5454", "21614", "9771", "22696", "19062", "19697", "17936", "7852", "18017", "21154", "1390", "24006", "3768", "792", "6464", "20765", "10250", "10867", "3945", "10059", "1952", "13122", "1767", "11585", "221", "2640", "4968", "16349", "14262", "2703", "5302", "2663", "8310", "4600", "24032", "4660", "6581", "10748", "4239", "5991", "12818", "16339", "6104", "46", "20516", "24504", "16487", "16334", "6706", "20870", "21708", "14662", "22233", "14513", "4696", "9602", "3715", "9880", "19928", "8361", "14817", "20788", "23024", "22479", "12056", "23516", "2606", "3592", "17702", "4995", "5633", "22215", "15767", "14110", "15159", "12978", "23371", "6904", "1897", "23151", "18312", "3390", "13812", "9370", "14403", "445", "9047", "17785", "2649", "6386", "1268", "22166", "13401", "20891", "1540", "22527", "7605", "24170", "9146", "24131", "19393", "19877", "16420", "24874", "24916", "4906", "17294", "17548", "11945", "543", "7193", "15523", "14683", "20122", "7936", "895", "19536", "14005", "21258", "25909", "15170", "8388", "24742", "11345", "355", "9699", "5191", "16671", "6402", "22678", "23530", "10151", "23641", "14530", "20932", "17274", "21702", "5416", "7256", "22580", "7786", "2506", "11657", "11310", "9598", "21443", "7516", "667", "22947", "5359", "12644", "25707", "23972", "10341", "20718", "8909", "4410", "2664", "20670", "4532", "9254", "5780", "24829", "21557", "2388", "21535", "11551", "20502", "7160", "6431", "9319", "24845", "17023", "25816", "18210", "22008", "22950", "25508", "16063", "5620", "10313", "3995", "12752", "1398", "3137", "1846", "9236", "15713", "11687", "11529", "23278", "13168", "4102", "14633", "6612", "21831", "2754", "6460", "5357", "23672", "3555", "271", "1183", "21961", "18533", "25453", "20023", "16125", "870", "5506", "10402", "5375", "9218", "15630", "1570", "18553", "12654", "891", "15892", "5972", "10812", "14321", "4978", "19372", "5927", "600", "3424", "1510", "16364", "7140", "1982", "10811", "11586", "19819", "4682", "21987", "23762", "10756", "2789", "3399", "22608", "8062", "24118", "17030", "19296", "18674", "15685", "6079", "10181", "3600", "10422", "24551", "9053", "21729", "19817", "1046", "17710", "2715", "5012", "19615", "21284", "3795", "8309", "23188", "19619", "1925", "11701", "13828", "25431", "8208", "6823", "18701", "9898", "21560", "541", "16822", "8649", "13568", "20503", "3888", "25170", "1757", "4268", "2254", "18599", "15113", "24701", "4915", "18967", "5307", "2243", "25208", "7144", "8120", "8389", "9566", "25496", "3851", "14324", "11792", "3134", "20602", "25753", "6934", "9703", "11992", "9282", "22368", "15176", "3357", "12742", "1641", "7830", "9629", "7222", "13840", "10009", "15301", "4344", "16220", "23550", "16506", "8448", "11825", "4137", "12826", "22311", "13625", "25684", "17075", "12260", "25274", "6856", "4758", "21584", "4038", "7096", "15765", "4418", "17367", "17581", "5792", "16874", "470", "12419", "24448", "18582", "13844", "20062", "16371", "16971", "12322", "9295", "7439", "13792", "18355", "13627", "19839", "18612", "8276", "23925", "21276", "8445", "22229", "1198", "23628", "21400", "2816", "25545", "4076", "20052", "14809", "17164", "21550", "24437", "5337", "5019", "21234", "14846", "606", "5478", "4701", "6254", "14721", "12596", "16120", "21887", "17016", "19866", "21856", "14140", "7388", "13409", "8574", "14096", "3254", "5348", "16036", "5295", "11371", "2666", "9997", "4988", "23676", "15180", "11879", "19732", "12578", "19091", "74", "12417", "3009", "6702", "8593", "2200", "5722", "14761", "4022", "23556", "25257", "15168", "4043", "8925", "13101", "7587", "9688", "4810", "12233", "13965", "21146", "8975", "24869", "14360", "24893", "11611", "8601", "25647", "12675", "19851", "6863", "5563", "20960", "21830", "24369", "18920", "8110", "7196", "5300", "3756", "1535", "23566", "5118", "23374", "10155", "11715", "2213", "17960", "3326", "16865", "24520", "7277", "16717", "5912", "7578", "22642", "13587", "8863", "21738", "723", "2976", "15513", "4807", "18307", "23167", "20003", "6704", "10295", "7640", "3371", "19701", "13760", "3830", "20292", "22723", "24212", "1695", "15519", "4504", "13647", "7664", "17399", "16964", "5423", "3465", "9705", "24385", "21579", "19798", "82", "2336", "23244", "25095", "2752", "10384", "2735", "7244", "24009", "16301", "11590", "7453", "15067", "397", "17453", "15756", "11003", "8023", "9995", "25852", "17230", "21929", "13026", "15959", "7204", "25595", "16609", "15271", "11850", "1473", "5150", "10794", "24050", "4424", "14803", "1785", "4133", "12036", "596", "8591", "12435", "17956", "22794", "25000", "4799", "18819", "3808", "513", "5682", "15557", "11458", "17193", "3460", "24481", "13801", "3573", "2709", "20547", "16401", "8302", "3070", "13975", "5466", "4756", "1429", "15268", "22148", "24478", "21685", "18835", "25903", "3566", "9868", "22764", "20804", "13108", "21208", "17352", "19824", "24686", "16204", "12345", "3463", "14060", "25451", "25565", "11673", "17713", "22528", "23506", "22736", "11979", "22902", "16619", "19932", "10036", "1222", "24510", "11613", "6892", "16961", "22486", "7649", "10502", "23245", "3845", "6318", "13494", "11856", "16482", "4039", "23560", "8380", "7306", "14535", "3437", "25111", "2774", "20064", "13874", "3391", "10461", "21167", "21320", "562", "17609", "8174", "9814", "11588", "23247", "12508", "6763", "5632", "21015", "5388", "23307", "6641", "3925", "21734", "12188", "17061", "21348", "12683", "715", "14454", "22427", "2859", "13362", "2879", "3315", "9870", "35", "9253", "3569", "18521", "11244", "18436", "25143", "14689", "24538", "12055", "19888", "9037", "22903", "5759", "21175", "24235", "498", "16067", "23017", "4219", "23168", "3783", "17287", "6562", "9286", "5431", "12846", "13428", "3542", "17402", "12840", "10663", "17963", "11513", "6270", "9386", "11190", "15109", "13082", "11344", "16796", "3589", "8924", "15927", "13345", "23161", "14855", "22224", "3614", "21221", "14668", "19400", "9109", "13526", "11536", "14820", "10015", "16484", "17397", "13897", "6262", "4291", "19072", "22767", "13980", "25483", "958", "7482", "24425", "14844", "15648", "3224", "23039", "17407", "11440", "20455", "7836", "24424", "12379", "25489", "7212", "8393", "6532", "6241", "1281", "6996", "20985", "3123", "938", "7146", "576", "6127", "12537", "13928", "17739", "9641", "12012", "23569", "23363", "15603", "13459", "15813", "2756", "2623", "22360", "2019", "14767", "11634", "24438", "14344", "15914", "11227", "6877", "13386", "832", "23802", "5890", "18309", "17906", "24107", "17271", "19052", "21079", "6822", "17169", "14155", "4196", "10012", "24451", "24532", "22541", "21602", "20797", "22324", "13826", "858", "13402", "25492", "16237", "25635", "25405", "20841", "12793", "16102", "24740", "14489", "4340", "12592", "10017", "1734", "4886", "6987", "9526", "5500", "23183", "2694", "7321", "9336", "26019", "21212", "23341", "12366", "5720", "20956", "21726", "15605", "10179", "24881", "5079", "7099", "25446", "13514", "18141", "6105", "16005", "18982", "15470", "488", "19659", "4453", "9272", "646", "8284", "17983", "4712", "8414", "16976", "2288", "23018", "12526", "18517", "25486", "7143", "19055", "21107", "5683", "19262", "18680", "18464", "13153", "7525", "4119", "10489", "24738", "25906", "12965", "7540", "5512", "8774", "4449", "22837", "13979", "18822", "15665", "21406", "10744", "4946", "14177", "11991", "22080", "18004", "4729", "8815", "8133", "8563", "13669", "104", "1495", "25081", "15414", "22447", "15837", "7958", "3042", "17431", "18575", "23068", "8219", "20701", "6272", "15666", "7229", "7818", "22823", "21580", "2426", "1416", "16563", "4648", "8831", "17426", "20444", "4709", "21054", "16672", "19180", "7574", "14922", "15941", "20518", "23871", "12147", "5524", "16725", "16250", "2168", "8891", "14158", "23410", "17964", "14480", "10571", "3167", "5246", "5614", "16802", "19803", "14308", "15799", "25223", "127", "3852", "7821", "9327", "12488", "9444", "14228", "20303", "5664", "10553", "25137", "12408", "4587", "23536", "2700", "4958", "18912", "18618", "20341", "3544", "6770", "7250", "24272", "22096", "22958", "19070", "13588", "17990", "1522", "12698", "14428", "12156", "6480", "7208", "19586", "7498", "18014", "25667", "13492", "15629", "7362", "13998", "10169", "9473", "2885", "19424", "21135", "22026", "4820", "8948", "1678", "20437", "14142", "23140", "25353", "13464", "21153", "2516", "17114", "13433", "17257", "7266", "13423", "724", "23581", "8456", "22525", "25824", "1539", "9238", "8767", "17814", "25750", "21291", "7111", "23133", "10985", "7759", "22272", "6230", "12182", "234", "13680", "3207", "18041", "2218", "21779", "10377", "7527", "4733", "15612", "20664", "17361", "15530", "14774", "19326", "20231", "12222", "16312", "17724", "24905", "13592", "5286", "1713", "22152", "354", "17331", "18527", "9912", "20833", "8129", "103", "19300", "19837", "19021", "18097", "6664", "8933", "15492", "23329", "6062", "17316", "18704", "22115", "2488", "12668", "4765", "19085", "5643", "5373", "5574", "15559", "2084", "7499", "11157", "14283", "9316", "18557", "17726", "14537", "1356", "17319", "601", "12937", "21485", "7940", "4084", "18606", "23816", "18095", "3217", "20267", "25258", "23099", "20202", "15442", "23280", "22523", "2936", "16527", "6920", "22886", "2055", "18672", "21523", "23934", "7390", "23893", "8832", "6731", "4473", "23171", "25680", "19188", "19902", "15847", "13375", "1462", "11270", "278", "12384", "4092", "14446", "8185", "10594", "8377", "17596", "10851", "10792", "22934", "17456", "3897", "20422", "18827", "11520", "22688", "17229", "12434", "1344", "1464", "2119", "2597", "23368", "21984", "13746", "21223", "13081", "14494", "19003", "12553", "6321", "23373", "25881", "22230", "817", "1006", "15615", "15441", "7757", "2081", "25561", "16062", "4651", "23936", "23841", "95", "22269", "24316", "22838", "1221", "743", "22442", "13887", "3580", "7046", "21197", "22889", "18566", "22551", "20391", "16156", "3456", "12923", "1523", "1110", "18499", "13764", "21931", "21922", "20663", "2930", "8996", "25867", "15098", "3", "14162", "6159", "4786", "8470", "18605", "11950", "25337", "15779", "386", "15297", "16748", "5543", "15018", "13616", "16905", "3658", "9208", "20395", "4688", "5131", "25034", "17215", "710", "9103", "8630", "23999", "22190", "22987", "17387", "16545", "9991", "20233", "9636", "3960", "1933", "11081", "21220", "9201", "8962", "11121", "25564", "21171", "20692", "7982", "18896", "13191", "19844", "11839", "12472", "18777", "17032", "21583", "19643", "3690", "21361", "22330", "20190", "6373", "5170", "18274", "17396", "19557", "20860", "23423", "1665", "8546", "23875", "12523", "829", "10260", "21564", "12114", "20465", "14526", "6887", "15041", "13420", "5336", "9592", "16330", "2528", "21988", "18555", "12657", "23885", "19616", "6950", "25016", "2753", "14216", "10942", "25189", "22432", "13650", "1365", "3253", "963", "183", "11102", "19684", "13442", "10029", "9189", "8567", "15208", "8457", "16682", "25180", "9679", "24776", "3010", "11820", "21274", "8643", "9133", "1131", "17660", "8001", "20507", "16199", "17602", "23189", "17612", "23126", "19760", "5457", "17573", "16595", "3560", "9746", "18227", "14971", "7728", "22063", "621", "20881", "12950", "9599", "19461", "16409", "22702", "20893", "5294", "6276", "15295", "6605", "18838", "8020", "2742", "169", "22971", "15661", "6794", "12131", "13149", "206", "20848", "20683", "6971", "19682", "9435", "10796", "9210", "2832", "10309", "9530", "25720", "24236", "887", "4787", "22874", "23253", "9729", "9193", "20065", "24487", "18053", "19201", "3029", "23218", "6912", "22407", "6789", "248", "21471", "17197", "19691", "9155", "1236", "23877", "23931", "8357", "24351", "11854", "3411", "17332", "4511", "11348", "9954", "13862", "22665", "6450", "756", "16951", "5773", "24597", "3016", "24466", "23990", "14272", "5866", "12901", "23269", "13703", "11699", "6855", "4272", "1547", "18332", "12602", "5584", "7965", "11582", "24188", "442", "7488", "9653", "5689", "3336", "25700", "4223", "8752", "6651", "16746", "17150", "19521", "20352", "12798", "3133", "18146", "16379", "14521", "22149", "15700", "15640", "20454", "11147", "5925", "12026", "24270", "847", "20186", "8314", "2236", "9433", "20380", "10955", "16406", "2474", "16827", "426", "21266", "9643", "10634", "7301", "11029", "16931", "6607", "5680", "663", "5671", "6208", "9402", "10574", "5072", "18200", "17420", "19327", "7262", "16242", "8135", "3157", "6650", "15025", "19683", "25841", "19395", "1836", "7109", "9858", "24523", "15408", "7602", "6097", "5562", "22239", "25611", "3919", "9712", "16677", "23153", "15310", "20759", "11187", "193", "16429", "1562", "17260", "17654", "9062", "8011", "13557", "18910", "10861", "16815", "10233", "10714", "13615", "5183", "23693", "19955", "22989", "16441", "23852", "18577", "3950", "21030", "10272", "20434", "23125", "9654", "24222", "8206", "5508"]]} \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn.config b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn.config new file mode 100644 index 0000000..b681784 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/configs/nas-benchmark/svhn.config @@ -0,0 +1,13 @@ +{ + "scheduler": ["str", "cos"], + "eta_min" : ["float", "0.0"], + "epochs" : ["int", "200"], + "warmup" : ["int", "0"], + "optim" : ["str", "SGD"], + "LR" : ["float", "0.1"], + "decay" : ["float", "0.0005"], + "momentum" : ["float", "0.9"], + "nesterov" : ["bool", "1"], + "criterion": ["str", "Softmax"], + "batch_size": ["int", "256"] +} diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/functions.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/functions.py new file mode 100644 index 0000000..b81f6cd --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/functions.py @@ -0,0 +1,153 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.08 # +##################################################### +import time +import torch +from procedures import prepare_seed, get_optim_scheduler +from nasbench_utils import get_model_infos, obtain_accuracy +from config_utils import dict2config +from log_utils import AverageMeter, time_string, convert_secs2time +from nas_bench_201_models import get_cell_based_tiny_net + + +__all__ = ['evaluate_for_seed', 'pure_evaluate'] + + +def pure_evaluate(xloader, network, criterion=torch.nn.CrossEntropyLoss()): + data_time, batch_time, batch = AverageMeter(), AverageMeter(), None + losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter() + latencies = [] + network.eval() + with torch.no_grad(): + end = time.time() + for i, (inputs, targets) in enumerate(xloader): + targets = targets.cuda(non_blocking=True) + inputs = inputs.cuda(non_blocking=True) + data_time.update(time.time() - end) + # forward + features, logits = network(inputs) + loss = criterion(logits, targets) + batch_time.update(time.time() - end) + if batch is None or batch == inputs.size(0): + batch = inputs.size(0) + latencies.append(batch_time.val - data_time.val) + # record loss and accuracy + prec1, prec5 = obtain_accuracy( + logits.data, targets.data, topk=(1, 5)) + losses.update(loss.item(), inputs.size(0)) + top1.update(prec1.item(), inputs.size(0)) + top5.update(prec5.item(), inputs.size(0)) + end = time.time() + if len(latencies) > 2: + latencies = latencies[1:] + return losses.avg, top1.avg, top5.avg, latencies + + +def procedure(xloader, network, criterion, scheduler, optimizer, mode): + losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter() + if mode == 'train': + network.train() + elif mode == 'valid': + network.eval() + else: + raise ValueError("The mode is not right : {:}".format(mode)) + + data_time, batch_time, end = AverageMeter(), AverageMeter(), time.time() + for i, (inputs, targets) in enumerate(xloader): + if mode == 'train': + scheduler.update(None, 1.0 * i / len(xloader)) + + targets = targets.cuda(non_blocking=True) + if mode == 'train': + optimizer.zero_grad() + # forward + features, logits = network(inputs) + loss = criterion(logits, targets) + # backward + if mode == 'train': + loss.backward() + optimizer.step() + # record loss and accuracy + prec1, prec5 = obtain_accuracy(logits.data, targets.data, topk=(1, 5)) + losses.update(loss.item(), inputs.size(0)) + top1.update(prec1.item(), inputs.size(0)) + top5.update(prec5.item(), inputs.size(0)) + # count time + batch_time.update(time.time() - end) + end = time.time() + return losses.avg, top1.avg, top5.avg, batch_time.sum + + +def evaluate_for_seed(arch_config, config, arch, train_loader, valid_loaders, seed, logger): + prepare_seed(seed) # random seed + net = get_cell_based_tiny_net(dict2config({'name': 'infer.tiny', + 'C': arch_config['channel'], 'N': arch_config['num_cells'], + 'genotype': arch, 'num_classes': config.class_num}, None) + ) + # net = TinyNetwork(arch_config['channel'], arch_config['num_cells'], arch, config.class_num) + if 'ckpt_path' in arch_config.keys(): + ckpt = torch.load(arch_config['ckpt_path']) + ckpt['classifier.weight'] = net.state_dict()['classifier.weight'] + ckpt['classifier.bias'] = net.state_dict()['classifier.bias'] + net.load_state_dict(ckpt) + + flop, param = get_model_infos(net, config.xshape) + logger.log('Network : {:}'.format(net.get_message()), False) + logger.log( + '{:} Seed-------------------------- {:} --------------------------'.format(time_string(), seed)) + logger.log('FLOP = {:} MB, Param = {:} MB'.format(flop, param)) + # train and valid + optimizer, scheduler, criterion = get_optim_scheduler( + net.parameters(), config) + network, criterion = torch.nn.DataParallel(net).cuda(), criterion.cuda() + # network, criterion = torch.nn.DataParallel(net).to(torch.device(f"cuda:{device}")), criterion.to(torch.device(f"cuda:{device}")) + # start training + start_time, epoch_time, total_epoch = time.time( + ), AverageMeter(), config.epochs + config.warmup + train_losses, train_acc1es, train_acc5es, valid_losses, valid_acc1es, valid_acc5es = { + }, {}, {}, {}, {}, {} + train_times, valid_times = {}, {} + for epoch in range(total_epoch): + scheduler.update(epoch, 0.0) + + train_loss, train_acc1, train_acc5, train_tm = procedure( + train_loader, network, criterion, scheduler, optimizer, 'train') + train_losses[epoch] = train_loss + train_acc1es[epoch] = train_acc1 + train_acc5es[epoch] = train_acc5 + train_times[epoch] = train_tm + with torch.no_grad(): + for key, xloder in valid_loaders.items(): + valid_loss, valid_acc1, valid_acc5, valid_tm = procedure( + xloder, network, criterion, None, None, 'valid') + valid_losses['{:}@{:}'.format(key, epoch)] = valid_loss + valid_acc1es['{:}@{:}'.format(key, epoch)] = valid_acc1 + valid_acc5es['{:}@{:}'.format(key, epoch)] = valid_acc5 + valid_times['{:}@{:}'.format(key, epoch)] = valid_tm + + # measure elapsed time + epoch_time.update(time.time() - start_time) + start_time = time.time() + need_time = 'Time Left: {:}'.format(convert_secs2time( + epoch_time.avg * (total_epoch-epoch-1), True)) + logger.log('{:} {:} epoch={:03d}/{:03d} :: Train [loss={:.5f}, acc@1={:.2f}%, acc@5={:.2f}%] Valid [loss={:.5f}, acc@1={:.2f}%, acc@5={:.2f}%]'.format( + time_string(), need_time, epoch, total_epoch, train_loss, train_acc1, train_acc5, valid_loss, valid_acc1, valid_acc5)) + info_seed = {'flop': flop, + 'param': param, + 'channel': arch_config['channel'], + 'num_cells': arch_config['num_cells'], + 'config': config._asdict(), + 'total_epoch': total_epoch, + 'train_losses': train_losses, + 'train_acc1es': train_acc1es, + 'train_acc5es': train_acc5es, + 'train_times': train_times, + 'valid_losses': valid_losses, + 'valid_acc1es': valid_acc1es, + 'valid_acc5es': valid_acc5es, + 'valid_times': valid_times, + 'net_state_dict': net.state_dict(), + 'net_string': '{:}'.format(net), + 'finish-train': True + } + return info_seed diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/__init__.py new file mode 100644 index 0000000..6175653 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/__init__.py @@ -0,0 +1,9 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +# every package does not rely on pytorch or tensorflow +# I tried to list all dependency here: os, sys, time, numpy, (possibly) matplotlib +from .logger import Logger#, PrintLogger +from .meter import AverageMeter +from .time_utils import time_for_file, time_string, time_string_short, time_print, convert_secs2time +from .time_utils import time_string, convert_secs2time diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/logger.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/logger.py new file mode 100644 index 0000000..e60c78f --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/logger.py @@ -0,0 +1,150 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from pathlib import Path +import importlib, warnings +import os, sys, time, numpy as np +if sys.version_info.major == 2: # Python 2.x + from StringIO import StringIO as BIO +else: # Python 3.x + from io import BytesIO as BIO + +if importlib.util.find_spec('tensorflow'): + import tensorflow as tf + + +class PrintLogger(object): + + def __init__(self): + """Create a summary writer logging to log_dir.""" + self.name = 'PrintLogger' + + def log(self, string): + print (string) + + def close(self): + print ('-'*30 + ' close printer ' + '-'*30) + + +class Logger(object): + + def __init__(self, log_dir, seed, create_model_dir=True, use_tf=False): + """Create a summary writer logging to log_dir.""" + self.seed = int(seed) + self.log_dir = Path(log_dir) + self.model_dir = Path(log_dir) / 'checkpoint' + self.log_dir.mkdir (parents=True, exist_ok=True) + if create_model_dir: + self.model_dir.mkdir(parents=True, exist_ok=True) + #self.meta_dir.mkdir(mode=0o775, parents=True, exist_ok=True) + + self.use_tf = bool(use_tf) + self.tensorboard_dir = self.log_dir / ('tensorboard-{:}'.format(time.strftime( '%d-%h', time.gmtime(time.time()) ))) + #self.tensorboard_dir = self.log_dir / ('tensorboard-{:}'.format(time.strftime( '%d-%h-at-%H:%M:%S', time.gmtime(time.time()) ))) + self.logger_path = self.log_dir / 'seed-{:}-T-{:}.log'.format(self.seed, time.strftime('%d-%h-at-%H-%M-%S', time.gmtime(time.time()))) + self.logger_file = open(self.logger_path, 'w') + + if self.use_tf: + self.tensorboard_dir.mkdir(mode=0o775, parents=True, exist_ok=True) + self.writer = tf.summary.FileWriter(str(self.tensorboard_dir)) + else: + self.writer = None + + def __repr__(self): + return ('{name}(dir={log_dir}, use-tf={use_tf}, writer={writer})'.format(name=self.__class__.__name__, **self.__dict__)) + + def path(self, mode): + valids = ('model', 'best', 'info', 'log') + if mode == 'model': return self.model_dir / 'seed-{:}-basic.pth'.format(self.seed) + elif mode == 'best' : return self.model_dir / 'seed-{:}-best.pth'.format(self.seed) + elif mode == 'info' : return self.log_dir / 'seed-{:}-last-info.pth'.format(self.seed) + elif mode == 'log' : return self.log_dir + else: raise TypeError('Unknow mode = {:}, valid modes = {:}'.format(mode, valids)) + + def extract_log(self): + return self.logger_file + + def close(self): + self.logger_file.close() + if self.writer is not None: + self.writer.close() + + def log(self, string, save=True, stdout=False): + if stdout: + sys.stdout.write(string); sys.stdout.flush() + else: + print (string) + if save: + self.logger_file.write('{:}\n'.format(string)) + self.logger_file.flush() + + def scalar_summary(self, tags, values, step): + """Log a scalar variable.""" + if not self.use_tf: + warnings.warn('Do set use-tensorflow installed but call scalar_summary') + else: + assert isinstance(tags, list) == isinstance(values, list), 'Type : {:} vs {:}'.format(type(tags), type(values)) + if not isinstance(tags, list): + tags, values = [tags], [values] + for tag, value in zip(tags, values): + summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)]) + self.writer.add_summary(summary, step) + self.writer.flush() + + def image_summary(self, tag, images, step): + """Log a list of images.""" + import scipy + if not self.use_tf: + warnings.warn('Do set use-tensorflow installed but call scalar_summary') + return + + img_summaries = [] + for i, img in enumerate(images): + # Write the image to a string + try: + s = StringIO() + except: + s = BytesIO() + scipy.misc.toimage(img).save(s, format="png") + + # Create an Image object + img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(), + height=img.shape[0], + width=img.shape[1]) + # Create a Summary value + img_summaries.append(tf.Summary.Value(tag='{}/{}'.format(tag, i), image=img_sum)) + + # Create and write Summary + summary = tf.Summary(value=img_summaries) + self.writer.add_summary(summary, step) + self.writer.flush() + + def histo_summary(self, tag, values, step, bins=1000): + """Log a histogram of the tensor of values.""" + if not self.use_tf: raise ValueError('Do not have tensorflow') + import tensorflow as tf + + # Create a histogram using numpy + counts, bin_edges = np.histogram(values, bins=bins) + + # Fill the fields of the histogram proto + hist = tf.HistogramProto() + hist.min = float(np.min(values)) + hist.max = float(np.max(values)) + hist.num = int(np.prod(values.shape)) + hist.sum = float(np.sum(values)) + hist.sum_squares = float(np.sum(values**2)) + + # Drop the start of the first bin + bin_edges = bin_edges[1:] + + # Add bin edges and counts + for edge in bin_edges: + hist.bucket_limit.append(edge) + for c in counts: + hist.bucket.append(c) + + # Create and write Summary + summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)]) + self.writer.add_summary(summary, step) + self.writer.flush() diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/meter.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/meter.py new file mode 100644 index 0000000..cbb9dd1 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/meter.py @@ -0,0 +1,98 @@ +import numpy as np + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self): + self.reset() + + def reset(self): + self.val = 0.0 + self.avg = 0.0 + self.sum = 0.0 + self.count = 0.0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + def __repr__(self): + return ('{name}(val={val}, avg={avg}, count={count})'.format(name=self.__class__.__name__, **self.__dict__)) + + +class RecorderMeter(object): + """Computes and stores the minimum loss value and its epoch index""" + def __init__(self, total_epoch): + self.reset(total_epoch) + + def reset(self, total_epoch): + assert total_epoch > 0, 'total_epoch should be greater than 0 vs {:}'.format(total_epoch) + self.total_epoch = total_epoch + self.current_epoch = 0 + self.epoch_losses = np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val] + self.epoch_losses = self.epoch_losses - 1 + self.epoch_accuracy= np.zeros((self.total_epoch, 2), dtype=np.float32) # [epoch, train/val] + self.epoch_accuracy= self.epoch_accuracy + + def update(self, idx, train_loss, train_acc, val_loss, val_acc): + assert idx >= 0 and idx < self.total_epoch, 'total_epoch : {} , but update with the {} index'.format(self.total_epoch, idx) + self.epoch_losses [idx, 0] = train_loss + self.epoch_losses [idx, 1] = val_loss + self.epoch_accuracy[idx, 0] = train_acc + self.epoch_accuracy[idx, 1] = val_acc + self.current_epoch = idx + 1 + return self.max_accuracy(False) == self.epoch_accuracy[idx, 1] + + def max_accuracy(self, istrain): + if self.current_epoch <= 0: return 0 + if istrain: return self.epoch_accuracy[:self.current_epoch, 0].max() + else: return self.epoch_accuracy[:self.current_epoch, 1].max() + + def plot_curve(self, save_path): + import matplotlib + matplotlib.use('agg') + import matplotlib.pyplot as plt + title = 'the accuracy/loss curve of train/val' + dpi = 100 + width, height = 1600, 1000 + legend_fontsize = 10 + figsize = width / float(dpi), height / float(dpi) + + fig = plt.figure(figsize=figsize) + x_axis = np.array([i for i in range(self.total_epoch)]) # epochs + y_axis = np.zeros(self.total_epoch) + + plt.xlim(0, self.total_epoch) + plt.ylim(0, 100) + interval_y = 5 + interval_x = 5 + plt.xticks(np.arange(0, self.total_epoch + interval_x, interval_x)) + plt.yticks(np.arange(0, 100 + interval_y, interval_y)) + plt.grid() + plt.title(title, fontsize=20) + plt.xlabel('the training epoch', fontsize=16) + plt.ylabel('accuracy', fontsize=16) + + y_axis[:] = self.epoch_accuracy[:, 0] + plt.plot(x_axis, y_axis, color='g', linestyle='-', label='train-accuracy', lw=2) + plt.legend(loc=4, fontsize=legend_fontsize) + + y_axis[:] = self.epoch_accuracy[:, 1] + plt.plot(x_axis, y_axis, color='y', linestyle='-', label='valid-accuracy', lw=2) + plt.legend(loc=4, fontsize=legend_fontsize) + + + y_axis[:] = self.epoch_losses[:, 0] + plt.plot(x_axis, y_axis*50, color='g', linestyle=':', label='train-loss-x50', lw=2) + plt.legend(loc=4, fontsize=legend_fontsize) + + y_axis[:] = self.epoch_losses[:, 1] + plt.plot(x_axis, y_axis*50, color='y', linestyle=':', label='valid-loss-x50', lw=2) + plt.legend(loc=4, fontsize=legend_fontsize) + + if save_path is not None: + fig.savefig(save_path, dpi=dpi, bbox_inches='tight') + print ('---- save figure {} into {}'.format(title, save_path)) + plt.close(fig) diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/time_utils.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/time_utils.py new file mode 100644 index 0000000..4a0f78e --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/log_utils/time_utils.py @@ -0,0 +1,42 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import time, sys +import numpy as np + +def time_for_file(): + ISOTIMEFORMAT='%d-%h-at-%H-%M-%S' + return '{:}'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) + +def time_string(): + ISOTIMEFORMAT='%Y-%m-%d %X' + string = '[{:}]'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) + return string + +def time_string_short(): + ISOTIMEFORMAT='%Y%m%d' + string = '{:}'.format(time.strftime( ISOTIMEFORMAT, time.gmtime(time.time()) )) + return string + +def time_print(string, is_print=True): + if (is_print): + print('{} : {}'.format(time_string(), string)) + +def convert_secs2time(epoch_time, return_str=False): + need_hour = int(epoch_time / 3600) + need_mins = int((epoch_time - 3600*need_hour) / 60) + need_secs = int(epoch_time - 3600*need_hour - 60*need_mins) + if return_str: + str = '[{:02d}:{:02d}:{:02d}]'.format(need_hour, need_mins, need_secs) + return str + else: + return need_hour, need_mins, need_secs + +def print_log(print_string, log): + #if isinstance(log, Logger): log.log('{:}'.format(print_string)) + if hasattr(log, 'log'): log.log('{:}'.format(print_string)) + else: + print("{:}".format(print_string)) + if log is not None: + log.write('{:}\n'.format(print_string)) + log.flush() diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/__init__.py new file mode 100644 index 0000000..1f31583 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/__init__.py @@ -0,0 +1,4 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from .get_dataset_with_transform import get_datasets diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/aircraft.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/aircraft.py new file mode 100644 index 0000000..e578eb1 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/aircraft.py @@ -0,0 +1,179 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from __future__ import print_function +import torch.utils.data as data +from torchvision.datasets.folder import pil_loader, accimage_loader, default_loader +from PIL import Image +import os +import numpy as np + + +def make_dataset(dir, image_ids, targets): + assert (len(image_ids) == len(targets)) + images = [] + dir = os.path.expanduser(dir) + for i in range(len(image_ids)): + item = (os.path.join(dir, 'data', 'images', + '%s.jpg' % image_ids[i]), targets[i]) + images.append(item) + return images + + +def find_classes(classes_file): + # read classes file, separating out image IDs and class names + image_ids = [] + targets = [] + f = open(classes_file, 'r') + for line in f: + split_line = line.split(' ') + image_ids.append(split_line[0]) + targets.append(' '.join(split_line[1:])) + f.close() + + # index class names + classes = np.unique(targets) + class_to_idx = {classes[i]: i for i in range(len(classes))} + targets = [class_to_idx[c] for c in targets] + + return (image_ids, targets, classes, class_to_idx) + + +class FGVCAircraft(data.Dataset): + """`FGVC-Aircraft `_ Dataset. + Args: + root (string): Root directory path to dataset. + class_type (string, optional): The level of FGVC-Aircraft fine-grain classification + to label data with (i.e., ``variant``, ``family``, or ``manufacturer``). + transform (callable, optional): A function/transform that takes in a PIL image + and returns a transformed version. E.g. ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + loader (callable, optional): A function to load an image given its path. + download (bool, optional): If true, downloads the dataset from the internet and + puts it in the root directory. If dataset is already downloaded, it is not + downloaded again. + """ + url = 'http://www.robots.ox.ac.uk/~vgg/data/fgvc-aircraft/archives/fgvc-aircraft-2013b.tar.gz' + class_types = ('variant', 'family', 'manufacturer') + splits = ('train', 'val', 'trainval', 'test') + + def __init__(self, root, class_type='variant', split='train', transform=None, + target_transform=None, loader=default_loader, download=False): + if split not in self.splits: + raise ValueError('Split "{}" not found. Valid splits are: {}'.format( + split, ', '.join(self.splits), + )) + if class_type not in self.class_types: + raise ValueError('Class type "{}" not found. Valid class types are: {}'.format( + class_type, ', '.join(self.class_types), + )) + self.root = os.path.expanduser(root) + self.root = os.path.join(self.root, 'fgvc-aircraft-2013b') + self.class_type = class_type + self.split = split + self.classes_file = os.path.join(self.root, 'data', + 'images_%s_%s.txt' % (self.class_type, self.split)) + + if download: + self.download() + + (image_ids, targets, classes, class_to_idx) = find_classes(self.classes_file) + samples = make_dataset(self.root, image_ids, targets) + + self.transform = transform + self.target_transform = target_transform + self.loader = loader + + self.samples = samples + self.classes = classes + self.class_to_idx = class_to_idx + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (sample, target) where target is class_index of the target class. + """ + + path, target = self.samples[index] + sample = self.loader(path) + if self.transform is not None: + sample = self.transform(sample) + if self.target_transform is not None: + target = self.target_transform(target) + + return sample, target + + def __len__(self): + return len(self.samples) + + def __repr__(self): + fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + fmt_str += ' Root Location: {}\n'.format(self.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + def _check_exists(self): + return os.path.exists(os.path.join(self.root, 'data', 'images')) and \ + os.path.exists(self.classes_file) + + def download(self): + """Download the FGVC-Aircraft data if it doesn't exist already.""" + from six.moves import urllib + import tarfile + + if self._check_exists(): + return + + # prepare to download data to PARENT_DIR/fgvc-aircraft-2013.tar.gz + print('Downloading %s ... (may take a few minutes)' % self.url) + parent_dir = os.path.abspath(os.path.join(self.root, os.pardir)) + tar_name = self.url.rpartition('/')[-1] + tar_path = os.path.join(parent_dir, tar_name) + data = urllib.request.urlopen(self.url) + + # download .tar.gz file + with open(tar_path, 'wb') as f: + f.write(data.read()) + + # extract .tar.gz to PARENT_DIR/fgvc-aircraft-2013b + data_folder = tar_path.strip('.tar.gz') + print('Extracting %s to %s ... (may take a few minutes)' % (tar_path, data_folder)) + tar = tarfile.open(tar_path) + tar.extractall(parent_dir) + + # if necessary, rename data folder to self.root + if not os.path.samefile(data_folder, self.root): + print('Renaming %s to %s ...' % (data_folder, self.root)) + os.rename(data_folder, self.root) + + # delete .tar.gz file + print('Deleting %s ...' % tar_path) + os.remove(tar_path) + + print('Done!') + + +if __name__ == '__main__': + air = FGVCAircraft('/w14/dataset/fgvc-aircraft-2013b', class_type='manufacturer', split='train', transform=None, + target_transform=None, loader=default_loader, download=False) + print(len(air)) + print(len(air)) + air = FGVCAircraft('/w14/dataset/fgvc-aircraft-2013b', class_type='manufacturer', split='val', transform=None, + target_transform=None, loader=default_loader, download=False) + air = FGVCAircraft('/w14/dataset/fgvc-aircraft-2013b', class_type='manufacturer', split='trainval', transform=None, + target_transform=None, loader=default_loader, download=False) + print(len(air)) + air = FGVCAircraft('/w14/dataset/fgvc-aircraft-2013b/', class_type='manufacturer', split='test', transform=None, + target_transform=None, loader=default_loader, download=False) + print(len(air)) + import pdb; + pdb.set_trace() + print(len(air)) \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/get_dataset_with_transform.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/get_dataset_with_transform.py new file mode 100644 index 0000000..249f403 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/get_dataset_with_transform.py @@ -0,0 +1,304 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +# Modified by Hayeon Lee, Eunyoung Hyung 2021. 03. +################################################## +import os +import sys +import torch +import os.path as osp +import numpy as np +import torchvision.datasets as dset +import torchvision.transforms as transforms +from copy import deepcopy +# from PIL import Image +import random +import pdb +from .aircraft import FGVCAircraft +from .pets import PetDataset +from config_utils import load_config + +Dataset2Class = {'cifar10': 10, + 'cifar100': 100, + 'mnist': 10, + 'svhn': 10, + 'aircraft': 30, + 'pets': 37} + + +class CUTOUT(object): + + def __init__(self, length): + self.length = length + + def __repr__(self): + return ('{name}(length={length})'.format(name=self.__class__.__name__, **self.__dict__)) + + def __call__(self, img): + h, w = img.size(1), img.size(2) + mask = np.ones((h, w), np.float32) + y = np.random.randint(h) + x = np.random.randint(w) + + y1 = np.clip(y - self.length // 2, 0, h) + y2 = np.clip(y + self.length // 2, 0, h) + x1 = np.clip(x - self.length // 2, 0, w) + x2 = np.clip(x + self.length // 2, 0, w) + + mask[y1: y2, x1: x2] = 0. + mask = torch.from_numpy(mask) + mask = mask.expand_as(img) + img *= mask + return img + + +imagenet_pca = { + 'eigval': np.asarray([0.2175, 0.0188, 0.0045]), + 'eigvec': np.asarray([ + [-0.5675, 0.7192, 0.4009], + [-0.5808, -0.0045, -0.8140], + [-0.5836, -0.6948, 0.4203], + ]) +} + + +class Lighting(object): + def __init__(self, alphastd, + eigval=imagenet_pca['eigval'], + eigvec=imagenet_pca['eigvec']): + self.alphastd = alphastd + assert eigval.shape == (3,) + assert eigvec.shape == (3, 3) + self.eigval = eigval + self.eigvec = eigvec + + def __call__(self, img): + if self.alphastd == 0.: + return img + rnd = np.random.randn(3) * self.alphastd + rnd = rnd.astype('float32') + v = rnd + old_dtype = np.asarray(img).dtype + v = v * self.eigval + v = v.reshape((3, 1)) + inc = np.dot(self.eigvec, v).reshape((3,)) + img = np.add(img, inc) + if old_dtype == np.uint8: + img = np.clip(img, 0, 255) + img = Image.fromarray(img.astype(old_dtype), 'RGB') + return img + + def __repr__(self): + return self.__class__.__name__ + '()' + + +def get_datasets(name, root, cutout, use_num_cls=None): + if name == 'cifar10': + mean = [x / 255 for x in [125.3, 123.0, 113.9]] + std = [x / 255 for x in [63.0, 62.1, 66.7]] + elif name == 'cifar100': + mean = [x / 255 for x in [129.3, 124.1, 112.4]] + std = [x / 255 for x in [68.2, 65.4, 70.4]] + elif name.startswith('mnist'): + mean, std = [0.1307, 0.1307, 0.1307], [0.3081, 0.3081, 0.3081] + elif name.startswith('svhn'): + mean, std = [0.4376821, 0.4437697, 0.47280442], [ + 0.19803012, 0.20101562, 0.19703614] + elif name.startswith('aircraft'): + mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] + elif name.startswith('pets'): + mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225] + else: + raise TypeError("Unknow dataset : {:}".format(name)) + + # Data Argumentation + if name == 'cifar10' or name == 'cifar100': + lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(), + transforms.Normalize(mean, std)] + if cutout > 0: + lists += [CUTOUT(cutout)] + train_transform = transforms.Compose(lists) + test_transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize(mean, std)]) + xshape = (1, 3, 32, 32) + elif name.startswith('cub200'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + xshape = (1, 3, 32, 32) + elif name.startswith('mnist'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Lambda(lambda x: x.repeat(3, 1, 1)), + transforms.Normalize(mean, std), + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Lambda(lambda x: x.repeat(3, 1, 1)), + transforms.Normalize(mean, std) + ]) + xshape = (1, 3, 32, 32) + elif name.startswith('svhn'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + xshape = (1, 3, 32, 32) + elif name.startswith('aircraft'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std), + ]) + xshape = (1, 3, 32, 32) + elif name.startswith('pets'): + train_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std) + ]) + test_transform = transforms.Compose([ + transforms.Resize((32, 32)), + transforms.ToTensor(), + transforms.Normalize(mean=mean, std=std), + ]) + xshape = (1, 3, 32, 32) + else: + raise TypeError("Unknow dataset : {:}".format(name)) + + if name == 'cifar10': + train_data = dset.CIFAR10( + root, train=True, transform=train_transform, download=True) + test_data = dset.CIFAR10( + root, train=False, transform=test_transform, download=True) + assert len(train_data) == 50000 and len(test_data) == 10000 + elif name == 'cifar100': + train_data = dset.CIFAR100( + root, train=True, transform=train_transform, download=True) + test_data = dset.CIFAR100( + root, train=False, transform=test_transform, download=True) + assert len(train_data) == 50000 and len(test_data) == 10000 + elif name == 'mnist': + train_data = dset.MNIST( + root, train=True, transform=train_transform, download=True) + test_data = dset.MNIST( + root, train=False, transform=test_transform, download=True) + assert len(train_data) == 60000 and len(test_data) == 10000 + elif name == 'svhn': + train_data = dset.SVHN(root, split='train', + transform=train_transform, download=True) + test_data = dset.SVHN(root, split='test', + transform=test_transform, download=True) + assert len(train_data) == 73257 and len(test_data) == 26032 + elif name == 'aircraft': + train_data = FGVCAircraft(root, class_type='manufacturer', split='trainval', + transform=train_transform, download=False) + test_data = FGVCAircraft(root, class_type='manufacturer', split='test', + transform=test_transform, download=False) + assert len(train_data) == 6667 and len(test_data) == 3333 + elif name == 'pets': + train_data = PetDataset(root, train=True, num_cl=37, + val_split=0.15, transforms=train_transform) + test_data = PetDataset(root, train=False, num_cl=37, + val_split=0.15, transforms=test_transform) + else: + raise TypeError("Unknow dataset : {:}".format(name)) + + class_num = Dataset2Class[name] if use_num_cls is None else len( + use_num_cls) + return train_data, test_data, xshape, class_num + + +def get_nas_search_loaders(train_data, valid_data, dataset, config_root, batch_size, workers, num_cls=None): + if isinstance(batch_size, (list, tuple)): + batch, test_batch = batch_size + else: + batch, test_batch = batch_size, batch_size + if dataset == 'cifar10': + # split_Fpath = 'configs/nas-benchmark/cifar-split.txt' + cifar_split = load_config( + '{:}/cifar-split.txt'.format(config_root), None, None) + # search over the proposed training and validation set + train_split, valid_split = cifar_split.train, cifar_split.valid + # logger.log('Load split file from {:}'.format(split_Fpath)) # they are two disjoint groups in the original CIFAR-10 training set + # To split data + xvalid_data = deepcopy(train_data) + if hasattr(xvalid_data, 'transforms'): # to avoid a print issue + xvalid_data.transforms = valid_data.transform + xvalid_data.transform = deepcopy(valid_data.transform) + search_data = SearchDataset( + dataset, train_data, train_split, valid_split) + # data loader + search_loader = torch.utils.data.DataLoader(search_data, batch_size=batch, shuffle=True, num_workers=workers, + pin_memory=True) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + train_split), + num_workers=workers, pin_memory=True) + valid_loader = torch.utils.data.DataLoader(xvalid_data, batch_size=test_batch, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + valid_split), + num_workers=workers, pin_memory=True) + elif dataset == 'cifar100': + cifar100_test_split = load_config( + '{:}/cifar100-test-split.txt'.format(config_root), None, None) + search_train_data = train_data + search_valid_data = deepcopy(valid_data) + search_valid_data.transform = train_data.transform + search_data = SearchDataset(dataset, [search_train_data, search_valid_data], + list(range(len(search_train_data))), + cifar100_test_split.xvalid) + search_loader = torch.utils.data.DataLoader(search_data, batch_size=batch, shuffle=True, num_workers=workers, + pin_memory=True) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch, shuffle=True, num_workers=workers, + pin_memory=True) + valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=test_batch, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + cifar100_test_split.xvalid), num_workers=workers, pin_memory=True) + elif dataset in ['mnist', 'svhn', 'aircraft', 'pets']: + if not os.path.exists('{:}/{}-test-split.txt'.format(config_root, dataset)): + import json + label_list = list(range(len(valid_data))) + random.shuffle(label_list) + strlist = [str(label_list[i]) for i in range(len(label_list))] + split = {'xvalid': ["int", strlist[:len(valid_data) // 2]], + 'xtest': ["int", strlist[len(valid_data) // 2:]]} + with open('{:}/{}-test-split.txt'.format(config_root, dataset), 'w') as f: + f.write(json.dumps(split)) + test_split = load_config( + '{:}/{}-test-split.txt'.format(config_root, dataset), None, None) + + search_train_data = train_data + search_valid_data = deepcopy(valid_data) + search_valid_data.transform = train_data.transform + search_data = SearchDataset(dataset, [search_train_data, search_valid_data], + list(range(len(search_train_data))), test_split.xvalid) + search_loader = torch.utils.data.DataLoader(search_data, batch_size=batch, shuffle=True, + num_workers=workers, pin_memory=True) + train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch, shuffle=True, + num_workers=workers, pin_memory=True) + valid_loader = torch.utils.data.DataLoader(valid_data, batch_size=test_batch, + sampler=torch.utils.data.sampler.SubsetRandomSampler( + test_split.xvalid), num_workers=workers, pin_memory=True) + else: + raise ValueError('invalid dataset : {:}'.format(dataset)) + return search_loader, train_loader, valid_loader diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/pets.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/pets.py new file mode 100644 index 0000000..899c793 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_datasets/pets.py @@ -0,0 +1,45 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +import torch +from glob import glob +from torch.utils.data.dataset import Dataset +import os +from PIL import Image + + +def load_image(filename): + img = Image.open(filename) + img = img.convert('RGB') + return img + +class PetDataset(Dataset): + def __init__(self, root, train=True, num_cl=37, val_split=0.2, transforms=None): + self.data = torch.load(os.path.join(root,'{}{}.pth'.format('train' if train else 'test', + int(100*(1-val_split)) if train else int(100*val_split)))) + self.len = len(self.data) + self.transform = transforms + def __getitem__(self, index): + img, label = self.data[index] + if self.transform: + img = self.transform(img) + return img, label + def __len__(self): + return self.len + +if __name__ == '__main__': + # Added + import torchvision.transforms as transforms + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + train_transform = transforms.Compose( + [transforms.Resize(256), transforms.RandomRotation(45), transforms.CenterCrop(224), + transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize]) + test_transform = transforms.Compose( + [transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize]) + root = '/w14/dataset/MetaGen/pets' + train_data, test_data = get_pets(root, num_cl=37, val_split=0.2, + tr_transform=train_transform, + te_transform=test_transform) + import pdb; + pdb.set_trace() diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/SharedUtils.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/SharedUtils.py new file mode 100644 index 0000000..8938752 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/SharedUtils.py @@ -0,0 +1,34 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch +import torch.nn as nn + + +def additive_func(A, B): + assert A.dim() == B.dim() and A.size(0) == B.size(0), '{:} vs {:}'.format(A.size(), B.size()) + C = min(A.size(1), B.size(1)) + if A.size(1) == B.size(1): + return A + B + elif A.size(1) < B.size(1): + out = B.clone() + out[:,:C] += A + return out + else: + out = A.clone() + out[:,:C] += B + return out + + +def change_key(key, value): + def func(m): + if hasattr(m, key): + setattr(m, key, value) + return func + + +def parse_channel_info(xstring): + blocks = xstring.split(' ') + blocks = [x.split('-') for x in blocks] + blocks = [[int(_) for _ in x] for x in blocks] + return blocks diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/__init__.py new file mode 100644 index 0000000..de56bc6 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/__init__.py @@ -0,0 +1,45 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from os import path as osp +from typing import List, Text +import torch + +__all__ = ['get_cell_based_tiny_net', 'get_search_spaces', \ + 'CellStructure', 'CellArchitectures' + ] + +# useful modules +from config_utils import dict2config +from .SharedUtils import change_key +from .cell_searchs import CellStructure, CellArchitectures + + +# Cell-based NAS Models +def get_cell_based_tiny_net(config): + if config.name == 'infer.tiny': + from .cell_infers import TinyNetwork + if hasattr(config, 'genotype'): + genotype = config.genotype + elif hasattr(config, 'arch_str'): + genotype = CellStructure.str2structure(config.arch_str) + else: raise ValueError('Can not find genotype from this config : {:}'.format(config)) + return TinyNetwork(config.C, config.N, genotype, config.num_classes) + else: + raise ValueError('invalid network name : {:}'.format(config.name)) + + +# obtain the search space, i.e., a dict mapping the operation name into a python-function for this op +def get_search_spaces(xtype, name) -> List[Text]: + if xtype == 'cell' or xtype == 'tss': # The topology search space. + from .cell_operations import SearchSpaceNames + assert name in SearchSpaceNames, 'invalid name [{:}] in {:}'.format(name, SearchSpaceNames.keys()) + return SearchSpaceNames[name] + elif xtype == 'sss': # The size search space. + if name == 'nas-bench-301': + return {'candidates': [8, 16, 24, 32, 40, 48, 56, 64], + 'numbers': 5} + else: + raise ValueError('Invalid name : {:}'.format(name)) + else: + raise ValueError('invalid search-space type is {:}'.format(xtype)) diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/__init__.py new file mode 100644 index 0000000..052b477 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/__init__.py @@ -0,0 +1,4 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +from .tiny_network import TinyNetwork diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/cells.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/cells.py new file mode 100644 index 0000000..7a279e9 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/cells.py @@ -0,0 +1,122 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### + +import torch +import torch.nn as nn +from copy import deepcopy +from ..cell_operations import OPS + + +# Cell for NAS-Bench-201 +class InferCell(nn.Module): + + def __init__(self, genotype, C_in, C_out, stride): + super(InferCell, self).__init__() + + self.layers = nn.ModuleList() + self.node_IN = [] + self.node_IX = [] + self.genotype = deepcopy(genotype) + for i in range(1, len(genotype)): + node_info = genotype[i-1] + cur_index = [] + cur_innod = [] + for (op_name, op_in) in node_info: + if op_in == 0: + layer = OPS[op_name](C_in , C_out, stride, True, True) + else: + layer = OPS[op_name](C_out, C_out, 1, True, True) + # import pdb; pdb.set_trace() + + cur_index.append( len(self.layers) ) + cur_innod.append( op_in ) + self.layers.append( layer ) + self.node_IX.append( cur_index ) + self.node_IN.append( cur_innod ) + self.nodes = len(genotype) + self.in_dim = C_in + self.out_dim = C_out + + def extra_repr(self): + string = 'info :: nodes={nodes}, inC={in_dim}, outC={out_dim}'.format(**self.__dict__) + laystr = [] + for i, (node_layers, node_innods) in enumerate(zip(self.node_IX,self.node_IN)): + y = ['I{:}-L{:}'.format(_ii, _il) for _il, _ii in zip(node_layers, node_innods)] + x = '{:}<-({:})'.format(i+1, ','.join(y)) + laystr.append( x ) + return string + ', [{:}]'.format( ' | '.join(laystr) ) + ', {:}'.format(self.genotype.tostr()) + + def forward(self, inputs): + nodes = [inputs] + for i, (node_layers, node_innods) in enumerate(zip(self.node_IX,self.node_IN)): + node_feature = sum( self.layers[_il](nodes[_ii]) for _il, _ii in zip(node_layers, node_innods) ) + nodes.append( node_feature ) + return nodes[-1] + + + +# Learning Transferable Architectures for Scalable Image Recognition, CVPR 2018 +class NASNetInferCell(nn.Module): + + def __init__(self, genotype, C_prev_prev, C_prev, C, reduction, reduction_prev, affine, track_running_stats): + super(NASNetInferCell, self).__init__() + self.reduction = reduction + if reduction_prev: self.preprocess0 = OPS['skip_connect'](C_prev_prev, C, 2, affine, track_running_stats) + else : self.preprocess0 = OPS['nor_conv_1x1'](C_prev_prev, C, 1, affine, track_running_stats) + self.preprocess1 = OPS['nor_conv_1x1'](C_prev, C, 1, affine, track_running_stats) + + if not reduction: + nodes, concats = genotype['normal'], genotype['normal_concat'] + else: + nodes, concats = genotype['reduce'], genotype['reduce_concat'] + self._multiplier = len(concats) + self._concats = concats + self._steps = len(nodes) + self._nodes = nodes + self.edges = nn.ModuleDict() + for i, node in enumerate(nodes): + for in_node in node: + name, j = in_node[0], in_node[1] + stride = 2 if reduction and j < 2 else 1 + node_str = '{:}<-{:}'.format(i+2, j) + self.edges[node_str] = OPS[name](C, C, stride, affine, track_running_stats) + + # [TODO] to support drop_prob in this function.. + def forward(self, s0, s1, unused_drop_prob): + s0 = self.preprocess0(s0) + s1 = self.preprocess1(s1) + + states = [s0, s1] + for i, node in enumerate(self._nodes): + clist = [] + for in_node in node: + name, j = in_node[0], in_node[1] + node_str = '{:}<-{:}'.format(i+2, j) + op = self.edges[ node_str ] + clist.append( op(states[j]) ) + states.append( sum(clist) ) + return torch.cat([states[x] for x in self._concats], dim=1) + + +class AuxiliaryHeadCIFAR(nn.Module): + + def __init__(self, C, num_classes): + """assuming input size 8x8""" + super(AuxiliaryHeadCIFAR, self).__init__() + self.features = nn.Sequential( + nn.ReLU(inplace=True), + nn.AvgPool2d(5, stride=3, padding=0, count_include_pad=False), # image size = 2 x 2 + nn.Conv2d(C, 128, 1, bias=False), + nn.BatchNorm2d(128), + nn.ReLU(inplace=True), + nn.Conv2d(128, 768, 2, bias=False), + nn.BatchNorm2d(768), + nn.ReLU(inplace=True) + ) + self.classifier = nn.Linear(768, num_classes) + + def forward(self, x): + x = self.features(x) + x = self.classifier(x.view(x.size(0),-1)) + return x diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/tiny_network.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/tiny_network.py new file mode 100644 index 0000000..d3c71db --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_infers/tiny_network.py @@ -0,0 +1,66 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +from ..cell_operations import ResNetBasicblock +from .cells import InferCell + + +# The macro structure for architectures in NAS-Bench-201 +class TinyNetwork(nn.Module): + + def __init__(self, C, N, genotype, num_classes): + super(TinyNetwork, self).__init__() + self._C = C + self._layerN = N + + self.stem = nn.Sequential( + nn.Conv2d(3, C, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(C)) + + layer_channels = [C ] * N + [C*2 ] + [C*2 ] * N + [C*4 ] + [C*4 ] * N + layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N + + C_prev = C + self.cells = nn.ModuleList() + for index, (C_curr, reduction) in enumerate(zip(layer_channels, layer_reductions)): + if reduction: + cell = ResNetBasicblock(C_prev, C_curr, 2, True) + else: + cell = InferCell(genotype, C_prev, C_curr, 1) + self.cells.append( cell ) + C_prev = cell.out_dim + self._Layer= len(self.cells) + + self.lastact = nn.Sequential(nn.BatchNorm2d(C_prev), nn.ReLU(inplace=True)) + self.global_pooling = nn.AdaptiveAvgPool2d(1) + self.classifier = nn.Linear(C_prev, num_classes) + + def get_message(self): + string = self.extra_repr() + for i, cell in enumerate(self.cells): + string += '\n {:02d}/{:02d} :: {:}'.format(i, len(self.cells), cell.extra_repr()) + return string + + def extra_repr(self): + return ('{name}(C={_C}, N={_layerN}, L={_Layer})'.format(name=self.__class__.__name__, **self.__dict__)) + + def forward(self, inputs): + feature = self.stem(inputs) + for i, cell in enumerate(self.cells): + feature = cell(feature) + ''' + out2 = self.lastact(feature) + out = self.global_pooling( out2 ) + out = out.view(out.size(0), -1) + out2 = out2.view(out2.size(0), -1) + logits = self.classifier(out) + return out2, logits + + ''' + out = self.lastact(feature) + out = self.global_pooling( out ) + out = out.view(out.size(0), -1) + logits = self.classifier(out) + + return out, logits diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_operations.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_operations.py new file mode 100644 index 0000000..c7528c1 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_operations.py @@ -0,0 +1,308 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +import torch +import torch.nn as nn + +__all__ = ['OPS', 'ResNetBasicblock', 'SearchSpaceNames'] + +OPS = { + 'none' : lambda C_in, C_out, stride, affine, track_running_stats: Zero(C_in, C_out, stride), + 'avg_pool_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: POOLING(C_in, C_out, stride, 'avg', affine, track_running_stats), + 'max_pool_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: POOLING(C_in, C_out, stride, 'max', affine, track_running_stats), + 'nor_conv_7x7' : lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(C_in, C_out, (7,7), (stride,stride), (3,3), (1,1), affine, track_running_stats), + 'nor_conv_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(C_in, C_out, (3,3), (stride,stride), (1,1), (1,1), affine, track_running_stats), + 'nor_conv_1x1' : lambda C_in, C_out, stride, affine, track_running_stats: ReLUConvBN(C_in, C_out, (1,1), (stride,stride), (0,0), (1,1), affine, track_running_stats), + 'dua_sepc_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: DualSepConv(C_in, C_out, (3,3), (stride,stride), (1,1), (1,1), affine, track_running_stats), + 'dua_sepc_5x5' : lambda C_in, C_out, stride, affine, track_running_stats: DualSepConv(C_in, C_out, (5,5), (stride,stride), (2,2), (1,1), affine, track_running_stats), + 'dil_sepc_3x3' : lambda C_in, C_out, stride, affine, track_running_stats: SepConv(C_in, C_out, (3,3), (stride,stride), (2,2), (2,2), affine, track_running_stats), + 'dil_sepc_5x5' : lambda C_in, C_out, stride, affine, track_running_stats: SepConv(C_in, C_out, (5,5), (stride,stride), (4,4), (2,2), affine, track_running_stats), + 'skip_connect' : lambda C_in, C_out, stride, affine, track_running_stats: Identity() if stride == 1 and C_in == C_out else FactorizedReduce(C_in, C_out, stride, affine, track_running_stats), +} + +CONNECT_NAS_BENCHMARK = ['none', 'skip_connect', 'nor_conv_3x3'] +NAS_BENCH_201 = ['none', 'skip_connect', 'nor_conv_1x1', 'nor_conv_3x3', 'avg_pool_3x3'] +DARTS_SPACE = ['none', 'skip_connect', 'dua_sepc_3x3', 'dua_sepc_5x5', 'dil_sepc_3x3', 'dil_sepc_5x5', 'avg_pool_3x3', 'max_pool_3x3'] + +SearchSpaceNames = {'connect-nas' : CONNECT_NAS_BENCHMARK, + 'nas-bench-201': NAS_BENCH_201, + 'nas-bench-301': NAS_BENCH_201, + 'darts' : DARTS_SPACE} + + +class ReLUConvBN(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine, track_running_stats=True): + super(ReLUConvBN, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_out, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=not affine), + nn.BatchNorm2d(C_out, affine=affine, track_running_stats=track_running_stats) + ) + + def forward(self, x): + return self.op(x) + + +class SepConv(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine, track_running_stats=True): + super(SepConv, self).__init__() + self.op = nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C_in, C_in, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, groups=C_in, bias=False), + nn.Conv2d(C_in, C_out, kernel_size=1, padding=0, bias=not affine), + nn.BatchNorm2d(C_out, affine=affine, track_running_stats=track_running_stats), + ) + + def forward(self, x): + return self.op(x) + + +class DualSepConv(nn.Module): + + def __init__(self, C_in, C_out, kernel_size, stride, padding, dilation, affine, track_running_stats=True): + super(DualSepConv, self).__init__() + self.op_a = SepConv(C_in, C_in , kernel_size, stride, padding, dilation, affine, track_running_stats) + self.op_b = SepConv(C_in, C_out, kernel_size, 1, padding, dilation, affine, track_running_stats) + + def forward(self, x): + x = self.op_a(x) + x = self.op_b(x) + return x + + +class ResNetBasicblock(nn.Module): + + def __init__(self, inplanes, planes, stride, affine=True): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + self.conv_a = ReLUConvBN(inplanes, planes, 3, stride, 1, 1, affine) + self.conv_b = ReLUConvBN( planes, planes, 3, 1, 1, 1, affine) + if stride == 2: + self.downsample = nn.Sequential( + nn.AvgPool2d(kernel_size=2, stride=2, padding=0), + nn.Conv2d(inplanes, planes, kernel_size=1, stride=1, padding=0, bias=False)) + elif inplanes != planes: + self.downsample = ReLUConvBN(inplanes, planes, 1, 1, 0, 1, affine) + else: + self.downsample = None + self.in_dim = inplanes + self.out_dim = planes + self.stride = stride + self.num_conv = 2 + + def extra_repr(self): + string = '{name}(inC={in_dim}, outC={out_dim}, stride={stride})'.format(name=self.__class__.__name__, **self.__dict__) + return string + + def forward(self, inputs): + + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + return residual + basicblock + + +class POOLING(nn.Module): + + def __init__(self, C_in, C_out, stride, mode, affine=True, track_running_stats=True): + super(POOLING, self).__init__() + if C_in == C_out: + self.preprocess = None + else: + self.preprocess = ReLUConvBN(C_in, C_out, 1, 1, 0, 1, affine, track_running_stats) + if mode == 'avg' : self.op = nn.AvgPool2d(3, stride=stride, padding=1, count_include_pad=False) + elif mode == 'max': self.op = nn.MaxPool2d(3, stride=stride, padding=1) + else : raise ValueError('Invalid mode={:} in POOLING'.format(mode)) + + def forward(self, inputs): + if self.preprocess: x = self.preprocess(inputs) + else : x = inputs + return self.op(x) + + +class Identity(nn.Module): + + def __init__(self): + super(Identity, self).__init__() + + def forward(self, x): + return x + + +class Zero(nn.Module): + + def __init__(self, C_in, C_out, stride): + super(Zero, self).__init__() + self.C_in = C_in + self.C_out = C_out + self.stride = stride + self.is_zero = True + + def forward(self, x): + if self.C_in == self.C_out: + if self.stride == 1: return x.mul(0.) + else : return x[:,:,::self.stride,::self.stride].mul(0.) + else: + shape = list(x.shape) + shape[1] = self.C_out + zeros = x.new_zeros(shape, dtype=x.dtype, device=x.device) + return zeros + + def extra_repr(self): + return 'C_in={C_in}, C_out={C_out}, stride={stride}'.format(**self.__dict__) + + +class FactorizedReduce(nn.Module): + + def __init__(self, C_in, C_out, stride, affine, track_running_stats): + super(FactorizedReduce, self).__init__() + self.stride = stride + self.C_in = C_in + self.C_out = C_out + self.relu = nn.ReLU(inplace=False) + if stride == 2: + #assert C_out % 2 == 0, 'C_out : {:}'.format(C_out) + C_outs = [C_out // 2, C_out - C_out // 2] + self.convs = nn.ModuleList() + for i in range(2): + self.convs.append(nn.Conv2d(C_in, C_outs[i], 1, stride=stride, padding=0, bias=not affine)) + self.pad = nn.ConstantPad2d((0, 1, 0, 1), 0) + elif stride == 1: + self.conv = nn.Conv2d(C_in, C_out, 1, stride=stride, padding=0, bias=False) + else: + raise ValueError('Invalid stride : {:}'.format(stride)) + self.bn = nn.BatchNorm2d(C_out, affine=affine, track_running_stats=track_running_stats) + + def forward(self, x): + if self.stride == 2: + x = self.relu(x) + y = self.pad(x) + out = torch.cat([self.convs[0](x), self.convs[1](y[:,:,1:,1:])], dim=1) + else: + out = self.conv(x) + out = self.bn(out) + return out + + def extra_repr(self): + return 'C_in={C_in}, C_out={C_out}, stride={stride}'.format(**self.__dict__) + + +# Auto-ReID: Searching for a Part-Aware ConvNet for Person Re-Identification, ICCV 2019 +class PartAwareOp(nn.Module): + + def __init__(self, C_in, C_out, stride, part=4): + super().__init__() + self.part = 4 + self.hidden = C_in // 3 + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.local_conv_list = nn.ModuleList() + for i in range(self.part): + self.local_conv_list.append( + nn.Sequential(nn.ReLU(), nn.Conv2d(C_in, self.hidden, 1), nn.BatchNorm2d(self.hidden, affine=True)) + ) + self.W_K = nn.Linear(self.hidden, self.hidden) + self.W_Q = nn.Linear(self.hidden, self.hidden) + + if stride == 2 : self.last = FactorizedReduce(C_in + self.hidden, C_out, 2) + elif stride == 1: self.last = FactorizedReduce(C_in + self.hidden, C_out, 1) + else: raise ValueError('Invalid Stride : {:}'.format(stride)) + + def forward(self, x): + batch, C, H, W = x.size() + assert H >= self.part, 'input size too small : {:} vs {:}'.format(x.shape, self.part) + IHs = [0] + for i in range(self.part): IHs.append( min(H, int((i+1)*(float(H)/self.part))) ) + local_feat_list = [] + for i in range(self.part): + feature = x[:, :, IHs[i]:IHs[i+1], :] + xfeax = self.avg_pool(feature) + xfea = self.local_conv_list[i]( xfeax ) + local_feat_list.append( xfea ) + part_feature = torch.cat(local_feat_list, dim=2).view(batch, -1, self.part) + part_feature = part_feature.transpose(1,2).contiguous() + part_K = self.W_K(part_feature) + part_Q = self.W_Q(part_feature).transpose(1,2).contiguous() + weight_att = torch.bmm(part_K, part_Q) + attention = torch.softmax(weight_att, dim=2) + aggreateF = torch.bmm(attention, part_feature).transpose(1,2).contiguous() + features = [] + for i in range(self.part): + feature = aggreateF[:, :, i:i+1].expand(batch, self.hidden, IHs[i+1]-IHs[i]) + feature = feature.view(batch, self.hidden, IHs[i+1]-IHs[i], 1) + features.append( feature ) + features = torch.cat(features, dim=2).expand(batch, self.hidden, H, W) + final_fea = torch.cat((x,features), dim=1) + outputs = self.last( final_fea ) + return outputs + + +def drop_path(x, drop_prob): + if drop_prob > 0.: + keep_prob = 1. - drop_prob + mask = x.new_zeros(x.size(0), 1, 1, 1) + mask = mask.bernoulli_(keep_prob) + x = torch.div(x, keep_prob) + x.mul_(mask) + return x + + +# Searching for A Robust Neural Architecture in Four GPU Hours +class GDAS_Reduction_Cell(nn.Module): + + def __init__(self, C_prev_prev, C_prev, C, reduction_prev, multiplier, affine, track_running_stats): + super(GDAS_Reduction_Cell, self).__init__() + if reduction_prev: + self.preprocess0 = FactorizedReduce(C_prev_prev, C, 2, affine, track_running_stats) + else: + self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0, 1, affine, track_running_stats) + self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0, 1, affine, track_running_stats) + self.multiplier = multiplier + + self.reduction = True + self.ops1 = nn.ModuleList( + [nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C, C, (1, 3), stride=(1, 2), padding=(0, 1), groups=8, bias=False), + nn.Conv2d(C, C, (3, 1), stride=(2, 1), padding=(1, 0), groups=8, bias=False), + nn.BatchNorm2d(C, affine=True), + nn.ReLU(inplace=False), + nn.Conv2d(C, C, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(C, affine=True)), + nn.Sequential( + nn.ReLU(inplace=False), + nn.Conv2d(C, C, (1, 3), stride=(1, 2), padding=(0, 1), groups=8, bias=False), + nn.Conv2d(C, C, (3, 1), stride=(2, 1), padding=(1, 0), groups=8, bias=False), + nn.BatchNorm2d(C, affine=True), + nn.ReLU(inplace=False), + nn.Conv2d(C, C, 1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(C, affine=True))]) + + self.ops2 = nn.ModuleList( + [nn.Sequential( + nn.MaxPool2d(3, stride=1, padding=1), + nn.BatchNorm2d(C, affine=True)), + nn.Sequential( + nn.MaxPool2d(3, stride=2, padding=1), + nn.BatchNorm2d(C, affine=True))]) + + def forward(self, s0, s1, drop_prob = -1): + s0 = self.preprocess0(s0) + s1 = self.preprocess1(s1) + + X0 = self.ops1[0] (s0) + X1 = self.ops1[1] (s1) + if self.training and drop_prob > 0.: + X0, X1 = drop_path(X0, drop_prob), drop_path(X1, drop_prob) + + #X2 = self.ops2[0] (X0+X1) + X2 = self.ops2[0] (s0) + X3 = self.ops2[1] (s1) + if self.training and drop_prob > 0.: + X2, X3 = drop_path(X2, drop_prob), drop_path(X3, drop_prob) + return torch.cat([X0, X1, X2, X3], dim=1) diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_searchs/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_searchs/__init__.py new file mode 100644 index 0000000..df26f92 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_searchs/__init__.py @@ -0,0 +1,26 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +# The macro structure is defined in NAS-Bench-201 +# from .search_model_darts import TinyNetworkDarts +# from .search_model_gdas import TinyNetworkGDAS +# from .search_model_setn import TinyNetworkSETN +# from .search_model_enas import TinyNetworkENAS +# from .search_model_random import TinyNetworkRANDOM +# from .generic_model import GenericNAS201Model +from .genotypes import Structure as CellStructure, architectures as CellArchitectures +# NASNet-based macro structure +# from .search_model_gdas_nasnet import NASNetworkGDAS +# from .search_model_darts_nasnet import NASNetworkDARTS + + +# nas201_super_nets = {'DARTS-V1': TinyNetworkDarts, +# "DARTS-V2": TinyNetworkDarts, +# "GDAS": TinyNetworkGDAS, +# "SETN": TinyNetworkSETN, +# "ENAS": TinyNetworkENAS, +# "RANDOM": TinyNetworkRANDOM, +# "generic": GenericNAS201Model} + +# nasnet_super_nets = {"GDAS": NASNetworkGDAS, +# "DARTS": NASNetworkDARTS} diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_searchs/genotypes.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_searchs/genotypes.py new file mode 100644 index 0000000..b2b4091 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/cell_searchs/genotypes.py @@ -0,0 +1,198 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from copy import deepcopy + + +def get_combination(space, num): + combs = [] + for i in range(num): + if i == 0: + for func in space: + combs.append( [(func, i)] ) + else: + new_combs = [] + for string in combs: + for func in space: + xstring = string + [(func, i)] + new_combs.append( xstring ) + combs = new_combs + return combs + + +class Structure: + + def __init__(self, genotype): + assert isinstance(genotype, list) or isinstance(genotype, tuple), 'invalid class of genotype : {:}'.format(type(genotype)) + self.node_num = len(genotype) + 1 + self.nodes = [] + self.node_N = [] + for idx, node_info in enumerate(genotype): + assert isinstance(node_info, list) or isinstance(node_info, tuple), 'invalid class of node_info : {:}'.format(type(node_info)) + assert len(node_info) >= 1, 'invalid length : {:}'.format(len(node_info)) + for node_in in node_info: + assert isinstance(node_in, list) or isinstance(node_in, tuple), 'invalid class of in-node : {:}'.format(type(node_in)) + assert len(node_in) == 2 and node_in[1] <= idx, 'invalid in-node : {:}'.format(node_in) + self.node_N.append( len(node_info) ) + self.nodes.append( tuple(deepcopy(node_info)) ) + + def tolist(self, remove_str): + # convert this class to the list, if remove_str is 'none', then remove the 'none' operation. + # note that we re-order the input node in this function + # return the-genotype-list and success [if unsuccess, it is not a connectivity] + genotypes = [] + for node_info in self.nodes: + node_info = list( node_info ) + node_info = sorted(node_info, key=lambda x: (x[1], x[0])) + node_info = tuple(filter(lambda x: x[0] != remove_str, node_info)) + if len(node_info) == 0: return None, False + genotypes.append( node_info ) + return genotypes, True + + def node(self, index): + assert index > 0 and index <= len(self), 'invalid index={:} < {:}'.format(index, len(self)) + return self.nodes[index] + + def tostr(self): + strings = [] + for node_info in self.nodes: + string = '|'.join([x[0]+'~{:}'.format(x[1]) for x in node_info]) + string = '|{:}|'.format(string) + strings.append( string ) + return '+'.join(strings) + + def check_valid(self): + nodes = {0: True} + for i, node_info in enumerate(self.nodes): + sums = [] + for op, xin in node_info: + if op == 'none' or nodes[xin] is False: x = False + else: x = True + sums.append( x ) + nodes[i+1] = sum(sums) > 0 + return nodes[len(self.nodes)] + + def to_unique_str(self, consider_zero=False): + # this is used to identify the isomorphic cell, which rerquires the prior knowledge of operation + # two operations are special, i.e., none and skip_connect + nodes = {0: '0'} + for i_node, node_info in enumerate(self.nodes): + cur_node = [] + for op, xin in node_info: + if consider_zero is None: + x = '('+nodes[xin]+')' + '@{:}'.format(op) + elif consider_zero: + if op == 'none' or nodes[xin] == '#': x = '#' # zero + elif op == 'skip_connect': x = nodes[xin] + else: x = '('+nodes[xin]+')' + '@{:}'.format(op) + else: + if op == 'skip_connect': x = nodes[xin] + else: x = '('+nodes[xin]+')' + '@{:}'.format(op) + cur_node.append(x) + nodes[i_node+1] = '+'.join( sorted(cur_node) ) + return nodes[ len(self.nodes) ] + + def check_valid_op(self, op_names): + for node_info in self.nodes: + for inode_edge in node_info: + #assert inode_edge[0] in op_names, 'invalid op-name : {:}'.format(inode_edge[0]) + if inode_edge[0] not in op_names: return False + return True + + def __repr__(self): + return ('{name}({node_num} nodes with {node_info})'.format(name=self.__class__.__name__, node_info=self.tostr(), **self.__dict__)) + + def __len__(self): + return len(self.nodes) + 1 + + def __getitem__(self, index): + return self.nodes[index] + + @staticmethod + def str2structure(xstr): + if isinstance(xstr, Structure): return xstr + assert isinstance(xstr, str), 'must take string (not {:}) as input'.format(type(xstr)) + nodestrs = xstr.split('+') + genotypes = [] + for i, node_str in enumerate(nodestrs): + inputs = list(filter(lambda x: x != '', node_str.split('|'))) + for xinput in inputs: assert len(xinput.split('~')) == 2, 'invalid input length : {:}'.format(xinput) + inputs = ( xi.split('~') for xi in inputs ) + input_infos = tuple( (op, int(IDX)) for (op, IDX) in inputs) + genotypes.append( input_infos ) + return Structure( genotypes ) + + @staticmethod + def str2fullstructure(xstr, default_name='none'): + assert isinstance(xstr, str), 'must take string (not {:}) as input'.format(type(xstr)) + nodestrs = xstr.split('+') + genotypes = [] + for i, node_str in enumerate(nodestrs): + inputs = list(filter(lambda x: x != '', node_str.split('|'))) + for xinput in inputs: assert len(xinput.split('~')) == 2, 'invalid input length : {:}'.format(xinput) + inputs = ( xi.split('~') for xi in inputs ) + input_infos = list( (op, int(IDX)) for (op, IDX) in inputs) + all_in_nodes= list(x[1] for x in input_infos) + for j in range(i): + if j not in all_in_nodes: input_infos.append((default_name, j)) + node_info = sorted(input_infos, key=lambda x: (x[1], x[0])) + genotypes.append( tuple(node_info) ) + return Structure( genotypes ) + + @staticmethod + def gen_all(search_space, num, return_ori): + assert isinstance(search_space, list) or isinstance(search_space, tuple), 'invalid class of search-space : {:}'.format(type(search_space)) + assert num >= 2, 'There should be at least two nodes in a neural cell instead of {:}'.format(num) + all_archs = get_combination(search_space, 1) + for i, arch in enumerate(all_archs): + all_archs[i] = [ tuple(arch) ] + + for inode in range(2, num): + cur_nodes = get_combination(search_space, inode) + new_all_archs = [] + for previous_arch in all_archs: + for cur_node in cur_nodes: + new_all_archs.append( previous_arch + [tuple(cur_node)] ) + all_archs = new_all_archs + if return_ori: + return all_archs + else: + return [Structure(x) for x in all_archs] + + + +ResNet_CODE = Structure( + [(('nor_conv_3x3', 0), ), # node-1 + (('nor_conv_3x3', 1), ), # node-2 + (('skip_connect', 0), ('skip_connect', 2))] # node-3 + ) + +AllConv3x3_CODE = Structure( + [(('nor_conv_3x3', 0), ), # node-1 + (('nor_conv_3x3', 0), ('nor_conv_3x3', 1)), # node-2 + (('nor_conv_3x3', 0), ('nor_conv_3x3', 1), ('nor_conv_3x3', 2))] # node-3 + ) + +AllFull_CODE = Structure( + [(('skip_connect', 0), ('nor_conv_1x1', 0), ('nor_conv_3x3', 0), ('avg_pool_3x3', 0)), # node-1 + (('skip_connect', 0), ('nor_conv_1x1', 0), ('nor_conv_3x3', 0), ('avg_pool_3x3', 0), ('skip_connect', 1), ('nor_conv_1x1', 1), ('nor_conv_3x3', 1), ('avg_pool_3x3', 1)), # node-2 + (('skip_connect', 0), ('nor_conv_1x1', 0), ('nor_conv_3x3', 0), ('avg_pool_3x3', 0), ('skip_connect', 1), ('nor_conv_1x1', 1), ('nor_conv_3x3', 1), ('avg_pool_3x3', 1), ('skip_connect', 2), ('nor_conv_1x1', 2), ('nor_conv_3x3', 2), ('avg_pool_3x3', 2))] # node-3 + ) + +AllConv1x1_CODE = Structure( + [(('nor_conv_1x1', 0), ), # node-1 + (('nor_conv_1x1', 0), ('nor_conv_1x1', 1)), # node-2 + (('nor_conv_1x1', 0), ('nor_conv_1x1', 1), ('nor_conv_1x1', 2))] # node-3 + ) + +AllIdentity_CODE = Structure( + [(('skip_connect', 0), ), # node-1 + (('skip_connect', 0), ('skip_connect', 1)), # node-2 + (('skip_connect', 0), ('skip_connect', 1), ('skip_connect', 2))] # node-3 + ) + +architectures = {'resnet' : ResNet_CODE, + 'all_c3x3': AllConv3x3_CODE, + 'all_c1x1': AllConv1x1_CODE, + 'all_idnt': AllIdentity_CODE, + 'all_full': AllFull_CODE} diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet.py new file mode 100644 index 0000000..a6524d6 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet.py @@ -0,0 +1,167 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +import torch.nn.functional as F +from ..initialization import initialize_resnet + + +class ConvBNReLU(nn.Module): + + def __init__(self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu): + super(ConvBNReLU, self).__init__() + if has_avg : self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) + else : self.avg = None + self.conv = nn.Conv2d(nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, dilation=1, groups=1, bias=bias) + if has_bn : self.bn = nn.BatchNorm2d(nOut) + else : self.bn = None + if has_relu: self.relu = nn.ReLU(inplace=True) + else : self.relu = None + + def forward(self, inputs): + if self.avg : out = self.avg( inputs ) + else : out = inputs + conv = self.conv( out ) + if self.bn : out = self.bn( conv ) + else : out = conv + if self.relu: out = self.relu( out ) + else : out = out + + return out + + +class ResNetBasicblock(nn.Module): + num_conv = 2 + expansion = 1 + def __init__(self, iCs, stride): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 3,'invalid lengths of iCs : {:}'.format(iCs) + + self.conv_a = ConvBNReLU(iCs[0], iCs[1], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_b = ConvBNReLU(iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=True, has_bn=False, has_relu=False) + residual_in = iCs[2] + elif iCs[0] != iCs[2]: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=False,has_bn=True , has_relu=False) + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[2]) + self.out_dim = iCs[2] + + def forward(self, inputs): + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + basicblock + return F.relu(out, inplace=True) + + + +class ResNetBottleneck(nn.Module): + expansion = 4 + num_conv = 3 + def __init__(self, iCs, stride): + super(ResNetBottleneck, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 4,'invalid lengths of iCs : {:}'.format(iCs) + self.conv_1x1 = ConvBNReLU(iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_3x3 = ConvBNReLU(iCs[1], iCs[2], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_1x4 = ConvBNReLU(iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=True , has_bn=False, has_relu=False) + residual_in = iCs[3] + elif iCs[0] != iCs[3]: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=False, has_relu=False) + residual_in = iCs[3] + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[3]) + self.out_dim = iCs[3] + + def forward(self, inputs): + + bottleneck = self.conv_1x1(inputs) + bottleneck = self.conv_3x3(bottleneck) + bottleneck = self.conv_1x4(bottleneck) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + bottleneck + return F.relu(out, inplace=True) + + + +class InferCifarResNet(nn.Module): + + def __init__(self, block_name, depth, xblocks, xchannels, num_classes, zero_init_residual): + super(InferCifarResNet, self).__init__() + + #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + if block_name == 'ResNetBasicblock': + block = ResNetBasicblock + assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' + layer_blocks = (depth - 2) // 6 + elif block_name == 'ResNetBottleneck': + block = ResNetBottleneck + assert (depth - 2) % 9 == 0, 'depth should be one of 164' + layer_blocks = (depth - 2) // 9 + else: + raise ValueError('invalid block : {:}'.format(block_name)) + assert len(xblocks) == 3, 'invalid xblocks : {:}'.format(xblocks) + + self.message = 'InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}'.format(depth, layer_blocks) + self.num_classes = num_classes + self.xchannels = xchannels + self.layers = nn.ModuleList( [ ConvBNReLU(xchannels[0], xchannels[1], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True) ] ) + last_channel_idx = 1 + for stage in range(3): + for iL in range(layer_blocks): + num_conv = block.num_conv + iCs = self.xchannels[last_channel_idx:last_channel_idx+num_conv+1] + stride = 2 if stage > 0 and iL == 0 else 1 + module = block(iCs, stride) + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + self.layers.append ( module ) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(stage, iL, layer_blocks, len(self.layers)-1, iCs, module.out_dim, stride) + if iL + 1 == xblocks[stage]: # reach the maximum depth + out_channel = module.out_dim + for iiL in range(iL+1, layer_blocks): + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + break + + self.avgpool = nn.AvgPool2d(8) + self.classifier = nn.Linear(self.xchannels[-1], num_classes) + + self.apply(initialize_resnet) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, ResNetBasicblock): + nn.init.constant_(m.conv_b.bn.weight, 0) + elif isinstance(m, ResNetBottleneck): + nn.init.constant_(m.conv_1x4.bn.weight, 0) + + def get_message(self): + return self.message + + def forward(self, inputs): + x = inputs + for i, layer in enumerate(self.layers): + x = layer( x ) + features = self.avgpool(x) + features = features.view(features.size(0), -1) + logits = self.classifier(features) + return features, logits diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_depth.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_depth.py new file mode 100644 index 0000000..d773fc5 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_depth.py @@ -0,0 +1,150 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +import torch.nn.functional as F +from ..initialization import initialize_resnet + + +class ConvBNReLU(nn.Module): + + def __init__(self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu): + super(ConvBNReLU, self).__init__() + if has_avg : self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) + else : self.avg = None + self.conv = nn.Conv2d(nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, dilation=1, groups=1, bias=bias) + if has_bn : self.bn = nn.BatchNorm2d(nOut) + else : self.bn = None + if has_relu: self.relu = nn.ReLU(inplace=True) + else : self.relu = None + + def forward(self, inputs): + if self.avg : out = self.avg( inputs ) + else : out = inputs + conv = self.conv( out ) + if self.bn : out = self.bn( conv ) + else : out = conv + if self.relu: out = self.relu( out ) + else : out = out + + return out + + +class ResNetBasicblock(nn.Module): + num_conv = 2 + expansion = 1 + def __init__(self, inplanes, planes, stride): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + + self.conv_a = ConvBNReLU(inplanes, planes, 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_b = ConvBNReLU( planes, planes, 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False) + if stride == 2: + self.downsample = ConvBNReLU(inplanes, planes, 1, 1, 0, False, has_avg=True, has_bn=False, has_relu=False) + elif inplanes != planes: + self.downsample = ConvBNReLU(inplanes, planes, 1, 1, 0, False, has_avg=False,has_bn=True , has_relu=False) + else: + self.downsample = None + self.out_dim = planes + + def forward(self, inputs): + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + basicblock + return F.relu(out, inplace=True) + + + +class ResNetBottleneck(nn.Module): + expansion = 4 + num_conv = 3 + def __init__(self, inplanes, planes, stride): + super(ResNetBottleneck, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + self.conv_1x1 = ConvBNReLU(inplanes, planes, 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_3x3 = ConvBNReLU( planes, planes, 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_1x4 = ConvBNReLU(planes, planes*self.expansion, 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + if stride == 2: + self.downsample = ConvBNReLU(inplanes, planes*self.expansion, 1, 1, 0, False, has_avg=True , has_bn=False, has_relu=False) + elif inplanes != planes*self.expansion: + self.downsample = ConvBNReLU(inplanes, planes*self.expansion, 1, 1, 0, False, has_avg=False, has_bn=False, has_relu=False) + else: + self.downsample = None + self.out_dim = planes*self.expansion + + def forward(self, inputs): + + bottleneck = self.conv_1x1(inputs) + bottleneck = self.conv_3x3(bottleneck) + bottleneck = self.conv_1x4(bottleneck) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + bottleneck + return F.relu(out, inplace=True) + + + +class InferDepthCifarResNet(nn.Module): + + def __init__(self, block_name, depth, xblocks, num_classes, zero_init_residual): + super(InferDepthCifarResNet, self).__init__() + + #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + if block_name == 'ResNetBasicblock': + block = ResNetBasicblock + assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' + layer_blocks = (depth - 2) // 6 + elif block_name == 'ResNetBottleneck': + block = ResNetBottleneck + assert (depth - 2) % 9 == 0, 'depth should be one of 164' + layer_blocks = (depth - 2) // 9 + else: + raise ValueError('invalid block : {:}'.format(block_name)) + assert len(xblocks) == 3, 'invalid xblocks : {:}'.format(xblocks) + + self.message = 'InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}'.format(depth, layer_blocks) + self.num_classes = num_classes + self.layers = nn.ModuleList( [ ConvBNReLU(3, 16, 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True) ] ) + self.channels = [16] + for stage in range(3): + for iL in range(layer_blocks): + iC = self.channels[-1] + planes = 16 * (2**stage) + stride = 2 if stage > 0 and iL == 0 else 1 + module = block(iC, planes, stride) + self.channels.append( module.out_dim ) + self.layers.append ( module ) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iC={:}, oC={:3d}, stride={:}".format(stage, iL, layer_blocks, len(self.layers)-1, planes, module.out_dim, stride) + if iL + 1 == xblocks[stage]: # reach the maximum depth + break + + self.avgpool = nn.AvgPool2d(8) + self.classifier = nn.Linear(self.channels[-1], num_classes) + + self.apply(initialize_resnet) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, ResNetBasicblock): + nn.init.constant_(m.conv_b.bn.weight, 0) + elif isinstance(m, ResNetBottleneck): + nn.init.constant_(m.conv_1x4.bn.weight, 0) + + def get_message(self): + return self.message + + def forward(self, inputs): + x = inputs + for i, layer in enumerate(self.layers): + x = layer( x ) + features = self.avgpool(x) + features = features.view(features.size(0), -1) + logits = self.classifier(features) + return features, logits diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_width.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_width.py new file mode 100644 index 0000000..7183875 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferCifarResNet_width.py @@ -0,0 +1,160 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +import torch.nn.functional as F +from ..initialization import initialize_resnet + + +class ConvBNReLU(nn.Module): + + def __init__(self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu): + super(ConvBNReLU, self).__init__() + if has_avg : self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) + else : self.avg = None + self.conv = nn.Conv2d(nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, dilation=1, groups=1, bias=bias) + if has_bn : self.bn = nn.BatchNorm2d(nOut) + else : self.bn = None + if has_relu: self.relu = nn.ReLU(inplace=True) + else : self.relu = None + + def forward(self, inputs): + if self.avg : out = self.avg( inputs ) + else : out = inputs + conv = self.conv( out ) + if self.bn : out = self.bn( conv ) + else : out = conv + if self.relu: out = self.relu( out ) + else : out = out + + return out + + +class ResNetBasicblock(nn.Module): + num_conv = 2 + expansion = 1 + def __init__(self, iCs, stride): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 3,'invalid lengths of iCs : {:}'.format(iCs) + + self.conv_a = ConvBNReLU(iCs[0], iCs[1], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_b = ConvBNReLU(iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=True, has_bn=False, has_relu=False) + residual_in = iCs[2] + elif iCs[0] != iCs[2]: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=False,has_bn=True , has_relu=False) + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[2]) + self.out_dim = iCs[2] + + def forward(self, inputs): + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + basicblock + return F.relu(out, inplace=True) + + + +class ResNetBottleneck(nn.Module): + expansion = 4 + num_conv = 3 + def __init__(self, iCs, stride): + super(ResNetBottleneck, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 4,'invalid lengths of iCs : {:}'.format(iCs) + self.conv_1x1 = ConvBNReLU(iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_3x3 = ConvBNReLU(iCs[1], iCs[2], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_1x4 = ConvBNReLU(iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=True , has_bn=False, has_relu=False) + residual_in = iCs[3] + elif iCs[0] != iCs[3]: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=False, has_relu=False) + residual_in = iCs[3] + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[3]) + self.out_dim = iCs[3] + + def forward(self, inputs): + + bottleneck = self.conv_1x1(inputs) + bottleneck = self.conv_3x3(bottleneck) + bottleneck = self.conv_1x4(bottleneck) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + bottleneck + return F.relu(out, inplace=True) + + + +class InferWidthCifarResNet(nn.Module): + + def __init__(self, block_name, depth, xchannels, num_classes, zero_init_residual): + super(InferWidthCifarResNet, self).__init__() + + #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + if block_name == 'ResNetBasicblock': + block = ResNetBasicblock + assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' + layer_blocks = (depth - 2) // 6 + elif block_name == 'ResNetBottleneck': + block = ResNetBottleneck + assert (depth - 2) % 9 == 0, 'depth should be one of 164' + layer_blocks = (depth - 2) // 9 + else: + raise ValueError('invalid block : {:}'.format(block_name)) + + self.message = 'InferWidthCifarResNet : Depth : {:} , Layers for each block : {:}'.format(depth, layer_blocks) + self.num_classes = num_classes + self.xchannels = xchannels + self.layers = nn.ModuleList( [ ConvBNReLU(xchannels[0], xchannels[1], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True) ] ) + last_channel_idx = 1 + for stage in range(3): + for iL in range(layer_blocks): + num_conv = block.num_conv + iCs = self.xchannels[last_channel_idx:last_channel_idx+num_conv+1] + stride = 2 if stage > 0 and iL == 0 else 1 + module = block(iCs, stride) + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + self.layers.append ( module ) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(stage, iL, layer_blocks, len(self.layers)-1, iCs, module.out_dim, stride) + + self.avgpool = nn.AvgPool2d(8) + self.classifier = nn.Linear(self.xchannels[-1], num_classes) + + self.apply(initialize_resnet) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, ResNetBasicblock): + nn.init.constant_(m.conv_b.bn.weight, 0) + elif isinstance(m, ResNetBottleneck): + nn.init.constant_(m.conv_1x4.bn.weight, 0) + + def get_message(self): + return self.message + + def forward(self, inputs): + x = inputs + for i, layer in enumerate(self.layers): + x = layer( x ) + features = self.avgpool(x) + features = features.view(features.size(0), -1) + logits = self.classifier(features) + return features, logits diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferImagenetResNet.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferImagenetResNet.py new file mode 100644 index 0000000..8f06db7 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferImagenetResNet.py @@ -0,0 +1,170 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import torch.nn as nn +import torch.nn.functional as F +from ..initialization import initialize_resnet + + +class ConvBNReLU(nn.Module): + + num_conv = 1 + def __init__(self, nIn, nOut, kernel, stride, padding, bias, has_avg, has_bn, has_relu): + super(ConvBNReLU, self).__init__() + if has_avg : self.avg = nn.AvgPool2d(kernel_size=2, stride=2, padding=0) + else : self.avg = None + self.conv = nn.Conv2d(nIn, nOut, kernel_size=kernel, stride=stride, padding=padding, dilation=1, groups=1, bias=bias) + if has_bn : self.bn = nn.BatchNorm2d(nOut) + else : self.bn = None + if has_relu: self.relu = nn.ReLU(inplace=True) + else : self.relu = None + + def forward(self, inputs): + if self.avg : out = self.avg( inputs ) + else : out = inputs + conv = self.conv( out ) + if self.bn : out = self.bn( conv ) + else : out = conv + if self.relu: out = self.relu( out ) + else : out = out + + return out + + +class ResNetBasicblock(nn.Module): + num_conv = 2 + expansion = 1 + def __init__(self, iCs, stride): + super(ResNetBasicblock, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 3,'invalid lengths of iCs : {:}'.format(iCs) + + self.conv_a = ConvBNReLU(iCs[0], iCs[1], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_b = ConvBNReLU(iCs[1], iCs[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=True, has_bn=True, has_relu=False) + residual_in = iCs[2] + elif iCs[0] != iCs[2]: + self.downsample = ConvBNReLU(iCs[0], iCs[2], 1, 1, 0, False, has_avg=False,has_bn=True , has_relu=False) + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[2]) + self.out_dim = iCs[2] + + def forward(self, inputs): + basicblock = self.conv_a(inputs) + basicblock = self.conv_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + basicblock + return F.relu(out, inplace=True) + + + +class ResNetBottleneck(nn.Module): + expansion = 4 + num_conv = 3 + def __init__(self, iCs, stride): + super(ResNetBottleneck, self).__init__() + assert stride == 1 or stride == 2, 'invalid stride {:}'.format(stride) + assert isinstance(iCs, tuple) or isinstance(iCs, list), 'invalid type of iCs : {:}'.format( iCs ) + assert len(iCs) == 4,'invalid lengths of iCs : {:}'.format(iCs) + self.conv_1x1 = ConvBNReLU(iCs[0], iCs[1], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_3x3 = ConvBNReLU(iCs[1], iCs[2], 3, stride, 1, False, has_avg=False, has_bn=True, has_relu=True) + self.conv_1x4 = ConvBNReLU(iCs[2], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[0] + if stride == 2: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=True , has_bn=True, has_relu=False) + residual_in = iCs[3] + elif iCs[0] != iCs[3]: + self.downsample = ConvBNReLU(iCs[0], iCs[3], 1, 1, 0, False, has_avg=False, has_bn=True, has_relu=False) + residual_in = iCs[3] + else: + self.downsample = None + #self.out_dim = max(residual_in, iCs[3]) + self.out_dim = iCs[3] + + def forward(self, inputs): + + bottleneck = self.conv_1x1(inputs) + bottleneck = self.conv_3x3(bottleneck) + bottleneck = self.conv_1x4(bottleneck) + + if self.downsample is not None: + residual = self.downsample(inputs) + else: + residual = inputs + out = residual + bottleneck + return F.relu(out, inplace=True) + + + +class InferImagenetResNet(nn.Module): + + def __init__(self, block_name, layers, xblocks, xchannels, deep_stem, num_classes, zero_init_residual): + super(InferImagenetResNet, self).__init__() + + #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + if block_name == 'BasicBlock': + block = ResNetBasicblock + elif block_name == 'Bottleneck': + block = ResNetBottleneck + else: + raise ValueError('invalid block : {:}'.format(block_name)) + assert len(xblocks) == len(layers), 'invalid layers : {:} vs xblocks : {:}'.format(layers, xblocks) + + self.message = 'InferImagenetResNet : Depth : {:} -> {:}, Layers for each block : {:}'.format(sum(layers)*block.num_conv, sum(xblocks)*block.num_conv, xblocks) + self.num_classes = num_classes + self.xchannels = xchannels + if not deep_stem: + self.layers = nn.ModuleList( [ ConvBNReLU(xchannels[0], xchannels[1], 7, 2, 3, False, has_avg=False, has_bn=True, has_relu=True) ] ) + last_channel_idx = 1 + else: + self.layers = nn.ModuleList( [ ConvBNReLU(xchannels[0], xchannels[1], 3, 2, 1, False, has_avg=False, has_bn=True, has_relu=True) + ,ConvBNReLU(xchannels[1], xchannels[2], 3, 1, 1, False, has_avg=False, has_bn=True, has_relu=True) ] ) + last_channel_idx = 2 + self.layers.append( nn.MaxPool2d(kernel_size=3, stride=2, padding=1) ) + for stage, layer_blocks in enumerate(layers): + for iL in range(layer_blocks): + num_conv = block.num_conv + iCs = self.xchannels[last_channel_idx:last_channel_idx+num_conv+1] + stride = 2 if stage > 0 and iL == 0 else 1 + module = block(iCs, stride) + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + self.layers.append ( module ) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, iCs={:}, oC={:3d}, stride={:}".format(stage, iL, layer_blocks, len(self.layers)-1, iCs, module.out_dim, stride) + if iL + 1 == xblocks[stage]: # reach the maximum depth + out_channel = module.out_dim + for iiL in range(iL+1, layer_blocks): + last_channel_idx += num_conv + self.xchannels[last_channel_idx] = module.out_dim + break + assert last_channel_idx + 1 == len(self.xchannels), '{:} vs {:}'.format(last_channel_idx, len(self.xchannels)) + self.avgpool = nn.AdaptiveAvgPool2d((1,1)) + self.classifier = nn.Linear(self.xchannels[-1], num_classes) + + self.apply(initialize_resnet) + if zero_init_residual: + for m in self.modules(): + if isinstance(m, ResNetBasicblock): + nn.init.constant_(m.conv_b.bn.weight, 0) + elif isinstance(m, ResNetBottleneck): + nn.init.constant_(m.conv_1x4.bn.weight, 0) + + def get_message(self): + return self.message + + def forward(self, inputs): + x = inputs + for i, layer in enumerate(self.layers): + x = layer( x ) + features = self.avgpool(x) + features = features.view(features.size(0), -1) + logits = self.classifier(features) + return features, logits diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferMobileNetV2.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferMobileNetV2.py new file mode 100644 index 0000000..d072b99 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferMobileNetV2.py @@ -0,0 +1,122 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +# MobileNetV2: Inverted Residuals and Linear Bottlenecks, CVPR 2018 +from torch import nn +from ..initialization import initialize_resnet +from ..SharedUtils import parse_channel_info + + +class ConvBNReLU(nn.Module): + def __init__(self, in_planes, out_planes, kernel_size, stride, groups, has_bn=True, has_relu=True): + super(ConvBNReLU, self).__init__() + padding = (kernel_size - 1) // 2 + self.conv = nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False) + if has_bn: self.bn = nn.BatchNorm2d(out_planes) + else : self.bn = None + if has_relu: self.relu = nn.ReLU6(inplace=True) + else : self.relu = None + + def forward(self, x): + out = self.conv( x ) + if self.bn: out = self.bn ( out ) + if self.relu: out = self.relu( out ) + return out + + +class InvertedResidual(nn.Module): + def __init__(self, channels, stride, expand_ratio, additive): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2], 'invalid stride : {:}'.format(stride) + assert len(channels) in [2, 3], 'invalid channels : {:}'.format(channels) + + if len(channels) == 2: + layers = [] + else: + layers = [ConvBNReLU(channels[0], channels[1], 1, 1, 1)] + layers.extend([ + # dw + ConvBNReLU(channels[-2], channels[-2], 3, stride, channels[-2]), + # pw-linear + ConvBNReLU(channels[-2], channels[-1], 1, 1, 1, True, False), + ]) + self.conv = nn.Sequential(*layers) + self.additive = additive + if self.additive and channels[0] != channels[-1]: + self.shortcut = ConvBNReLU(channels[0], channels[-1], 1, 1, 1, True, False) + else: + self.shortcut = None + self.out_dim = channels[-1] + + def forward(self, x): + out = self.conv(x) + # if self.additive: return additive_func(out, x) + if self.shortcut: return out + self.shortcut(x) + else : return out + + +class InferMobileNetV2(nn.Module): + def __init__(self, num_classes, xchannels, xblocks, dropout): + super(InferMobileNetV2, self).__init__() + block = InvertedResidual + inverted_residual_setting = [ + # t, c, n, s + [1, 16 , 1, 1], + [6, 24 , 2, 2], + [6, 32 , 3, 2], + [6, 64 , 4, 2], + [6, 96 , 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + assert len(inverted_residual_setting) == len(xblocks), 'invalid number of layers : {:} vs {:}'.format(len(inverted_residual_setting), len(xblocks)) + for block_num, ir_setting in zip(xblocks, inverted_residual_setting): + assert block_num <= ir_setting[2], '{:} vs {:}'.format(block_num, ir_setting) + xchannels = parse_channel_info(xchannels) + #for i, chs in enumerate(xchannels): + # if i > 0: assert chs[0] == xchannels[i-1][-1], 'Layer[{:}] is invalid {:} vs {:}'.format(i, xchannels[i-1], chs) + self.xchannels = xchannels + self.message = 'InferMobileNetV2 : xblocks={:}'.format(xblocks) + # building first layer + features = [ConvBNReLU(xchannels[0][0], xchannels[0][1], 3, 2, 1)] + last_channel_idx = 1 + + # building inverted residual blocks + for stage, (t, c, n, s) in enumerate(inverted_residual_setting): + for i in range(n): + stride = s if i == 0 else 1 + additv = True if i > 0 else False + module = block(self.xchannels[last_channel_idx], stride, t, additv) + features.append(module) + self.message += "\nstage={:}, ilayer={:02d}/{:02d}, block={:03d}, Cs={:}, stride={:}, expand={:}, original-C={:}".format(stage, i, n, len(features), self.xchannels[last_channel_idx], stride, t, c) + last_channel_idx += 1 + if i + 1 == xblocks[stage]: + out_channel = module.out_dim + for iiL in range(i+1, n): + last_channel_idx += 1 + self.xchannels[last_channel_idx][0] = module.out_dim + break + # building last several layers + features.append(ConvBNReLU(self.xchannels[last_channel_idx][0], self.xchannels[last_channel_idx][1], 1, 1, 1)) + assert last_channel_idx + 2 == len(self.xchannels), '{:} vs {:}'.format(last_channel_idx, len(self.xchannels)) + # make it nn.Sequential + self.features = nn.Sequential(*features) + + # building classifier + self.classifier = nn.Sequential( + nn.Dropout(dropout), + nn.Linear(self.xchannels[last_channel_idx][1], num_classes), + ) + + # weight initialization + self.apply( initialize_resnet ) + + def get_message(self): + return self.message + + def forward(self, inputs): + features = self.features(inputs) + vectors = features.mean([2, 3]) + predicts = self.classifier(vectors) + return features, predicts diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferTinyCellNet.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferTinyCellNet.py new file mode 100644 index 0000000..d92c222 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/InferTinyCellNet.py @@ -0,0 +1,58 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +from typing import List, Text, Any +import torch.nn as nn +from models.cell_operations import ResNetBasicblock +from models.cell_infers.cells import InferCell + + +class DynamicShapeTinyNet(nn.Module): + + def __init__(self, channels: List[int], genotype: Any, num_classes: int): + super(DynamicShapeTinyNet, self).__init__() + self._channels = channels + if len(channels) % 3 != 2: + raise ValueError('invalid number of layers : {:}'.format(len(channels))) + self._num_stage = N = len(channels) // 3 + + self.stem = nn.Sequential( + nn.Conv2d(3, channels[0], kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(channels[0])) + + # layer_channels = [C ] * N + [C*2 ] + [C*2 ] * N + [C*4 ] + [C*4 ] * N + layer_reductions = [False] * N + [True] + [False] * N + [True] + [False] * N + + c_prev = channels[0] + self.cells = nn.ModuleList() + for index, (c_curr, reduction) in enumerate(zip(channels, layer_reductions)): + if reduction : cell = ResNetBasicblock(c_prev, c_curr, 2, True) + else : cell = InferCell(genotype, c_prev, c_curr, 1) + self.cells.append( cell ) + c_prev = cell.out_dim + self._num_layer = len(self.cells) + + self.lastact = nn.Sequential(nn.BatchNorm2d(c_prev), nn.ReLU(inplace=True)) + self.global_pooling = nn.AdaptiveAvgPool2d(1) + self.classifier = nn.Linear(c_prev, num_classes) + + def get_message(self) -> Text: + string = self.extra_repr() + for i, cell in enumerate(self.cells): + string += '\n {:02d}/{:02d} :: {:}'.format(i, len(self.cells), cell.extra_repr()) + return string + + def extra_repr(self): + return ('{name}(C={_channels}, N={_num_stage}, L={_num_layer})'.format(name=self.__class__.__name__, **self.__dict__)) + + def forward(self, inputs): + feature = self.stem(inputs) + for i, cell in enumerate(self.cells): + feature = cell(feature) + + out = self.lastact(feature) + out = self.global_pooling( out ) + out = out.view(out.size(0), -1) + logits = self.classifier(out) + + return out, logits diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/__init__.py new file mode 100644 index 0000000..0f6cf36 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/__init__.py @@ -0,0 +1,9 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +from .InferCifarResNet_width import InferWidthCifarResNet +from .InferImagenetResNet import InferImagenetResNet +from .InferCifarResNet_depth import InferDepthCifarResNet +from .InferCifarResNet import InferCifarResNet +from .InferMobileNetV2 import InferMobileNetV2 +from .InferTinyCellNet import DynamicShapeTinyNet \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/shared_utils.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/shared_utils.py new file mode 100644 index 0000000..c29620c --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nas_bench_201_models/shape_infers/shared_utils.py @@ -0,0 +1,5 @@ +def parse_channel_info(xstring): + blocks = xstring.split(' ') + blocks = [x.split('-') for x in blocks] + blocks = [[int(_) for _ in x] for x in blocks] + return blocks diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/__init__.py new file mode 100644 index 0000000..61cc68f --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/__init__.py @@ -0,0 +1,2 @@ +from .evaluation_utils import obtain_accuracy +from .flop_benchmark import get_model_infos diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/evaluation_utils.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/evaluation_utils.py new file mode 100644 index 0000000..78733d9 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/evaluation_utils.py @@ -0,0 +1,17 @@ +import torch + +def obtain_accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + # correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/flop_benchmark.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/flop_benchmark.py new file mode 100644 index 0000000..133cf2c --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/nasbench_utils/flop_benchmark.py @@ -0,0 +1,181 @@ +import torch +import torch.nn as nn +import numpy as np + + +def count_parameters_in_MB(model): + if isinstance(model, nn.Module): + return np.sum(np.prod(v.size()) for v in model.parameters())/1e6 + else: + return np.sum(np.prod(v.size()) for v in model)/1e6 + + +def get_model_infos(model, shape): + #model = copy.deepcopy( model ) + + model = add_flops_counting_methods(model) + #model = model.cuda() + model.eval() + + #cache_inputs = torch.zeros(*shape).cuda() + #cache_inputs = torch.zeros(*shape) + cache_inputs = torch.rand(*shape) + if next(model.parameters()).is_cuda: cache_inputs = cache_inputs.cuda() + #print_log('In the calculating function : cache input size : {:}'.format(cache_inputs.size()), log) + with torch.no_grad(): + _____ = model(cache_inputs) + FLOPs = compute_average_flops_cost( model ) / 1e6 + Param = count_parameters_in_MB(model) + + if hasattr(model, 'auxiliary_param'): + aux_params = count_parameters_in_MB(model.auxiliary_param()) + print ('The auxiliary params of this model is : {:}'.format(aux_params)) + print ('We remove the auxiliary params from the total params ({:}) when counting'.format(Param)) + Param = Param - aux_params + + #print_log('FLOPs : {:} MB'.format(FLOPs), log) + torch.cuda.empty_cache() + model.apply( remove_hook_function ) + return FLOPs, Param + + +# ---- Public functions +def add_flops_counting_methods( model ): + model.__batch_counter__ = 0 + add_batch_counter_hook_function( model ) + model.apply( add_flops_counter_variable_or_reset ) + model.apply( add_flops_counter_hook_function ) + return model + + + +def compute_average_flops_cost(model): + """ + A method that will be available after add_flops_counting_methods() is called on a desired net object. + Returns current mean flops consumption per image. + """ + batches_count = model.__batch_counter__ + flops_sum = 0 + #or isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d) \ + for module in model.modules(): + if isinstance(module, torch.nn.Conv2d) or isinstance(module, torch.nn.Linear) \ + or isinstance(module, torch.nn.Conv1d) \ + or hasattr(module, 'calculate_flop_self'): + flops_sum += module.__flops__ + return flops_sum / batches_count + + +# ---- Internal functions +def pool_flops_counter_hook(pool_module, inputs, output): + batch_size = inputs[0].size(0) + kernel_size = pool_module.kernel_size + out_C, output_height, output_width = output.shape[1:] + assert out_C == inputs[0].size(1), '{:} vs. {:}'.format(out_C, inputs[0].size()) + + overall_flops = batch_size * out_C * output_height * output_width * kernel_size * kernel_size + pool_module.__flops__ += overall_flops + + +def self_calculate_flops_counter_hook(self_module, inputs, output): + overall_flops = self_module.calculate_flop_self(inputs[0].shape, output.shape) + self_module.__flops__ += overall_flops + + +def fc_flops_counter_hook(fc_module, inputs, output): + batch_size = inputs[0].size(0) + xin, xout = fc_module.in_features, fc_module.out_features + assert xin == inputs[0].size(1) and xout == output.size(1), 'IO=({:}, {:})'.format(xin, xout) + overall_flops = batch_size * xin * xout + if fc_module.bias is not None: + overall_flops += batch_size * xout + fc_module.__flops__ += overall_flops + + +def conv1d_flops_counter_hook(conv_module, inputs, outputs): + batch_size = inputs[0].size(0) + outL = outputs.shape[-1] + [kernel] = conv_module.kernel_size + in_channels = conv_module.in_channels + out_channels = conv_module.out_channels + groups = conv_module.groups + conv_per_position_flops = kernel * in_channels * out_channels / groups + + active_elements_count = batch_size * outL + overall_flops = conv_per_position_flops * active_elements_count + + if conv_module.bias is not None: + overall_flops += out_channels * active_elements_count + conv_module.__flops__ += overall_flops + + +def conv2d_flops_counter_hook(conv_module, inputs, output): + batch_size = inputs[0].size(0) + output_height, output_width = output.shape[2:] + + kernel_height, kernel_width = conv_module.kernel_size + in_channels = conv_module.in_channels + out_channels = conv_module.out_channels + groups = conv_module.groups + conv_per_position_flops = kernel_height * kernel_width * in_channels * out_channels / groups + + active_elements_count = batch_size * output_height * output_width + overall_flops = conv_per_position_flops * active_elements_count + + if conv_module.bias is not None: + overall_flops += out_channels * active_elements_count + conv_module.__flops__ += overall_flops + + +def batch_counter_hook(module, inputs, output): + # Can have multiple inputs, getting the first one + inputs = inputs[0] + batch_size = inputs.shape[0] + module.__batch_counter__ += batch_size + + +def add_batch_counter_hook_function(module): + if not hasattr(module, '__batch_counter_handle__'): + handle = module.register_forward_hook(batch_counter_hook) + module.__batch_counter_handle__ = handle + + +def add_flops_counter_variable_or_reset(module): + if isinstance(module, torch.nn.Conv2d) or isinstance(module, torch.nn.Linear) \ + or isinstance(module, torch.nn.Conv1d) \ + or isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d) \ + or hasattr(module, 'calculate_flop_self'): + module.__flops__ = 0 + + +def add_flops_counter_hook_function(module): + if isinstance(module, torch.nn.Conv2d): + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(conv2d_flops_counter_hook) + module.__flops_handle__ = handle + elif isinstance(module, torch.nn.Conv1d): + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(conv1d_flops_counter_hook) + module.__flops_handle__ = handle + elif isinstance(module, torch.nn.Linear): + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(fc_flops_counter_hook) + module.__flops_handle__ = handle + elif isinstance(module, torch.nn.AvgPool2d) or isinstance(module, torch.nn.MaxPool2d): + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(pool_flops_counter_hook) + module.__flops_handle__ = handle + elif hasattr(module, 'calculate_flop_self'): # self-defined module + if not hasattr(module, '__flops_handle__'): + handle = module.register_forward_hook(self_calculate_flops_counter_hook) + module.__flops_handle__ = handle + + +def remove_hook_function(module): + hookers = ['__batch_counter_handle__', '__flops_handle__'] + for hooker in hookers: + if hasattr(module, hooker): + handle = getattr(module, hooker) + handle.remove() + keys = ['__flops__', '__batch_counter__', '__flops__'] + hookers + for ckey in keys: + if hasattr(module, ckey): delattr(module, ckey) diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/__init__.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/__init__.py new file mode 100644 index 0000000..df1c298 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/__init__.py @@ -0,0 +1,28 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +from .starts import get_machine_info, save_checkpoint, copy_checkpoint +from .optimizers import get_optim_scheduler +from .starts import prepare_seed #, prepare_logger, get_machine_info, save_checkpoint, copy_checkpoint +''' +from .funcs_nasbench import evaluate_for_seed as bench_evaluate_for_seed +from .funcs_nasbench import pure_evaluate as bench_pure_evaluate +from .funcs_nasbench import get_nas_bench_loaders + +def get_procedures(procedure): + from .basic_main import basic_train, basic_valid + from .search_main import search_train, search_valid + from .search_main_v2 import search_train_v2 + from .simple_KD_main import simple_KD_train, simple_KD_valid + + train_funcs = {'basic' : basic_train, \ + 'search': search_train,'Simple-KD': simple_KD_train, \ + 'search-v2': search_train_v2} + valid_funcs = {'basic' : basic_valid, \ + 'search': search_valid,'Simple-KD': simple_KD_valid, \ + 'search-v2': search_valid} + + train_func = train_funcs[procedure] + valid_func = valid_funcs[procedure] + return train_func, valid_func +''' diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/optimizers.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/optimizers.py new file mode 100644 index 0000000..7fe086d --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/optimizers.py @@ -0,0 +1,204 @@ +##################################################### +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019.01 # +##################################################### +import math, torch +import torch.nn as nn +from bisect import bisect_right +from torch.optim import Optimizer + + +class _LRScheduler(object): + + def __init__(self, optimizer, warmup_epochs, epochs): + if not isinstance(optimizer, Optimizer): + raise TypeError('{:} is not an Optimizer'.format(type(optimizer).__name__)) + self.optimizer = optimizer + for group in optimizer.param_groups: + group.setdefault('initial_lr', group['lr']) + self.base_lrs = list(map(lambda group: group['initial_lr'], optimizer.param_groups)) + self.max_epochs = epochs + self.warmup_epochs = warmup_epochs + self.current_epoch = 0 + self.current_iter = 0 + + def extra_repr(self): + return '' + + def __repr__(self): + return ('{name}(warmup={warmup_epochs}, max-epoch={max_epochs}, current::epoch={current_epoch}, iter={current_iter:.2f}'.format(name=self.__class__.__name__, **self.__dict__) + + ', {:})'.format(self.extra_repr())) + + def state_dict(self): + return {key: value for key, value in self.__dict__.items() if key != 'optimizer'} + + def load_state_dict(self, state_dict): + self.__dict__.update(state_dict) + + def get_lr(self): + raise NotImplementedError + + def get_min_info(self): + lrs = self.get_lr() + return '#LR=[{:.6f}~{:.6f}] epoch={:03d}, iter={:4.2f}#'.format(min(lrs), max(lrs), self.current_epoch, self.current_iter) + + def get_min_lr(self): + return min( self.get_lr() ) + + def update(self, cur_epoch, cur_iter): + if cur_epoch is not None: + assert isinstance(cur_epoch, int) and cur_epoch>=0, 'invalid cur-epoch : {:}'.format(cur_epoch) + self.current_epoch = cur_epoch + if cur_iter is not None: + assert isinstance(cur_iter, float) and cur_iter>=0, 'invalid cur-iter : {:}'.format(cur_iter) + self.current_iter = cur_iter + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr + + + +class CosineAnnealingLR(_LRScheduler): + + def __init__(self, optimizer, warmup_epochs, epochs, T_max, eta_min): + self.T_max = T_max + self.eta_min = eta_min + super(CosineAnnealingLR, self).__init__(optimizer, warmup_epochs, epochs) + + def extra_repr(self): + return 'type={:}, T-max={:}, eta-min={:}'.format('cosine', self.T_max, self.eta_min) + + def get_lr(self): + lrs = [] + for base_lr in self.base_lrs: + if self.current_epoch >= self.warmup_epochs and self.current_epoch < self.max_epochs: + last_epoch = self.current_epoch - self.warmup_epochs + #if last_epoch < self.T_max: + #if last_epoch < self.max_epochs: + lr = self.eta_min + (base_lr - self.eta_min) * (1 + math.cos(math.pi * last_epoch / self.T_max)) / 2 + #else: + # lr = self.eta_min + (base_lr - self.eta_min) * (1 + math.cos(math.pi * (self.T_max-1.0) / self.T_max)) / 2 + elif self.current_epoch >= self.max_epochs: + lr = self.eta_min + else: + lr = (self.current_epoch / self.warmup_epochs + self.current_iter / self.warmup_epochs) * base_lr + lrs.append( lr ) + return lrs + + + +class MultiStepLR(_LRScheduler): + + def __init__(self, optimizer, warmup_epochs, epochs, milestones, gammas): + assert len(milestones) == len(gammas), 'invalid {:} vs {:}'.format(len(milestones), len(gammas)) + self.milestones = milestones + self.gammas = gammas + super(MultiStepLR, self).__init__(optimizer, warmup_epochs, epochs) + + def extra_repr(self): + return 'type={:}, milestones={:}, gammas={:}, base-lrs={:}'.format('multistep', self.milestones, self.gammas, self.base_lrs) + + def get_lr(self): + lrs = [] + for base_lr in self.base_lrs: + if self.current_epoch >= self.warmup_epochs: + last_epoch = self.current_epoch - self.warmup_epochs + idx = bisect_right(self.milestones, last_epoch) + lr = base_lr + for x in self.gammas[:idx]: lr *= x + else: + lr = (self.current_epoch / self.warmup_epochs + self.current_iter / self.warmup_epochs) * base_lr + lrs.append( lr ) + return lrs + + +class ExponentialLR(_LRScheduler): + + def __init__(self, optimizer, warmup_epochs, epochs, gamma): + self.gamma = gamma + super(ExponentialLR, self).__init__(optimizer, warmup_epochs, epochs) + + def extra_repr(self): + return 'type={:}, gamma={:}, base-lrs={:}'.format('exponential', self.gamma, self.base_lrs) + + def get_lr(self): + lrs = [] + for base_lr in self.base_lrs: + if self.current_epoch >= self.warmup_epochs: + last_epoch = self.current_epoch - self.warmup_epochs + assert last_epoch >= 0, 'invalid last_epoch : {:}'.format(last_epoch) + lr = base_lr * (self.gamma ** last_epoch) + else: + lr = (self.current_epoch / self.warmup_epochs + self.current_iter / self.warmup_epochs) * base_lr + lrs.append( lr ) + return lrs + + +class LinearLR(_LRScheduler): + + def __init__(self, optimizer, warmup_epochs, epochs, max_LR, min_LR): + self.max_LR = max_LR + self.min_LR = min_LR + super(LinearLR, self).__init__(optimizer, warmup_epochs, epochs) + + def extra_repr(self): + return 'type={:}, max_LR={:}, min_LR={:}, base-lrs={:}'.format('LinearLR', self.max_LR, self.min_LR, self.base_lrs) + + def get_lr(self): + lrs = [] + for base_lr in self.base_lrs: + if self.current_epoch >= self.warmup_epochs: + last_epoch = self.current_epoch - self.warmup_epochs + assert last_epoch >= 0, 'invalid last_epoch : {:}'.format(last_epoch) + ratio = (self.max_LR - self.min_LR) * last_epoch / self.max_epochs / self.max_LR + lr = base_lr * (1-ratio) + else: + lr = (self.current_epoch / self.warmup_epochs + self.current_iter / self.warmup_epochs) * base_lr + lrs.append( lr ) + return lrs + + + +class CrossEntropyLabelSmooth(nn.Module): + + def __init__(self, num_classes, epsilon): + super(CrossEntropyLabelSmooth, self).__init__() + self.num_classes = num_classes + self.epsilon = epsilon + self.logsoftmax = nn.LogSoftmax(dim=1) + + def forward(self, inputs, targets): + log_probs = self.logsoftmax(inputs) + targets = torch.zeros_like(log_probs).scatter_(1, targets.unsqueeze(1), 1) + targets = (1 - self.epsilon) * targets + self.epsilon / self.num_classes + loss = (-targets * log_probs).mean(0).sum() + return loss + + + +def get_optim_scheduler(parameters, config): + assert hasattr(config, 'optim') and hasattr(config, 'scheduler') and hasattr(config, 'criterion'), 'config must have optim / scheduler / criterion keys instead of {:}'.format(config) + if config.optim == 'SGD': + optim = torch.optim.SGD(parameters, config.LR, momentum=config.momentum, weight_decay=config.decay, nesterov=config.nesterov) + elif config.optim == 'RMSprop': + optim = torch.optim.RMSprop(parameters, config.LR, momentum=config.momentum, weight_decay=config.decay) + else: + raise ValueError('invalid optim : {:}'.format(config.optim)) + + if config.scheduler == 'cos': + T_max = getattr(config, 'T_max', config.epochs) + scheduler = CosineAnnealingLR(optim, config.warmup, config.epochs, T_max, config.eta_min) + elif config.scheduler == 'multistep': + scheduler = MultiStepLR(optim, config.warmup, config.epochs, config.milestones, config.gammas) + elif config.scheduler == 'exponential': + scheduler = ExponentialLR(optim, config.warmup, config.epochs, config.gamma) + elif config.scheduler == 'linear': + scheduler = LinearLR(optim, config.warmup, config.epochs, config.LR, config.LR_min) + else: + raise ValueError('invalid scheduler : {:}'.format(config.scheduler)) + + if config.criterion == 'Softmax': + criterion = torch.nn.CrossEntropyLoss() + elif config.criterion == 'SmoothSoftmax': + criterion = CrossEntropyLabelSmooth(config.class_num, config.label_smooth) + else: + raise ValueError('invalid criterion : {:}'.format(config.criterion)) + return optim, scheduler, criterion diff --git a/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/starts.py b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/starts.py new file mode 100644 index 0000000..b1b19d3 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/nas_bench_201/procedures/starts.py @@ -0,0 +1,64 @@ +################################################## +# Copyright (c) Xuanyi Dong [GitHub D-X-Y], 2019 # +################################################## +import os, sys, torch, random, PIL, copy, numpy as np +from os import path as osp +from shutil import copyfile + + +def prepare_seed(rand_seed): + random.seed(rand_seed) + np.random.seed(rand_seed) + torch.manual_seed(rand_seed) + torch.cuda.manual_seed(rand_seed) + torch.cuda.manual_seed_all(rand_seed) + + +def prepare_logger(xargs): + args = copy.deepcopy( xargs ) + from log_utils import Logger + logger = Logger(args.save_dir, args.rand_seed) + logger.log('Main Function with logger : {:}'.format(logger)) + logger.log('Arguments : -------------------------------') + for name, value in args._get_kwargs(): + logger.log('{:16} : {:}'.format(name, value)) + logger.log("Python Version : {:}".format(sys.version.replace('\n', ' '))) + logger.log("Pillow Version : {:}".format(PIL.__version__)) + logger.log("PyTorch Version : {:}".format(torch.__version__)) + logger.log("cuDNN Version : {:}".format(torch.backends.cudnn.version())) + logger.log("CUDA available : {:}".format(torch.cuda.is_available())) + logger.log("CUDA GPU numbers : {:}".format(torch.cuda.device_count())) + logger.log("CUDA_VISIBLE_DEVICES : {:}".format(os.environ['CUDA_VISIBLE_DEVICES'] if 'CUDA_VISIBLE_DEVICES' in os.environ else 'None')) + return logger + + +def get_machine_info(): + info = "Python Version : {:}".format(sys.version.replace('\n', ' ')) + info+= "\nPillow Version : {:}".format(PIL.__version__) + info+= "\nPyTorch Version : {:}".format(torch.__version__) + info+= "\ncuDNN Version : {:}".format(torch.backends.cudnn.version()) + info+= "\nCUDA available : {:}".format(torch.cuda.is_available()) + info+= "\nCUDA GPU numbers : {:}".format(torch.cuda.device_count()) + if 'CUDA_VISIBLE_DEVICES' in os.environ: + info+= "\nCUDA_VISIBLE_DEVICES={:}".format(os.environ['CUDA_VISIBLE_DEVICES']) + else: + info+= "\nDoes not set CUDA_VISIBLE_DEVICES" + return info + + +def save_checkpoint(state, filename, logger): + if osp.isfile(filename): + if hasattr(logger, 'log'): logger.log('Find {:} exist, delete is at first before saving'.format(filename)) + os.remove(filename) + torch.save(state, filename) + assert osp.isfile(filename), 'save filename : {:} failed, which is not found.'.format(filename) + if hasattr(logger, 'log'): logger.log('save checkpoint into {:}'.format(filename)) + return filename + + +def copy_checkpoint(src, dst, logger): + if osp.isfile(dst): + if hasattr(logger, 'log'): logger.log('Find {:} exist, delete is at first before saving'.format(dst)) + os.remove(dst) + copyfile(src, dst) + if hasattr(logger, 'log'): logger.log('copy the file from {:} into {:}'.format(src, dst)) diff --git a/NAS-Bench-201/main_exp/transfer_nag/run_multi_proc.py b/NAS-Bench-201/main_exp/transfer_nag/run_multi_proc.py new file mode 100644 index 0000000..a40fb58 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/run_multi_proc.py @@ -0,0 +1,83 @@ +from torch.multiprocessing import Process +import os +from absl import app, flags +import sys +import torch + +sys.path.append(os.path.join(os.getcwd(), 'main_exp')) +from nas_bench_201 import train_single_model +from all_path import NASBENCH201 + +FLAGS = flags.FLAGS +flags.DEFINE_integer("num_split", 15, "The number of splits") +flags.DEFINE_list("arch_idx_lst", None, "arch index list") +flags.DEFINE_list("arch_str_lst", None, "arch str list") +flags.DEFINE_string("meta_test_path", None, "meta test path") +flags.DEFINE_string("data_name", None, "data_name") +flags.DEFINE_string("raw_data_path", None, "raw_data_path") + + +def run_single_process(rank, seed, arch_idx, meta_test_path, data_name, + raw_data_path, num_split=15, backend="nccl"): + # 8 GPUs + device = ['0', '1', '2', '3', '4', '5', '6', '7', '0', '1', '2', '3', '4', '5', '6', '7', + '0', '1', '2', '3', '4', '5', '6', '7', '0', '1', '2', '3', '4', '5', '6', '7'][rank] + os.environ["CUDA_VISIBLE_DEVICES"] = device + + save_path = os.path.join(meta_test_path, str(arch_idx)) + if type(seed) == int: + seeds = [seed] + elif type(seed) in [list, tuple]: + seeds = seed + + nasbench201 = torch.load(NASBENCH201) + arch_str = nasbench201['arch']['str'][arch_idx] + os.makedirs(save_path, exist_ok=True) + train_single_model(save_dir=save_path, + workers=24, + datasets=[data_name], + xpaths=[f'{raw_data_path}/{data_name}'], + splits=[0], + use_less=False, + seeds=seeds, + model_str=arch_str, + arch_config={'channel': 16, 'num_cells': 5}) + + +def run_multi_process(argv): + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "1234" + os.environ["WANDB_SILENT"] = "true" + processes = [] + + arch_idx_lst = [int(i) for i in FLAGS.arch_idx_lst] + seeds = [777, 888, 999] * len(arch_idx_lst) + arch_idx_lst_ = [] + for i in arch_idx_lst: + arch_idx_lst_ += [i] * 3 + + for arch_idx in arch_idx_lst: + os.makedirs(os.path.join(FLAGS.meta_test_path, str(arch_idx)), exist_ok=True) + + for rank in range(FLAGS.num_split): + arch_idx = arch_idx_lst_[rank] + seed = seeds[rank] + p = Process(target=run_single_process, args=(rank, + seed, + arch_idx, + FLAGS.meta_test_path, + FLAGS.data_name, + FLAGS.raw_data_path)) + p.start() + processes.append(p) + + for p in processes: + p.join() + + while any(p.is_alive() for p in processes): + continue + print("All processes have completed.") + + +if __name__ == "__main__": + app.run(run_multi_process) \ No newline at end of file diff --git a/NAS-Bench-201/main_exp/transfer_nag/set_encoder/setenc_models.py b/NAS-Bench-201/main_exp/transfer_nag/set_encoder/setenc_models.py new file mode 100644 index 0000000..72b0fb8 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/set_encoder/setenc_models.py @@ -0,0 +1,38 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from set_encoder.setenc_modules import * + + +class SetPool(nn.Module): + def __init__(self, dim_input, num_outputs, dim_output, + num_inds=32, dim_hidden=128, num_heads=4, ln=False, mode=None): + super(SetPool, self).__init__() + if 'sab' in mode: # [32, 400, 128] + self.enc = nn.Sequential( + SAB(dim_input, dim_hidden, num_heads, ln=ln), # SAB? + SAB(dim_hidden, dim_hidden, num_heads, ln=ln)) + else: # [32, 400, 128] + self.enc = nn.Sequential( + ISAB(dim_input, dim_hidden, num_heads, num_inds, ln=ln), # SAB? + ISAB(dim_hidden, dim_hidden, num_heads, num_inds, ln=ln)) + if 'PF' in mode: # [32, 1, 501] + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), + nn.Linear(dim_hidden, dim_output)) + elif 'P' in mode: + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln)) + else: # torch.Size([32, 1, 501]) + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), # 32 1 128 + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + nn.Linear(dim_hidden, dim_output)) + # "", sm, sab, sabsm + + def forward(self, X): + x1 = self.enc(X) + x2 = self.dec(x1) + return x2 diff --git a/NAS-Bench-201/main_exp/transfer_nag/set_encoder/setenc_modules.py b/NAS-Bench-201/main_exp/transfer_nag/set_encoder/setenc_modules.py new file mode 100644 index 0000000..54fe4d7 --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/set_encoder/setenc_modules.py @@ -0,0 +1,67 @@ +##################################################################################### +# Copyright (c) Juho Lee SetTransformer, ICML 2019 [GitHub set_transformer] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + +class MAB(nn.Module): + def __init__(self, dim_Q, dim_K, dim_V, num_heads, ln=False): + super(MAB, self).__init__() + self.dim_V = dim_V + self.num_heads = num_heads + self.fc_q = nn.Linear(dim_Q, dim_V) + self.fc_k = nn.Linear(dim_K, dim_V) + self.fc_v = nn.Linear(dim_K, dim_V) + if ln: + self.ln0 = nn.LayerNorm(dim_V) + self.ln1 = nn.LayerNorm(dim_V) + self.fc_o = nn.Linear(dim_V, dim_V) + + def forward(self, Q, K): + Q = self.fc_q(Q) + K, V = self.fc_k(K), self.fc_v(K) + + dim_split = self.dim_V // self.num_heads + Q_ = torch.cat(Q.split(dim_split, 2), 0) + K_ = torch.cat(K.split(dim_split, 2), 0) + V_ = torch.cat(V.split(dim_split, 2), 0) + + A = torch.softmax(Q_.bmm(K_.transpose(1,2))/math.sqrt(self.dim_V), 2) + O = torch.cat((Q_ + A.bmm(V_)).split(Q.size(0), 0), 2) + O = O if getattr(self, 'ln0', None) is None else self.ln0(O) + O = O + F.relu(self.fc_o(O)) + O = O if getattr(self, 'ln1', None) is None else self.ln1(O) + return O + +class SAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, ln=False): + super(SAB, self).__init__() + self.mab = MAB(dim_in, dim_in, dim_out, num_heads, ln=ln) + + def forward(self, X): + return self.mab(X, X) + +class ISAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, num_inds, ln=False): + super(ISAB, self).__init__() + self.I = nn.Parameter(torch.Tensor(1, num_inds, dim_out)) + nn.init.xavier_uniform_(self.I) + self.mab0 = MAB(dim_out, dim_in, dim_out, num_heads, ln=ln) + self.mab1 = MAB(dim_in, dim_out, dim_out, num_heads, ln=ln) + + def forward(self, X): + H = self.mab0(self.I.repeat(X.size(0), 1, 1), X) + return self.mab1(X, H) + +class PMA(nn.Module): + def __init__(self, dim, num_heads, num_seeds, ln=False): + super(PMA, self).__init__() + self.S = nn.Parameter(torch.Tensor(1, num_seeds, dim)) + nn.init.xavier_uniform_(self.S) + self.mab = MAB(dim, dim, dim, num_heads, ln=ln) + + def forward(self, X): + return self.mab(self.S.repeat(X.size(0), 1, 1), X) diff --git a/NAS-Bench-201/main_exp/transfer_nag/unnoised_model.py b/NAS-Bench-201/main_exp/transfer_nag/unnoised_model.py new file mode 100644 index 0000000..977b45e --- /dev/null +++ b/NAS-Bench-201/main_exp/transfer_nag/unnoised_model.py @@ -0,0 +1,243 @@ +###################################################################################### +# Copyright (c) muhanzhang, D-VAE, NeurIPS 2019 [GitHub D-VAE] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +from torch import nn +from set_encoder.setenc_models import SetPool + + +class MetaSurrogateUnnoisedModel(nn.Module): + def __init__(self, args, graph_config): + super(MetaSurrogateUnnoisedModel, self).__init__() + self.max_n = graph_config['max_n'] # maximum number of vertices + self.nvt = args.nvt # number of vertex types + self.START_TYPE = graph_config['START_TYPE'] + self.END_TYPE = graph_config['END_TYPE'] + self.hs = args.hs # hidden state size of each vertex + self.nz = args.nz # size of latent representation z + self.gs = args.hs # size of graph state + self.bidir = True # whether to use bidirectional encoding + self.vid = True + self.device = None + self.input_type = 'DG' + self.num_sample = args.num_sample + + if self.vid: + self.vs = self.hs + self.max_n # vertex state size = hidden state + vid + else: + self.vs = self.hs + + # 0. encoding-related + self.grue_forward = nn.GRUCell(self.nvt, self.hs) # encoder GRU + self.grue_backward = nn.GRUCell( + self.nvt, self.hs) # backward encoder GRU + self.fc1 = nn.Linear(self.gs, self.nz) # latent mean + self.fc2 = nn.Linear(self.gs, self.nz) # latent logvar + + # 2. gate-related + self.gate_forward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.gate_backward = nn.Sequential( + nn.Linear(self.vs, self.hs), + nn.Sigmoid() + ) + self.mapper_forward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) # disable bias to ensure padded zeros also mapped to zeros + self.mapper_backward = nn.Sequential( + nn.Linear(self.vs, self.hs, bias=False), + ) + + # 3. bidir-related, to unify sizes + if self.bidir: + self.hv_unify = nn.Sequential( + nn.Linear(self.hs * 2, self.hs), + ) + self.hg_unify = nn.Sequential( + nn.Linear(self.gs * 2, self.gs), + ) + + # 4. other + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + self.tanh = nn.Tanh() + self.logsoftmax1 = nn.LogSoftmax(1) + + # 6. predictor + np = self.gs + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + input_dim = 0 + if 'D' in self.input_type: + input_dim += self.nz + if 'G' in self.input_type: + input_dim += self.nz + + self.pred_fc = nn.Sequential( + nn.Linear(input_dim, self.hs), + nn.Tanh(), + nn.Linear(self.hs, 1) + ) + self.mseloss = nn.MSELoss(reduction='sum') + + def predict(self, D_mu, G_mu): + input_vec = [] + if 'D' in self.input_type: + input_vec.append(D_mu) + if 'G' in self.input_type: + input_vec.append(G_mu) + input_vec = torch.cat(input_vec, dim=1) + return self.pred_fc(input_vec) + + def get_device(self): + if self.device is None: + self.device = next(self.parameters()).device + return self.device + + def _get_zeros(self, n, length): + # get a zero hidden state + return torch.zeros(n, length).to(self.get_device()) + + def _get_zero_hidden(self, n=1): + return self._get_zeros(n, self.hs) # get a zero hidden state + + def _one_hot(self, idx, length): + if type(idx) in [list, range]: + if idx == []: + return None + idx = torch.LongTensor(idx).unsqueeze(0).t() + x = torch.zeros((len(idx), length)).scatter_( + 1, idx, 1).to(self.get_device()) + else: + idx = torch.LongTensor([idx]).unsqueeze(0) + x = torch.zeros((1, length)).scatter_( + 1, idx, 1).to(self.get_device()) + return x + + def _gated(self, h, gate, mapper): + return gate(h) * mapper(h) + + def _collate_fn(self, G): + return [g.copy() for g in G] + + def _propagate_to(self, G, v, propagator, H=None, reverse=False, gate=None, mapper=None): + # propagate messages to vertex index v for all graphs in G + # return the new messages (states) at v + G = [g for g in G if g.vcount() > v] + if len(G) == 0: + return + if H is not None: + idx = [i for i, g in enumerate(G) if g.vcount() > v] + H = H[idx] + v_types = [g.vs[v]['type'] for g in G] + X = self._one_hot(v_types, self.nvt) + if reverse: + H_name = 'H_backward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.successors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.successors(v), self.max_n) for g in G] + gate, mapper = self.gate_backward, self.mapper_backward + else: + H_name = 'H_forward' # name of the hidden states attribute + H_pred = [[g.vs[x][H_name] for x in g.predecessors(v)] for g in G] + if self.vid: + vids = [self._one_hot(g.predecessors(v), self.max_n) + for g in G] + if gate is None: + gate, mapper = self.gate_forward, self.mapper_forward + if self.vid: + H_pred = [[torch.cat([x[i], y[i:i + 1]], 1) + for i in range(len(x))] for x, y in zip(H_pred, vids)] + # if h is not provided, use gated sum of v's predecessors' states as the input hidden state + if H is None: + # maximum number of predecessors + max_n_pred = max([len(x) for x in H_pred]) + if max_n_pred == 0: + H = self._get_zero_hidden(len(G)) + else: + H_pred = [torch.cat(h_pred + + [self._get_zeros(max_n_pred - len(h_pred), self.vs)], 0).unsqueeze(0) + for h_pred in H_pred] # pad all to same length + H_pred = torch.cat(H_pred, 0) # batch * max_n_pred * vs + H = self._gated(H_pred, gate, mapper).sum(1) # batch * hs + Hv = propagator(X, H) + for i, g in enumerate(G): + g.vs[v][H_name] = Hv[i:i + 1] + return Hv + + def _propagate_from(self, G, v, propagator, H0=None, reverse=False): + # perform a series of propagation_to steps starting from v following a topo order + # assume the original vertex indices are in a topological order + if reverse: + prop_order = range(v, -1, -1) + else: + prop_order = range(v, self.max_n) + Hv = self._propagate_to(G, v, propagator, H0, + reverse=reverse) # the initial vertex + for v_ in prop_order[1:]: + self._propagate_to(G, v_, propagator, reverse=reverse) + return Hv + + def _get_graph_state(self, G, decode=False): + # get the graph states + # when decoding, use the last generated vertex's state as the graph state + # when encoding, use the ending vertex state or unify the starting and ending vertex states + Hg = [] + for g in G: + hg = g.vs[g.vcount() - 1]['H_forward'] + if self.bidir and not decode: # decoding never uses backward propagation + hg_b = g.vs[0]['H_backward'] + hg = torch.cat([hg, hg_b], 1) + Hg.append(hg) + Hg = torch.cat(Hg, 0) + if self.bidir and not decode: + Hg = self.hg_unify(Hg) + return Hg + + def set_encode(self, X): + proto_batch = [] + for x in X: + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + return v + + def graph_encode(self, G): + # encode graphs G into latent vectors + if type(G) != list: + G = [G] + self._propagate_from(G, 0, self.grue_forward, H0=self._get_zero_hidden(len(G)), + reverse=False) + if self.bidir: + self._propagate_from(G, self.max_n - 1, self.grue_backward, + H0=self._get_zero_hidden(len(G)), reverse=True) + Hg = self._get_graph_state(G) + mu = self.fc1(Hg) + # logvar = self.fc2(Hg) + return mu # , logvar + + def reparameterize(self, mu, logvar, eps_scale=0.01): + # return z ~ N(mu, std) + if self.training: + std = logvar.mul(0.5).exp_() + eps = torch.randn_like(std) * eps_scale + return eps.mul(std).add_(mu) + else: + return mu diff --git a/NAS-Bench-201/main_exp/utils.py b/NAS-Bench-201/main_exp/utils.py new file mode 100644 index 0000000..8f52551 --- /dev/null +++ b/NAS-Bench-201/main_exp/utils.py @@ -0,0 +1,33 @@ +import os +import logging +import torch +import numpy as np +import random + +def restore_checkpoint(ckpt_dir, state, device, resume=False): + if not resume: + os.makedirs(os.path.dirname(ckpt_dir), exist_ok=True) + return state + elif not os.path.exists(ckpt_dir): + if not os.path.exists(os.path.dirname(ckpt_dir)): + os.makedirs(os.path.dirname(ckpt_dir)) + logging.warning(f"No checkpoint found at {ckpt_dir}. " + f"Returned the same state as input") + return state + else: + loaded_state = torch.load(ckpt_dir, map_location=device) + for k in state: + if k in ['optimizer', 'model', 'ema']: + state[k].load_state_dict(loaded_state[k]) + else: + state[k] = loaded_state[k] + return state + + +def reset_seed(seed): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + np.random.seed(seed) + random.seed(seed) + torch.backends.cudnn.deterministic = True + diff --git a/NAS-Bench-201/models/__init__.py b/NAS-Bench-201/models/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/NAS-Bench-201/models/cate.py b/NAS-Bench-201/models/cate.py new file mode 100644 index 0000000..628666c --- /dev/null +++ b/NAS-Bench-201/models/cate.py @@ -0,0 +1,391 @@ +# Most of this code is from https://github.com/AIoT-MLSys-Lab/CATE.git +# which was authored by Shen Yan, Kaiqiang Song, Fei Liu, Mi Zhang, 2021 + + +import torch.nn as nn +import torch +import math +import torch +import torch.nn as nn +import torch.nn.functional as F +from . import utils +from .transformer import Encoder, SemanticEmbedding +from .set_encoder.setenc_models import SetPool + + +class MLP(torch.nn.Module): + def __init__(self, num_layers, input_dim, hidden_dim, output_dim, use_bn=False, activate_func=F.relu): + """ + num_layers: number of layers in the neural networks (EXCLUDING the input layer). If num_layers=1, this reduces to linear model. + input_dim: dimensionality of input features + hidden_dim: dimensionality of hidden units at ALL layers + output_dim: number of classes for prediction + num_classes: the number of classes of input, to be treated with different gains and biases, + (see the definition of class `ConditionalLayer1d`) + """ + + super(MLP, self).__init__() + + self.linear_or_not = True # default is linear model + self.num_layers = num_layers + self.use_bn = use_bn + self.activate_func = activate_func + + if num_layers < 1: + raise ValueError("number of layers should be positive!") + elif num_layers == 1: + # Linear model + self.linear = torch.nn.Linear(input_dim, output_dim) + else: + # Multi-layer model + self.linear_or_not = False + self.linears = torch.nn.ModuleList() + + self.linears.append(torch.nn.Linear(input_dim, hidden_dim)) + for layer in range(num_layers - 2): + self.linears.append(torch.nn.Linear(hidden_dim, hidden_dim)) + self.linears.append(torch.nn.Linear(hidden_dim, output_dim)) + + if self.use_bn: + self.batch_norms = torch.nn.ModuleList() + for layer in range(num_layers - 1): + self.batch_norms.append(torch.nn.BatchNorm1d(hidden_dim)) + + + def forward(self, x): + """ + :param x: [num_classes * batch_size, N, F_i], batch of node features + note that in self.cond_layers[layer], + `x` is splited into `num_classes` groups in dim=0, + and then treated with different gains and biases + """ + if self.linear_or_not: + # If linear model + return self.linear(x) + else: + # If MLP + h = x + for layer in range(self.num_layers - 1): + h = self.linears[layer](h) + if self.use_bn: + h = self.batch_norms[layer](h) + h = self.activate_func(h) + return self.linears[self.num_layers - 1](h) + + +""" Transformer Encoder """ +class GraphEncoder(nn.Module): + def __init__(self, config): + super(GraphEncoder, self).__init__() + # Forward Transformers + self.encoder_f = Encoder(config) + + def forward(self, x, mask): + h_f, hs_f, attns_f = self.encoder_f(x, mask) + h = torch.cat(hs_f, dim=-1) + return h + + @staticmethod + def get_embeddings(h_x): + h_x = h_x.cpu() + return h_x[:, -1] + + +class CLSHead(nn.Module): + def __init__(self, config, init_weights=None): + super(CLSHead, self).__init__() + self.layer_1 = nn.Linear(config.d_model, config.d_model) + self.dropout = nn.Dropout(p=config.dropout) + self.layer_2 = nn.Linear(config.d_model, config.n_vocab) + if init_weights is not None: + self.layer_2.weight = init_weights + + def forward(self, x): + x = self.dropout(torch.tanh(self.layer_1(x))) + return F.log_softmax(self.layer_2(x), dim=-1) + + +@utils.register_model(name='CATE') +class CATE(nn.Module): + def __init__(self, config): + super(CATE, self).__init__() + # Shared Embedding Layer + self.opEmb = SemanticEmbedding(config.model.graph_encoder) + self.dropout_op = nn.Dropout(p=config.model.dropout) + self.d_model = config.model.graph_encoder.d_model + self.act = act = get_act(config) + # Time + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + + # 2 GraphEncoder for X and Y + self.graph_encoder = GraphEncoder(config.model.graph_encoder) + + self.fdim = int(config.model.graph_encoder.n_layers * config.model.graph_encoder.d_model) + self.final = MLP(num_layers=3, input_dim=self.fdim, hidden_dim=2*self.fdim, output_dim=config.data.n_vocab, + use_bn=False, activate_func=F.elu) + + if 'pos_enc_type' in config.model: + self.pos_enc_type = config.model.pos_enc_type + if self.pos_enc_type == 1: + raise NotImplementedError + elif self.pos_enc_type == 2: + if config.data.name == 'NASBench201': + self.pos_encoder = PositionalEncoding_Cell(d_model=self.d_model, max_len=config.data.max_node) + else: + self.pos_encoder = PositionalEncoding_StageWise(d_model=self.d_model, max_len=config.data.max_node) + elif self.pos_enc_type == 3: + raise NotImplementedError + else: + self.pos_encoder = None + else: + self.pos_encoder = None + + + def forward(self, X, time_cond, maskX): + + emb_x = self.dropout_op(self.opEmb(X)) + + if self.pos_encoder is not None: + emb_p = self.pos_encoder(emb_x) + emb_x = emb_x + emb_p + + # Time embedding + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model) + emb_t = self.timeEmb1(emb_t) # [32, 512] + emb_t = self.timeEmb2(self.act(emb_t)) # [32, 64] + emb_t = emb_t.unsqueeze(1) + emb = emb_x + emb_t + + h_x = self.graph_encoder(emb, maskX) + h_x = self.final(h_x) + + return h_x + + +@utils.register_model(name='PredictorCATE') +class PredictorCATE(nn.Module): + def __init__(self, config): + super(PredictorCATE, self).__init__() + # Shared Embedding Layer + self.opEmb = SemanticEmbedding(config.model.graph_encoder) + self.dropout_op = nn.Dropout(p=config.model.dropout) + self.d_model = config.model.graph_encoder.d_model + self.act = act = get_act(config) + # Time + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + + # 2 GraphEncoder for X and Y + self.graph_encoder = GraphEncoder(config.model.graph_encoder) + + self.fdim = int(config.model.graph_encoder.n_layers * config.model.graph_encoder.d_model) + self.final = MLP(num_layers=3, input_dim=self.fdim, hidden_dim=2*self.fdim, output_dim=config.data.n_vocab, + use_bn=False, activate_func=F.elu) + + self.rdim = int(config.data.max_node * config.data.n_vocab) + self.regeress = MLP(num_layers=2, input_dim=self.rdim, hidden_dim=2*self.rdim, output_dim=1, + use_bn=False, activate_func=F.elu) + + def forward(self, X, time_cond, maskX): + + emb_x = self.dropout_op(self.opEmb(X)) + + # Time embedding + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model) + emb_t = self.timeEmb1(emb_t) + emb_t = self.timeEmb2(self.act(emb_t)) + emb_t = emb_t.unsqueeze(1) + emb = emb_x + emb_t + + h_x = self.graph_encoder(emb, maskX) + h_x = self.final(h_x) + h_x = h_x.reshape(h_x.size(0), -1) + h_x = self.regeress(h_x) + + return h_x + + +class PositionalEncoding_StageWise(nn.Module): + + def __init__(self, d_model, max_len): + super(PositionalEncoding_StageWise, self).__init__() + NUM_STAGE = 5 + max_len = int(max_len / NUM_STAGE) + self.encoding = torch.zeros(max_len, d_model) + self.encoding.requires_grad = False + pos = torch.arange(0, max_len) + pos = pos.float().unsqueeze(dim=1) + _2i = torch.arange(0, d_model, step=2).float() + self.encoding[:, ::2] = torch.sin(pos / (10000 ** (_2i / d_model))) + self.encoding[:, 1::2] = torch.cos(pos / (10000 ** (_2i / d_model))) + self.encoding = torch.cat([self.encoding] * NUM_STAGE, dim=0) + + def forward(self, x): + batch_size, seq_len, _ = x.size() + return self.encoding[:seq_len, :].to(x.device) + + +class PositionalEncoding_Cell(nn.Module): + + def __init__(self, d_model, max_len): + super(PositionalEncoding_Cell, self).__init__() + NUM_STAGE = 1 + max_len = int(max_len / NUM_STAGE) + self.encoding = torch.zeros(max_len, d_model) + self.encoding.requires_grad = False + pos = torch.arange(0, max_len) + pos = pos.float().unsqueeze(dim=1) + _2i = torch.arange(0, d_model, step=2).float() + self.encoding[:, ::2] = torch.sin(pos / (10000 ** (_2i / d_model))) + self.encoding[:, 1::2] = torch.cos(pos / (10000 ** (_2i / d_model))) + self.encoding = torch.cat([self.encoding] * NUM_STAGE, dim=0) + + def forward(self, x): + batch_size, seq_len, _ = x.size() + return self.encoding[:seq_len, :].to(x.device) + + +@utils.register_model(name='MetaPredictorCATE') +class MetaPredictorCATE(nn.Module): + def __init__(self, config): + super(MetaPredictorCATE, self).__init__() + + self.input_type= config.model.input_type + self.hs = config.model.hs + + self.opEmb = SemanticEmbedding(config.model.graph_encoder) + self.dropout_op = nn.Dropout(p=config.model.dropout) + self.d_model = config.model.graph_encoder.d_model + self.act = act = get_act(config) + + # Time + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + + self.graph_encoder = GraphEncoder(config.model.graph_encoder) + + self.fdim = int(config.model.graph_encoder.n_layers * config.model.graph_encoder.d_model) + self.final = MLP(num_layers=3, input_dim=self.fdim, hidden_dim=2*self.fdim, output_dim=config.data.n_vocab, + use_bn=False, activate_func=F.elu) + + self.rdim = int(config.data.max_node * config.data.n_vocab) + self.regeress = MLP(num_layers=2, input_dim=self.rdim, hidden_dim=2*self.rdim, output_dim=2*self.rdim, + use_bn=False, activate_func=F.elu) + + # Set + self.nz = config.model.nz + self.num_sample = config.model.num_sample + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + input_dim = 0 + if 'D' in self.input_type: + input_dim += self.nz + if 'A' in self.input_type: + input_dim += 2*self.rdim + + self.pred_fc = nn.Sequential( + nn.Linear(input_dim, self.hs), + nn.Tanh(), + nn.Linear(self.hs, 1) + ) + + self.sample_state = False + self.D_mu = None + + + def arch_encode(self, X, time_cond, maskX): + emb_x = self.dropout_op(self.opEmb(X)) + + # Time embedding + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model)# time embedding + emb_t = self.timeEmb1(emb_t) # [32, 512] + emb_t = self.timeEmb2(self.act(emb_t)) # [32, 64] + emb_t = emb_t.unsqueeze(1) + emb = emb_x + emb_t + + h_x = self.graph_encoder(emb, maskX) + h_x = self.final(h_x) + + h_x = h_x.reshape(h_x.size(0), -1) + h_x = self.regeress(h_x) + return h_x + + + def set_encode(self, task): + proto_batch = [] + for x in task: + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + return v + + + def predict(self, D_mu, A_mu): + input_vec = [] + if 'D' in self.input_type: + input_vec.append(D_mu) + if 'A' in self.input_type: + input_vec.append(A_mu) + input_vec = torch.cat(input_vec, dim=1) + return self.pred_fc(input_vec) + + + def forward(self, X, time_cond, maskX, task): + if self.sample_state: + if self.D_mu is None: + self.D_mu = self.set_encode(task) + D_mu = self.D_mu + else: + D_mu = self.set_encode(task) + A_mu = self.arch_encode(X, time_cond, maskX) + y_pred = self.predict(D_mu, A_mu) + return y_pred + + +def get_timestep_embedding(timesteps, embedding_dim, max_positions=10000): + assert len(timesteps.shape) == 1 + half_dim = embedding_dim // 2 + # magic number 10000 is from transformers + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) * -emb) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = F.pad(emb, (0, 1), mode='constant') + assert emb.shape == (timesteps.shape[0], embedding_dim) + return emb + + +def get_act(config): + """Get actiuvation functions from the config file.""" + + if config.model.nonlinearity.lower() == 'elu': + return nn.ELU() + elif config.model.nonlinearity.lower() == 'relu': + return nn.ReLU() + elif config.model.nonlinearity.lower() == 'lrelu': + return nn.LeakyReLU(negative_slope=0.2) + elif config.model.nonlinearity.lower() == 'swish': + return nn.SiLU() + elif config.model.nonlinearity.lower() == 'tanh': + return nn.Tanh() + else: + raise NotImplementedError('activation function does not exist!') diff --git a/NAS-Bench-201/models/digcn.py b/NAS-Bench-201/models/digcn.py new file mode 100644 index 0000000..78ce683 --- /dev/null +++ b/NAS-Bench-201/models/digcn.py @@ -0,0 +1,125 @@ +# Most of this code is from https://github.com/ultmaster/neuralpredictor.pytorch +# which was authored by Yuge Zhang, 2020 + +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + +from . import utils +from models.cate import PositionalEncoding_StageWise + + +def normalize_adj(adj): + # Row-normalize matrix + last_dim = adj.size(-1) + rowsum = adj.sum(2, keepdim=True).repeat(1, 1, last_dim) + return torch.div(adj, rowsum) + + +def graph_pooling(inputs, num_vertices): + num_vertices = num_vertices.to(inputs.device) + out = inputs.sum(1) + return torch.div(out, num_vertices.unsqueeze(-1).expand_as(out)) + + +class DirectedGraphConvolution(nn.Module): + def __init__(self, in_features, out_features): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.weight1 = nn.Parameter(torch.zeros((in_features, out_features))) + self.weight2 = nn.Parameter(torch.zeros((in_features, out_features))) + self.dropout = nn.Dropout(0.1) + self.reset_parameters() + + def reset_parameters(self): + nn.init.xavier_uniform_(self.weight1.data) + nn.init.xavier_uniform_(self.weight2.data) + + def forward(self, inputs, adj): + inputs = inputs.to(self.weight1.device) + adj = adj.to(self.weight1.device) + norm_adj = normalize_adj(adj) + output1 = F.relu(torch.matmul(norm_adj, torch.matmul(inputs, self.weight1))) + inv_norm_adj = normalize_adj(adj.transpose(1, 2)) + output2 = F.relu(torch.matmul(inv_norm_adj, torch.matmul(inputs, self.weight2))) + out = (output1 + output2) / 2 + out = self.dropout(out) + return out + + def __repr__(self): + return self.__class__.__name__ + ' (' \ + + str(self.in_features) + ' -> ' \ + + str(self.out_features) + ')' + + +@utils.register_model(name='NeuralPredictor') +class NeuralPredictor(nn.Module): + def __init__(self, config): + super().__init__() + self.gcn = [DirectedGraphConvolution(config.model.graph_encoder.initial_hidden if i == 0 else config.model.graph_encoder.gcn_hidden, + config.model.graph_encoder.gcn_hidden) + for i in range(config.model.graph_encoder.gcn_layers)] + self.gcn = nn.ModuleList(self.gcn) + self.dropout = nn.Dropout(0.1) + self.fc1 = nn.Linear(config.model.graph_encoder.gcn_hidden, config.model.graph_encoder.linear_hidden, bias=False) + self.fc2 = nn.Linear(config.model.graph_encoder.linear_hidden, 1, bias=False) + # Time + self.d_model = config.model.graph_encoder.gcn_hidden + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + self.act = act = get_act(config) + + def forward(self, X, time_cond, maskX): + out = X + adj = maskX + + numv = torch.tensor([adj.size(1)] * adj.size(0)).to(out.device) # 20 + gs = adj.size(1) # graph node number + + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model)# time embedding + emb_t = self.timeEmb1(emb_t) + emb_t = self.timeEmb2(self.act(emb_t)) # (5, 144) + + adj_with_diag = normalize_adj(adj + torch.eye(gs, device=adj.device)) # assuming diagonal is not 1 + for layer in self.gcn: + out = layer(out, adj_with_diag) + out = graph_pooling(out, numv) + # time + out = out + emb_t + out = self.fc1(out) + out = self.dropout(out) + out = self.fc2(out) + return out + + +def get_timestep_embedding(timesteps, embedding_dim, max_positions=10000): + assert len(timesteps.shape) == 1 + half_dim = embedding_dim // 2 + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) * -emb) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = F.pad(emb, (0, 1), mode='constant') + assert emb.shape == (timesteps.shape[0], embedding_dim) + return emb + + +def get_act(config): + """Get actiuvation functions from the config file.""" + + if config.model.nonlinearity.lower() == 'elu': + return nn.ELU() + elif config.model.nonlinearity.lower() == 'relu': + return nn.ReLU() + elif config.model.nonlinearity.lower() == 'lrelu': + return nn.LeakyReLU(negative_slope=0.2) + elif config.model.nonlinearity.lower() == 'swish': + return nn.SiLU() + elif config.model.nonlinearity.lower() == 'tanh': + return nn.Tanh() + else: + raise NotImplementedError('activation function does not exist!') \ No newline at end of file diff --git a/NAS-Bench-201/models/digcn_meta.py b/NAS-Bench-201/models/digcn_meta.py new file mode 100644 index 0000000..4eb80aa --- /dev/null +++ b/NAS-Bench-201/models/digcn_meta.py @@ -0,0 +1,190 @@ +# Most of this code is from https://github.com/ultmaster/neuralpredictor.pytorch +# which was authored by Yuge Zhang, 2020 + +import torch +import torch.nn as nn +import torch.nn.functional as F +import math +from . import utils +from .set_encoder.setenc_models import SetPool + + +def normalize_adj(adj): + # Row-normalize matrix + last_dim = adj.size(-1) + rowsum = adj.sum(2, keepdim=True).repeat(1, 1, last_dim) + return torch.div(adj, rowsum) + + +def graph_pooling(inputs, num_vertices): + num_vertices = num_vertices.to(inputs.device) + out = inputs.sum(1) + return torch.div(out, num_vertices.unsqueeze(-1).expand_as(out)) + + +class DirectedGraphConvolution(nn.Module): + def __init__(self, in_features, out_features): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.weight1 = nn.Parameter(torch.zeros((in_features, out_features))) + self.weight2 = nn.Parameter(torch.zeros((in_features, out_features))) + self.dropout = nn.Dropout(0.1) + self.reset_parameters() + + def reset_parameters(self): + nn.init.xavier_uniform_(self.weight1.data) + nn.init.xavier_uniform_(self.weight2.data) + + def forward(self, inputs, adj): + inputs = inputs.to(self.weight1.device) + adj = adj.to(self.weight1.device) + norm_adj = normalize_adj(adj) + output1 = F.relu(torch.matmul(norm_adj, torch.matmul(inputs, self.weight1))) + inv_norm_adj = normalize_adj(adj.transpose(1, 2)) + output2 = F.relu(torch.matmul(inv_norm_adj, torch.matmul(inputs, self.weight2))) + out = (output1 + output2) / 2 + out = self.dropout(out) + return out + + def __repr__(self): + return self.__class__.__name__ + ' (' \ + + str(self.in_features) + ' -> ' \ + + str(self.out_features) + ')' + + +@utils.register_model(name='MetaNeuralPredictor') +class MetaeuralPredictor(nn.Module): + def __init__(self, config): + super().__init__() + # Arch + self.gcn = [DirectedGraphConvolution(config.model.graph_encoder.initial_hidden if i == 0 else config.model.graph_encoder.gcn_hidden, + config.model.graph_encoder.gcn_hidden) + for i in range(config.model.graph_encoder.gcn_layers)] + self.gcn = nn.ModuleList(self.gcn) + self.dropout = nn.Dropout(0.1) + self.fc1 = nn.Linear(config.model.graph_encoder.gcn_hidden, config.model.graph_encoder.linear_hidden, bias=False) + + # Time + self.d_model = config.model.graph_encoder.gcn_hidden + self.timeEmb1 = nn.Linear(self.d_model, self.d_model * 4) + self.timeEmb2 = nn.Linear(self.d_model * 4, self.d_model) + + self.act = act = get_act(config) + self.input_type = config.model.input_type + self.hs = config.model.hs + + # Set + self.nz = config.model.nz + self.num_sample = config.model.num_sample + self.intra_setpool = SetPool(dim_input=512, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.inter_setpool = SetPool(dim_input=self.nz, + num_outputs=1, + dim_output=self.nz, + dim_hidden=self.nz, + mode='sabPF') + self.set_fc = nn.Sequential( + nn.Linear(512, self.nz), + nn.ReLU()) + + input_dim = 0 + if 'D' in self.input_type: + input_dim += self.nz + if 'A' in self.input_type: + input_dim += config.model.graph_encoder.linear_hidden + + self.pred_fc = nn.Sequential( + nn.Linear(input_dim, self.hs), + nn.Tanh(), + nn.Linear(self.hs, 1) + ) + + self.sample_state = False + self.D_mu = None + + def arch_encode(self, X, time_cond, maskX): + out = X + adj = maskX + numv = torch.tensor([adj.size(1)] * adj.size(0)).to(out.device) + gs = adj.size(1) # graph node number + + timesteps = time_cond + emb_t = get_timestep_embedding(timesteps, self.d_model) + emb_t = self.timeEmb1(emb_t) + emb_t = self.timeEmb2(self.act(emb_t)) + + adj_with_diag = normalize_adj(adj + torch.eye(gs, device=adj.device)) + for layer in self.gcn: + out = layer(out, adj_with_diag) + out = graph_pooling(out, numv) + # time + out = out + emb_t + out = self.fc1(out) + out = self.dropout(out) + + return out + + def set_encode(self, task): + proto_batch = [] + for x in task: + cls_protos = self.intra_setpool( + x.view(-1, self.num_sample, 512)).squeeze(1) + proto_batch.append( + self.inter_setpool(cls_protos.unsqueeze(0))) + v = torch.stack(proto_batch).squeeze() + return v + + def predict(self, D_mu, A_mu): + input_vec = [] + if 'D' in self.input_type: + input_vec.append(D_mu) + if 'A' in self.input_type: + input_vec.append(A_mu) + input_vec = torch.cat(input_vec, dim=1) + return self.pred_fc(input_vec) + + def forward(self, X, time_cond, maskX, task): + if self.sample_state: + if self.D_mu is None: + self.D_mu = self.set_encode(task) + D_mu = self.D_mu + else: + D_mu = self.set_encode(task) + A_mu = self.arch_encode(X, time_cond, maskX) + y_pred = self.predict(D_mu, A_mu) + return y_pred + + +def get_timestep_embedding(timesteps, embedding_dim, max_positions=10000): + assert len(timesteps.shape) == 1 + half_dim = embedding_dim // 2 + # magic number 10000 is from transformers + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) * -emb) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = F.pad(emb, (0, 1), mode='constant') + assert emb.shape == (timesteps.shape[0], embedding_dim) + return emb + + +def get_act(config): + """Get actiuvation functions from the config file.""" + + if config.model.nonlinearity.lower() == 'elu': + return nn.ELU() + elif config.model.nonlinearity.lower() == 'relu': + return nn.ReLU() + elif config.model.nonlinearity.lower() == 'lrelu': + return nn.LeakyReLU(negative_slope=0.2) + elif config.model.nonlinearity.lower() == 'swish': + return nn.SiLU() + elif config.model.nonlinearity.lower() == 'tanh': + return nn.Tanh() + else: + raise NotImplementedError('activation function does not exist!') \ No newline at end of file diff --git a/NAS-Bench-201/models/ema.py b/NAS-Bench-201/models/ema.py new file mode 100644 index 0000000..5eca0b4 --- /dev/null +++ b/NAS-Bench-201/models/ema.py @@ -0,0 +1,85 @@ +import torch + + +class ExponentialMovingAverage: + """ + Maintains (exponential) moving average of a set of parameters. + """ + + def __init__(self, parameters, decay, use_num_updates=True): + """ + Args: + parameters: Iterable of `torch.nn.Parameter`; usually the result of `model.parameters()`. + decay: The exponential decay. + use_num_updates: Whether to use number of updates when computing averages. + """ + if decay < 0.0 or decay > 1.0: + raise ValueError('Decay must be between 0 and 1') + self.decay = decay + self.num_updates = 0 if use_num_updates else None + self.shadow_params = [p.clone().detach() + for p in parameters if p.requires_grad] + self.collected_params = [] + + def update(self, parameters): + """ + Update currently maintained parameters. + + Call this every time the parameters are updated, such as the result of the `optimizer.step()` call. + + Args: + parameters: Iterable of `torch.nn.Parameter`; usually the same set of parameters used to + initialize this object. + """ + decay = self.decay + if self.num_updates is not None: + self.num_updates += 1 + decay = min(decay, (1 + self.num_updates) / (10 + self.num_updates)) + one_minus_decay = 1.0 - decay + with torch.no_grad(): + parameters = [p for p in parameters if p.requires_grad] + for s_param, param in zip(self.shadow_params, parameters): + s_param.sub_(one_minus_decay * (s_param - param)) + + def copy_to(self, parameters): + """ + Copy current parameters into given collection of parameters. + + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be + updated with the stored moving averages. + """ + parameters = [p for p in parameters if p.requires_grad] + for s_param, param in zip(self.shadow_params, parameters): + if param.requires_grad: + param.data.copy_(s_param.data) + + def store(self, parameters): + """ + Save the current parameters for restoring later. + + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be temporarily stored. + """ + self.collected_params = [param.clone() for param in parameters] + + def restore(self, parameters): + """ + Restore the parameters stored with the `store` method. + Useful to validate the model with EMA parameters without affecting the original optimization process. + Store the parameters before the `copy_to` method. + After validation (or model saving), use this to restore the former parameters. + + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be updated with the stored parameters. + """ + for c_param, param in zip(self.collected_params, parameters): + param.data.copy_(c_param.data) + + def state_dict(self): + return dict(decay=self.decay, num_updates=self.num_updates, shadow_params=self.shadow_params) + + def load_state_dict(self, state_dict): + self.decay = state_dict['decay'] + self.num_updates = state_dict['num_updates'] + self.shadow_params = state_dict['shadow_params'] diff --git a/NAS-Bench-201/models/gnns.py b/NAS-Bench-201/models/gnns.py new file mode 100644 index 0000000..2ba0351 --- /dev/null +++ b/NAS-Bench-201/models/gnns.py @@ -0,0 +1,82 @@ +import torch.nn as nn +import torch +from .trans_layers import * + + +class pos_gnn(nn.Module): + def __init__(self, act, x_ch, pos_ch, out_ch, max_node, graph_layer, n_layers=3, edge_dim=None, heads=4, + temb_dim=None, dropout=0.1, attn_clamp=False): + super().__init__() + self.out_ch = out_ch + self.Dropout_0 = nn.Dropout(dropout) + self.act = act + self.max_node = max_node + self.n_layers = n_layers + + if temb_dim is not None: + self.Dense_node0 = nn.Linear(temb_dim, x_ch) + self.Dense_node1 = nn.Linear(temb_dim, pos_ch) + self.Dense_edge0 = nn.Linear(temb_dim, edge_dim) + self.Dense_edge1 = nn.Linear(temb_dim, edge_dim) + + self.convs = nn.ModuleList() + self.edge_convs = nn.ModuleList() + self.edge_layer = nn.Linear(edge_dim * 2 + self.out_ch, edge_dim) + + for i in range(n_layers): + if i == 0: + self.convs.append(eval(graph_layer)(x_ch, pos_ch, self.out_ch//heads, heads, edge_dim=edge_dim*2, + act=act, attn_clamp=attn_clamp)) + else: + self.convs.append(eval(graph_layer) + (self.out_ch, pos_ch, self.out_ch//heads, heads, edge_dim=edge_dim*2, act=act, + attn_clamp=attn_clamp)) + self.edge_convs.append(nn.Linear(self.out_ch, edge_dim*2)) + + def forward(self, x_degree, x_pos, edge_index, dense_ori, dense_spd, dense_index, temb=None): + """ + Args: + x_degree: node degree feature [B*N, x_ch] + x_pos: node rwpe feature [B*N, pos_ch] + edge_index: [2, edge_length] + dense_ori: edge feature [B, N, N, nf//2] + dense_spd: edge shortest path distance feature [B, N, N, nf//2] # Do we need this part? # TODO + dense_index + temb: [B, temb_dim] + """ + + B, N, _, _ = dense_ori.shape + + if temb is not None: + dense_ori = dense_ori + self.Dense_edge0(self.act(temb))[:, None, None, :] + dense_spd = dense_spd + self.Dense_edge1(self.act(temb))[:, None, None, :] + + temb = temb.unsqueeze(1).repeat(1, self.max_node, 1) + temb = temb.reshape(-1, temb.shape[-1]) + x_degree = x_degree + self.Dense_node0(self.act(temb)) + x_pos = x_pos + self.Dense_node1(self.act(temb)) + + dense_edge = torch.cat([dense_ori, dense_spd], dim=-1) + + ori_edge_attr = dense_edge + h = x_degree + h_pos = x_pos + + for i_layer in range(self.n_layers): + h_edge = dense_edge[dense_index] + # update node feature + h, h_pos = self.convs[i_layer](h, h_pos, edge_index, h_edge) + h = self.Dropout_0(h) + h_pos = self.Dropout_0(h_pos) + + # update dense edge feature + h_dense_node = h.reshape(B, N, -1) + cur_edge_attr = h_dense_node.unsqueeze(1) + h_dense_node.unsqueeze(2) # [B, N, N, nf] + dense_edge = (dense_edge + self.act(self.edge_convs[i_layer](cur_edge_attr))) / math.sqrt(2.) + dense_edge = self.Dropout_0(dense_edge) + + # Concat edge attribute + h_dense_edge = torch.cat([ori_edge_attr, dense_edge], dim=-1) + h_dense_edge = self.edge_layer(h_dense_edge).permute(0, 3, 1, 2) + + return h_dense_edge diff --git a/NAS-Bench-201/models/layers.py b/NAS-Bench-201/models/layers.py new file mode 100644 index 0000000..a74efee --- /dev/null +++ b/NAS-Bench-201/models/layers.py @@ -0,0 +1,44 @@ +"""Common layers""" + +import torch.nn as nn +import torch +import torch.nn.functional as F +import math + + +def get_act(config): + """Get actiuvation functions from the config file.""" + + if config.model.nonlinearity.lower() == 'elu': + return nn.ELU() + elif config.model.nonlinearity.lower() == 'relu': + return nn.ReLU() + elif config.model.nonlinearity.lower() == 'lrelu': + return nn.LeakyReLU(negative_slope=0.2) + elif config.model.nonlinearity.lower() == 'swish': + return nn.SiLU() + elif config.model.nonlinearity.lower() == 'tanh': + return nn.Tanh() + else: + raise NotImplementedError('activation function does not exist!') + + +def conv1x1(in_planes, out_planes, stride=1, bias=True, dilation=1, padding=0): + conv = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=bias, dilation=dilation, + padding=padding) + return conv + + +# from DDPM +def get_timestep_embedding(timesteps, embedding_dim, max_positions=10000): + assert len(timesteps.shape) == 1 + half_dim = embedding_dim // 2 + # magic number 10000 is from transformers + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32, device=timesteps.device) * -emb) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = F.pad(emb, (0, 1), mode='constant') + assert emb.shape == (timesteps.shape[0], embedding_dim) + return emb diff --git a/NAS-Bench-201/models/set_encoder/setenc_models.py b/NAS-Bench-201/models/set_encoder/setenc_models.py new file mode 100644 index 0000000..61fab26 --- /dev/null +++ b/NAS-Bench-201/models/set_encoder/setenc_models.py @@ -0,0 +1,38 @@ +########################################################################################### +# Copyright (c) Hayeon Lee, Eunyoung Hyung [GitHub MetaD2A], 2021 +# Rapid Neural Architecture Search by Learning to Generate Graphs from Datasets, ICLR 2021 +########################################################################################### +from .setenc_modules import * + + +class SetPool(nn.Module): + def __init__(self, dim_input, num_outputs, dim_output, + num_inds=32, dim_hidden=128, num_heads=4, ln=False, mode=None): + super(SetPool, self).__init__() + if 'sab' in mode: # [32, 400, 128] + self.enc = nn.Sequential( + SAB(dim_input, dim_hidden, num_heads, ln=ln), # SAB? + SAB(dim_hidden, dim_hidden, num_heads, ln=ln)) + else: # [32, 400, 128] + self.enc = nn.Sequential( + ISAB(dim_input, dim_hidden, num_heads, num_inds, ln=ln), # SAB? + ISAB(dim_hidden, dim_hidden, num_heads, num_inds, ln=ln)) + if 'PF' in mode: # [32, 1, 501] + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), + nn.Linear(dim_hidden, dim_output)) + elif 'P' in mode: + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln)) + else: # torch.Size([32, 1, 501]) + self.dec = nn.Sequential( + PMA(dim_hidden, num_heads, num_outputs, ln=ln), # 32 1 128 + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + SAB(dim_hidden, dim_hidden, num_heads, ln=ln), + nn.Linear(dim_hidden, dim_output)) + # "", sm, sab, sabsm + + def forward(self, X): + x1 = self.enc(X) + x2 = self.dec(x1) + return x2 diff --git a/NAS-Bench-201/models/set_encoder/setenc_modules.py b/NAS-Bench-201/models/set_encoder/setenc_modules.py new file mode 100644 index 0000000..1e09c70 --- /dev/null +++ b/NAS-Bench-201/models/set_encoder/setenc_modules.py @@ -0,0 +1,67 @@ +##################################################################################### +# Copyright (c) Juho Lee SetTransformer, ICML 2019 [GitHub set_transformer] +# Modified by Hayeon Lee, Eunyoung Hyung, MetaD2A, ICLR2021, 2021. 03 [GitHub MetaD2A] +###################################################################################### +import torch +import torch.nn as nn +import torch.nn.functional as F +import math + +class MAB(nn.Module): + def __init__(self, dim_Q, dim_K, dim_V, num_heads, ln=False): + super(MAB, self).__init__() + self.dim_V = dim_V + self.num_heads = num_heads + self.fc_q = nn.Linear(dim_Q, dim_V) + self.fc_k = nn.Linear(dim_K, dim_V) + self.fc_v = nn.Linear(dim_K, dim_V) + if ln: + self.ln0 = nn.LayerNorm(dim_V) + self.ln1 = nn.LayerNorm(dim_V) + self.fc_o = nn.Linear(dim_V, dim_V) + + def forward(self, Q, K): + Q = self.fc_q(Q) + K, V = self.fc_k(K), self.fc_v(K) + + dim_split = self.dim_V // self.num_heads + Q_ = torch.cat(Q.split(dim_split, 2), 0) + K_ = torch.cat(K.split(dim_split, 2), 0) + V_ = torch.cat(V.split(dim_split, 2), 0) + + A = torch.softmax(Q_.bmm(K_.transpose(1,2))/math.sqrt(self.dim_V), 2) + O = torch.cat((Q_ + A.bmm(V_)).split(Q.size(0), 0), 2) + O = O if getattr(self, 'ln0', None) is None else self.ln0(O) + O = O + F.relu(self.fc_o(O)) + O = O if getattr(self, 'ln1', None) is None else self.ln1(O) + return O + +class SAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, ln=False): + super(SAB, self).__init__() + self.mab = MAB(dim_in, dim_in, dim_out, num_heads, ln=ln) + + def forward(self, X): + return self.mab(X, X) + +class ISAB(nn.Module): + def __init__(self, dim_in, dim_out, num_heads, num_inds, ln=False): + super(ISAB, self).__init__() + self.I = nn.Parameter(torch.Tensor(1, num_inds, dim_out)) + nn.init.xavier_uniform_(self.I) + self.mab0 = MAB(dim_out, dim_in, dim_out, num_heads, ln=ln) + self.mab1 = MAB(dim_in, dim_out, dim_out, num_heads, ln=ln) + + def forward(self, X): + H = self.mab0(self.I.repeat(X.size(0), 1, 1), X) + return self.mab1(X, H) + +class PMA(nn.Module): + def __init__(self, dim, num_heads, num_seeds, ln=False): + super(PMA, self).__init__() + self.S = nn.Parameter(torch.Tensor(1, num_seeds, dim)) + nn.init.xavier_uniform_(self.S) + self.mab = MAB(dim, dim, dim, num_heads, ln=ln) + + def forward(self, X): + return self.mab(self.S.repeat(X.size(0), 1, 1), X) \ No newline at end of file diff --git a/NAS-Bench-201/models/trans_layers.py b/NAS-Bench-201/models/trans_layers.py new file mode 100644 index 0000000..576f74d --- /dev/null +++ b/NAS-Bench-201/models/trans_layers.py @@ -0,0 +1,144 @@ +import math +from typing import Union, Tuple, Optional +from torch_geometric.typing import PairTensor, Adj, OptTensor + +import torch +import torch.nn as nn +from torch import Tensor +import torch.nn.functional as F +from torch.nn import Linear +from torch_scatter import scatter +from torch_geometric.nn.conv import MessagePassing +from torch_geometric.utils import softmax +import numpy as np + + +class PosTransLayer(MessagePassing): + """Involving the edge feature and updating position feature. Multiply Msg.""" + + _alpha: OptTensor + + def __init__(self, x_channels: int, pos_channels: int, out_channels: int, + heads: int = 1, dropout: float = 0., edge_dim: Optional[int] = None, + bias: bool = True, act=None, attn_clamp: bool = False, **kwargs): + kwargs.setdefault('aggr', 'add') + super(PosTransLayer, self).__init__(node_dim=0, **kwargs) + + self.x_channels = x_channels + self.pos_channels = pos_channels + self.in_channels = in_channels = x_channels + pos_channels + self.out_channels = out_channels + self.heads = heads + self.dropout = dropout + self.edge_dim = edge_dim + self.attn_clamp = attn_clamp + + if act is None: + self.act = nn.LeakyReLU(negative_slope=0.2) + else: + self.act = act + + self.lin_key = Linear(in_channels, heads * out_channels) + self.lin_query = Linear(in_channels, heads * out_channels) + self.lin_value = Linear(in_channels, heads * out_channels) + + self.lin_edge0 = Linear(edge_dim, heads * out_channels, bias=False) + self.lin_edge1 = Linear(edge_dim, heads * out_channels, bias=False) + + self.lin_pos = Linear(heads * out_channels, pos_channels, bias=False) + + self.lin_skip = Linear(x_channels, heads * out_channels, bias=bias) + self.norm1 = nn.GroupNorm(num_groups=min(heads * out_channels // 4, 32), + num_channels=heads * out_channels, eps=1e-6) + self.norm2 = nn.GroupNorm(num_groups=min(heads * out_channels // 4, 32), + num_channels=heads * out_channels, eps=1e-6) + # FFN + self.FFN = nn.Sequential(Linear(heads * out_channels, heads * out_channels), + self.act, + Linear(heads * out_channels, heads * out_channels)) + + self.reset_parameters() + + def reset_parameters(self): + self.lin_key.reset_parameters() + self.lin_query.reset_parameters() + self.lin_value.reset_parameters() + self.lin_skip.reset_parameters() + self.lin_edge0.reset_parameters() + self.lin_edge1.reset_parameters() + self.lin_pos.reset_parameters() + + def forward(self, x: OptTensor, + pos: Tensor, + edge_index: Adj, + edge_attr: OptTensor = None + ) -> Tuple[Tensor, Tensor]: + """""" + + H, C = self.heads, self.out_channels + + x_feat = torch.cat([x, pos], -1) + query = self.lin_query(x_feat).view(-1, H, C) + key = self.lin_key(x_feat).view(-1, H, C) + value = self.lin_value(x_feat).view(-1, H, C) + + # propagate_type: (x: PairTensor, edge_attr: OptTensor) + out_x, out_pos = self.propagate(edge_index, query=query, key=key, value=value, pos=pos, edge_attr=edge_attr, + size=None) + + out_x = out_x.view(-1, self.heads * self.out_channels) + + # skip connection for x + x_r = self.lin_skip(x) + out_x = (out_x + x_r) / math.sqrt(2) + out_x = self.norm1(out_x) + + # FFN + out_x = (out_x + self.FFN(out_x)) / math.sqrt(2) + out_x = self.norm2(out_x) + + # skip connection for pos + out_pos = pos + torch.tanh(pos + out_pos) + + return out_x, out_pos + + def message(self, query_i: Tensor, key_j: Tensor, value_j: Tensor, + pos_j: Tensor, + edge_attr: OptTensor, + index: Tensor, ptr: OptTensor, + size_i: Optional[int]) -> Tuple[Tensor, Tensor]: + + edge_attn = self.lin_edge0(edge_attr).view(-1, self.heads, self.out_channels) + alpha = (query_i * key_j * edge_attn).sum(dim=-1) / math.sqrt(self.out_channels) + if self.attn_clamp: + alpha = alpha.clamp(min=-5., max=5.) + + alpha = softmax(alpha, index, ptr, size_i) + alpha = F.dropout(alpha, p=self.dropout, training=self.training) + + # node feature message + msg = value_j + msg = msg * self.lin_edge1(edge_attr).view(-1, self.heads, self.out_channels) + msg = msg * alpha.view(-1, self.heads, 1) + + # node position message + pos_msg = pos_j * self.lin_pos(msg.reshape(-1, self.heads * self.out_channels)) + + return msg, pos_msg + + def aggregate(self, inputs: Tuple[Tensor, Tensor], index: Tensor, + ptr: Optional[Tensor] = None, + dim_size: Optional[int] = None) -> Tuple[Tensor, Tensor]: + if ptr is not None: + raise NotImplementedError("Not implement Ptr in aggregate") + else: + return (scatter(inputs[0], index, 0, dim_size=dim_size, reduce=self.aggr), + scatter(inputs[1], index, 0, dim_size=dim_size, reduce="mean")) + + def update(self, inputs: Tuple[Tensor, Tensor]) -> Tuple[Tensor, Tensor]: + return inputs + + def __repr__(self): + return '{}({}, {}, heads={})'.format(self.__class__.__name__, + self.in_channels, + self.out_channels, self.heads) diff --git a/NAS-Bench-201/models/transformer.py b/NAS-Bench-201/models/transformer.py new file mode 100755 index 0000000..72d9b11 --- /dev/null +++ b/NAS-Bench-201/models/transformer.py @@ -0,0 +1,255 @@ +from copy import deepcopy as cp +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def clones(module, N): + return nn.ModuleList([cp(module) for _ in range(N)]) + + +def attention(query, key, value, mask = None, dropout = None): + d_k = query.size(-1) + scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k) + if mask is not None: + scores = scores.masked_fill(mask == 0, -1e9) + attn = F.softmax(scores, dim = -1) + if dropout is not None: + attn = dropout(attn) + return torch.matmul(attn, value), attn + + +class MultiHeadAttention(nn.Module): + def __init__(self, config): + super(MultiHeadAttention, self).__init__() + + self.d_model = config.d_model + self.n_head = config.n_head + self.d_k = config.d_model // config.n_head + + self.linears = clones(nn.Linear(self.d_model, self.d_model), 4) + self.dropout = nn.Dropout(p=config.dropout) + + def forward(self, query, key, value, mask = None): + if mask is not None: + mask = mask.unsqueeze(1) + batch_size = query.size(0) + + query, key , value = [l(x).view(batch_size, -1, self.n_head, self.d_k).transpose(1,2) for l, x in zip(self.linears, (query, key, value))] + x, attn = attention(query, key, value, mask = mask, dropout = self.dropout) + x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.n_head * self.d_k) + return self.linears[3](x), attn + + +class PositionwiseFeedForward(nn.Module): + def __init__(self, config): + super(PositionwiseFeedForward, self).__init__() + + self.w_1 = nn.Linear(config.d_model, config.d_ff) + self.w_2 = nn.Linear(config.d_ff, config.d_model) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x): + return self.w_2(self.dropout(F.relu(self.w_1(x)))) + + +class PositionwiseFeedForwardLast(nn.Module): + def __init__(self, config): + super(PositionwiseFeedForwardLast, self).__init__() + + self.w_1 = nn.Linear(config.d_model, config.d_ff) + self.w_2 = nn.Linear(config.d_ff, config.n_vocab) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x): + return self.w_2(self.dropout(F.relu(self.w_1(x)))) + + +class SelfAttentionBlock(nn.Module): + def __init__(self, config): + super(SelfAttentionBlock, self).__init__() + + self.norm = nn.LayerNorm(config.d_model) + self.attn = MultiHeadAttention(config) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x, mask): + x_ = self.norm(x) + x_ , attn = self.attn(x_, x_, x_, mask) + return self.dropout(x_) + x, attn + + +class SourceAttentionBlock(nn.Module): + def __init__(self, config): + super(SourceAttentionBlock, self).__init__() + + self.norm = nn.LayerNorm(config.d_model) + self.attn = MultiHeadAttention(config) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x, m, mask): + x_ = self.norm(x) + x_, attn = self.attn(x_, m, m, mask) + return self.dropout(x_) + x, attn + + +class FeedForwardBlock(nn.Module): + def __init__(self, config): + super(FeedForwardBlock, self).__init__() + + self.norm = nn.LayerNorm(config.d_model) + self.feed_forward = PositionwiseFeedForward(config) + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, x): + x_ = self.norm(x) + x_ = self.feed_forward(x_) + return self.dropout(x_) + x + + +class FeedForwardBlockLast(nn.Module): + def __init__(self, config): + super(FeedForwardBlockLast, self).__init__() + + self.norm = nn.LayerNorm(config.d_model) + self.feed_forward = PositionwiseFeedForwardLast(config) + self.dropout = nn.Dropout(p = config.dropout) + # Only for the last layer + self.proj_fc = nn.Linear(config.d_model, config.n_vocab) + + def forward(self, x): + x_ = self.norm(x) + x_ = self.feed_forward(x_) + return self.dropout(x_) + self.proj_fc(x) + + +class EncoderBlock(nn.Module): + def __init__(self, config): + super(EncoderBlock, self).__init__() + self.self_attn = SelfAttentionBlock(config) + self.feed_forward = FeedForwardBlock(config) + + def forward(self, x, mask): + x, attn = self.self_attn(x, mask) + x = self.feed_forward(x) + return x, attn + + +class EncoderBlockLast(nn.Module): + def __init__(self, config): + super(EncoderBlockLast, self).__init__() + self.self_attn = SelfAttentionBlock(config) + self.feed_forward = FeedForwardBlockLast(config) + + def forward(self, x, mask): + x, attn = self.self_attn(x, mask) + x = self.feed_forward(x) + return x, attn + + +class DecoderBlock(nn.Module): + def __init__(self, config): + super(DecoderBlock, self).__init__() + + self.self_attn = SelfAttentionBlock(config) + self.src_attn = SourceAttentionBlock(config) + self.feed_forward = FeedForwardBlock(config) + + def forward(self, x, m, src_mask, tgt_mask): + x, attn_tgt = self.self_attn(x, tgt_mask) + x, attn_src = self.src_attn(x, m, src_mask) + x = self.feed_forward(x) + return x, attn_src, attn_tgt + + +class Encoder(nn.Module): + def __init__(self, config): + super(Encoder, self).__init__() + self.layers = clones(EncoderBlock(config), config.n_layers) + self.norms = clones(nn.LayerNorm(config.d_model), config.n_layers) + + def forward(self, x, mask): + outputs = [] + attns = [] + for layer, norm in zip(self.layers, self.norms): + x, attn = layer(x, mask) + outputs.append(norm(x)) + attns.append(attn) + return outputs[-1], outputs, attns + + +class PositionalEmbedding(nn.Module): + def __init__(self, config): + super(PositionalEmbedding, self).__init__() + + p2e = torch.zeros(config.max_len, config.d_model) + position = torch.arange(0.0, config.max_len).unsqueeze(1) + div_term = torch.exp(torch.arange(0.0, config.d_model, 2) * (- math.log(10000.0) / config.d_model)) + p2e[:, 0::2] = torch.sin(position * div_term) + p2e[:, 1::2] = torch.cos(position * div_term) + + self.register_buffer('p2e', p2e) + + def forward(self, x): + shp = x.size() + with torch.no_grad(): + emb = torch.index_select(self.p2e, 0, x.view(-1)).view(shp + (-1,)) + return emb + + +class Transformer(nn.Module): + def __init__(self, config): + super(Transformer, self).__init__() + self.p2e = PositionalEmbedding(config) + self.encoder = Encoder(config) + + def forward(self, input_emb, position_ids, attention_mask): + # position embedding projection + projection = self.p2e(position_ids) + input_emb + return self.encoder(projection, attention_mask) + + +class TokenTypeEmbedding(nn.Module): + def __init__(self, config): + super(TokenTypeEmbedding, self).__init__() + self.t2e = nn.Embedding(config.n_token_type, config.d_model) + self.d_model = config.d_model + + def forward(self, x): + return self.t2e(x) * math.sqrt(self.d_model) + + +class SemanticEmbedding(nn.Module): + def __init__(self, config): + super(SemanticEmbedding, self).__init__() + self.d_model = config.d_model + self.fc = nn.Linear(config.n_vocab, config.d_model) + + def forward(self, x): + return self.fc(x) * math.sqrt(self.d_model) + + +class Embeddings(nn.Module): + def __init__(self, config): + super(Embeddings, self).__init__() + + self.w2e = SemanticEmbedding(config) + self.p2e = PositionalEmbedding(config) + self.t2e = TokenTypeEmbedding(config) + + self.dropout = nn.Dropout(p = config.dropout) + + def forward(self, input_ids, position_ids = None, token_type_ids = None): + if position_ids is None: + batch_size, length = input_ids.size() + with torch.no_grad(): + position_ids = torch.arange(0, length).repeat(batch_size, 1) + if torch.cuda.is_available(): + position_ids = position_ids.cuda(device=input_ids.device) + + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + embeddings = self.w2e(input_ids) + self.p2e(position_ids) + self.t2e(token_type_ids) + return self.dropout(embeddings) \ No newline at end of file diff --git a/NAS-Bench-201/models/utils.py b/NAS-Bench-201/models/utils.py new file mode 100644 index 0000000..f7a4993 --- /dev/null +++ b/NAS-Bench-201/models/utils.py @@ -0,0 +1,289 @@ +import torch +import torch.nn.functional as F +import sde_lib + +_MODELS = {} + + +def register_model(cls=None, *, name=None): + """A decorator for registering model classes.""" + + def _register(cls): + if name is None: + local_name = cls.__name__ + else: + local_name = name + if local_name in _MODELS: + raise ValueError( + f'Already registered model with name: {local_name}') + _MODELS[local_name] = cls + return cls + + if cls is None: + return _register + else: + return _register(cls) + + +def get_model(name): + return _MODELS[name] + + +def create_model(config): + """Create the model.""" + model_name = config.model.name + model = get_model(model_name)(config) + model = model.to(config.device) + return model + + +def get_model_fn(model, train=False): + """Create a function to give the output of the score-based model. + + Args: + model: The score model. + train: `True` for training and `False` for evaluation. + + Returns: + A model function. + """ + + def model_fn(x, labels, *args, **kwargs): + """Compute the output of the score-based model. + + Args: + x: A mini-batch of input data (Adjacency matrices). + labels: A mini-batch of conditioning variables for time steps. Should be interpreted differently + for different models. + mask: Mask for adjacency matrices. + + Returns: + A tuple of (model output, new mutable states) + """ + if not train: + model.eval() + return model(x, labels, *args, **kwargs) + else: + model.train() + return model(x, labels, *args, **kwargs) + + return model_fn + + +def get_score_fn(sde, model, train=False, continuous=False): + """Wraps `score_fn` so that the model output corresponds to a real time-dependent score function. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + model: A score model. + train: `True` for training and `False` for evaluation. + continuous: If `True`, the score-based model is expected to directly take continuous time steps. + + Returns: + A score function. + """ + model_fn = get_model_fn(model, train=train) + + if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + def score_fn(x, t, *args, **kwargs): + # Scale neural network output by standard deviation and flip sign + if continuous or isinstance(sde, sde_lib.subVPSDE): + # For VP-trained models, t=0 corresponds to the lowest noise level + # The maximum value of time embedding is assumed to 999 for continuously-trained models. + labels = t * 999 + score = model_fn(x, labels, *args, **kwargs) + std = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VP-trained models, t=0 corresponds to the lowest noise level + labels = t * (sde.N - 1) + score = model_fn(x, labels, *args, **kwargs) + std = sde.sqrt_1m_alpha_cumprod.to(labels.device)[ + labels.long()] + + score = -score / std[:, None, None] + return score + + elif isinstance(sde, sde_lib.VESDE): + def score_fn(x, t, *args, **kwargs): + if continuous: + labels = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VE-trained models, t=0 corresponds to the highest noise level + labels = sde.T - t + labels *= sde.N - 1 + labels = torch.round(labels).long() + + score = model_fn(x, labels, *args, **kwargs) + return score + + else: + raise NotImplementedError( + f"SDE class {sde.__class__.__name__} not yet supported.") + + return score_fn + + +def get_classifier_grad_fn(sde, classifier, train=False, continuous=False, + regress=True, labels='max'): + logit_fn = get_logit_fn(sde, classifier, train, continuous) + + def classifier_grad_fn(x, t, *args, **kwargs): + with torch.enable_grad(): + x_in = x.detach().requires_grad_(True) + if regress: + assert labels in ['max', 'min'] + logit = logit_fn(x_in, t, *args, **kwargs) + if labels == 'max': + prob = logit.sum() + elif labels == 'min': + prob = -logit.sum() + else: + logit = logit_fn(x_in, t, *args, **kwargs) + log_prob = F.log_softmax(logit, dim=-1) + prob = log_prob[range(len(logit)), labels.view(-1)].sum() + classifier_grad = torch.autograd.grad(prob, x_in)[0] + return classifier_grad + + return classifier_grad_fn + + +def get_logit_fn(sde, classifier, train=False, continuous=False): + classifier_fn = get_model_fn(classifier, train=train) + + if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + def logit_fn(x, t, *args, **kwargs): + # Scale neural network output by standard deviation and flip sign + if continuous or isinstance(sde, sde_lib.subVPSDE): + # For VP-trained models, t=0 corresponds to the lowest noise level + # The maximum value of time embedding is assumed to 999 for continuously-trained models. + labels = t * 999 + logit = classifier_fn(x, labels, *args, **kwargs) + else: + # For VP-trained models, t=0 corresponds to the lowest noise level + labels = t * (sde.N - 1) + logit = classifier_fn(x, labels, *args, **kwargs) + return logit + + elif isinstance(sde, sde_lib.VESDE): + def logit_fn(x, t, *args, **kwargs): + if continuous: + labels = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VE-trained models, t=0 corresponds to the highest noise level + labels = sde.T - t + labels *= sde.N - 1 + labels = torch.round(labels).long() + logit = classifier_fn(x, labels, *args, **kwargs) + return logit + + return logit_fn + + +def get_predictor_fn(sde, model, train=False, continuous=False): + """Wraps `score_fn` so that the model output corresponds to a real time-dependent score function. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + model: A predictor model. + train: `True` for training and `False` for evaluation. + continuous: If `True`, the score-based model is expected to directly take continuous time steps. + + Returns: + A score function. + """ + model_fn = get_model_fn(model, train=train) + + if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + def predictor_fn(x, t, *args, **kwargs): + # Scale neural network output by standard deviation and flip sign + if continuous or isinstance(sde, sde_lib.subVPSDE): + # For VP-trained models, t=0 corresponds to the lowest noise level + # The maximum value of time embedding is assumed to 999 for continuously-trained models. + labels = t * 999 + pred = model_fn(x, labels, *args, **kwargs) + std = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VP-trained models, t=0 corresponds to the lowest noise level + labels = t * (sde.N - 1) + pred = model_fn(x, labels, *args, **kwargs) + std = sde.sqrt_1m_alpha_cumprod.to(labels.device)[ + labels.long()] + + return pred + + elif isinstance(sde, sde_lib.VESDE): + def predictor_fn(x, t, *args, **kwargs): + if continuous: + labels = sde.marginal_prob(torch.zeros_like(x), t)[1] + else: + # For VE-trained models, t=0 corresponds to the highest noise level + labels = sde.T - t + labels *= sde.N - 1 + labels = torch.round(labels).long() + + pred = model_fn(x, labels, *args, **kwargs) + return pred + + else: + raise NotImplementedError( + f"SDE class {sde.__class__.__name__} not yet supported.") + + return predictor_fn + + +def to_flattened_numpy(x): + """Flatten a torch tensor `x` and convert it to numpy.""" + return x.detach().cpu().numpy().reshape((-1,)) + + +def from_flattened_numpy(x, shape): + """Form a torch tensor with the given `shape` from a flattened numpy array `x`.""" + return torch.from_numpy(x.reshape(shape)) + + +@torch.no_grad() +def mask_adj2node(adj_mask): + """Convert batched adjacency mask matrices to batched node mask matrices. + + Args: + adj_mask: [B, N, N] Batched adjacency mask matrices without self-loop edge. + + Output: + node_mask: [B, N] Batched node mask matrices indicating the valid nodes. + """ + + batch_size, max_num_nodes, _ = adj_mask.shape + + node_mask = adj_mask[:, 0, :].clone() + node_mask[:, 0] = 1 + + return node_mask + + +@torch.no_grad() +def get_rw_feat(k_step, dense_adj): + """Compute k_step Random Walk for given dense adjacency matrix.""" + + rw_list = [] + deg = dense_adj.sum(-1, keepdims=True) + AD = dense_adj / (deg + 1e-8) + rw_list.append(AD) + + for _ in range(k_step): + rw = torch.bmm(rw_list[-1], AD) + rw_list.append(rw) + rw_map = torch.stack(rw_list[1:], dim=1) # [B, k_step, N, N] + + rw_landing = torch.diagonal( + rw_map, offset=0, dim1=2, dim2=3) # [B, k_step, N] + rw_landing = rw_landing.permute(0, 2, 1) # [B, N, rw_depth] + + # get the shortest path distance indices + tmp_rw = rw_map.sort(dim=1)[0] + spd_ind = (tmp_rw <= 0).sum(dim=1) # [B, N, N] + + spd_onehot = torch.nn.functional.one_hot( + spd_ind, num_classes=k_step+1).to(torch.float) + spd_onehot = spd_onehot.permute(0, 3, 1, 2) # [B, kstep, N, N] + + return rw_landing, spd_onehot diff --git a/NAS-Bench-201/run_lib.py b/NAS-Bench-201/run_lib.py new file mode 100644 index 0000000..6c8b538 --- /dev/null +++ b/NAS-Bench-201/run_lib.py @@ -0,0 +1,520 @@ +import os +import torch +import numpy as np +import random +import logging +from absl import flags +from scipy.stats import pearsonr, spearmanr +import torch + +from models import cate +from models import digcn +from models import digcn_meta +import losses +import sampling +from models import utils as mutils +from models.ema import ExponentialMovingAverage +import datasets_nas +import sde_lib +from utils import * +from logger import Logger +from analysis.arch_metrics import SamplingArchMetrics, SamplingArchMetricsMeta + +FLAGS = flags.FLAGS + + +def set_exp_name(config): + if config.task == 'tr_scorenet': + exp_name = f'./results/{config.task}/{config.folder_name}' + data = config.data + + elif config.task == 'tr_meta_surrogate': + exp_name = f'./results/{config.task}/{config.folder_name}' + + os.makedirs(exp_name, exist_ok=True) + config.exp_name = exp_name + set_random_seed(config) + + return exp_name + + +def set_random_seed(config): + seed = config.seed + os.environ['PYTHONHASHSEED'] = str(seed) + + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + np.random.seed(seed) + random.seed(seed) + + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def scorenet_train(config): + """Runs the score network training pipeline. + Args: + config: Configuration to use. + """ + + ## Set logger + exp_name = set_exp_name(config) + logger = Logger( + log_dir=exp_name, + write_textfile=True) + logger.update_config(config, is_args=True) + logger.write_str(str(vars(config))) + logger.write_str('-' * 100) + + ## Create directories for experimental logs + sample_dir = os.path.join(exp_name, "samples") + os.makedirs(sample_dir, exist_ok=True) + + ## Initialize model and optimizer + score_model = mutils.create_model(config) + ema = ExponentialMovingAverage(score_model.parameters(), decay=config.model.ema_rate) + optimizer = losses.get_optimizer(config, score_model.parameters()) + state = dict(optimizer=optimizer, model=score_model, ema=ema, step=0, config=config) + + ## Create checkpoints directory + checkpoint_dir = os.path.join(exp_name, "checkpoints") + + ## Intermediate checkpoints to resume training + checkpoint_meta_dir = os.path.join(exp_name, "checkpoints-meta", "checkpoint.pth") + os.makedirs(checkpoint_dir, exist_ok=True) + os.makedirs(os.path.dirname(checkpoint_meta_dir), exist_ok=True) + + ## Resume training when intermediate checkpoints are detected + if config.resume: + state = restore_checkpoint(config.resume_ckpt_path, state, config.device, resume=config.resume) + initial_step = int(state['step']) + + ## Build dataloader and iterators + train_ds, eval_ds, test_ds = datasets_nas.get_dataset(config) + train_loader, eval_loader, test_loader = datasets_nas.get_dataloader(config, train_ds, eval_ds, test_ds) + train_iter = iter(train_loader) + + # Create data normalizer and its inverse + scaler = datasets_nas.get_data_scaler(config) + inverse_scaler = datasets_nas.get_data_inverse_scaler(config) + + ## Setup SDEs + if config.training.sde.lower() == 'vpsde': + sde = sde_lib.VPSDE(beta_min=config.model.beta_min, beta_max=config.model.beta_max, N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'vesde': + sde = sde_lib.VESDE(sigma_min=config.model.sigma_min, sigma_max=config.model.sigma_max, N=config.model.num_scales) + sampling_eps = 1e-5 + else: + raise NotImplementedError(f"SDE {config.training.sde} unknown.") + + # Build one-step training and evaluation functions + optimize_fn = losses.optimization_manager(config) + continuous = config.training.continuous + reduce_mean = config.training.reduce_mean + likelihood_weighting = config.training.likelihood_weighting + train_step_fn = losses.get_step_fn(sde=sde, + train=True, + optimize_fn=optimize_fn, + reduce_mean=reduce_mean, + continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name) + eval_step_fn = losses.get_step_fn(sde=sde, + train=False, + optimize_fn=optimize_fn, + reduce_mean=reduce_mean, + continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name) + + ## Build sampling functions + if config.training.snapshot_sampling: + sampling_shape = (config.training.eval_batch_size, config.data.max_node, config.data.n_vocab) + sampling_fn = sampling.get_sampling_fn(config=config, + sde=sde, + shape=sampling_shape, + inverse_scaler=inverse_scaler, + eps=sampling_eps) + + ## Build analysis tools + sampling_metrics = SamplingArchMetrics(config, train_ds, exp_name) + + ## Start training the score network + logging.info("Starting training loop at step %d." % (initial_step,)) + element = {'train': ['training_loss'], + 'eval': ['eval_loss'], + 'test': ['test_loss'], + 'sample': ['r_valid', 'r_unique', 'r_novel']} + + num_train_steps = config.training.n_iters + is_best = False + min_test_loss = 1e05 + for step in range(initial_step, num_train_steps+1): + try: + x, adj, extra = next(train_iter) + except StopIteration: + train_iter = train_loader.__iter__() + x, adj, extra = next(train_iter) + mask = aug_mask(adj, algo=config.data.aug_mask_algo, data=config.data.name) + x, adj, mask = scaler(x.to(config.device)), adj.to(config.device), mask.to(config.device) + batch = (x, adj, mask) + + ## Execute one training step + loss = train_step_fn(state, batch) + logger.update(key="training_loss", v=loss.item()) + if step % config.training.log_freq == 0: + logging.info("step: %d, training_loss: %.5e" % (step, loss.item())) + + ## Report the loss on evaluation dataset periodically + if step % config.training.eval_freq == 0: + for eval_x, eval_adj, eval_extra in eval_loader: + eval_mask = aug_mask(eval_adj, algo=config.data.aug_mask_algo, data=config.data.name) + eval_x, eval_adj, eval_mask = scaler(eval_x.to(config.device)), eval_adj.to(config.device), eval_mask.to(config.device) + eval_batch = (eval_x, eval_adj, eval_mask) + eval_loss = eval_step_fn(state, eval_batch) + logging.info("step: %d, eval_loss: %.5e" % (step, eval_loss.item())) + logger.update(key="eval_loss", v=eval_loss.item()) + for test_x, test_adj, test_extra in test_loader: + test_mask = aug_mask(test_adj, algo=config.data.aug_mask_algo, data=config.data.name) + test_x, test_adj, test_mask = scaler(test_x.to(config.device)), test_adj.to(config.device), test_mask.to(config.device) + test_batch = (test_x, test_adj, test_mask) + test_loss = eval_step_fn(state, test_batch) + logging.info("step: %d, test_loss: %.5e" % (step, test_loss.item())) + logger.update(key="test_loss", v=test_loss.item()) + if logger.logs['test_loss'].avg < min_test_loss: + is_best = True + + ## Save the checkpoint + if step != 0 and step % config.training.snapshot_freq == 0 or step == num_train_steps: + save_step = step // config.training.snapshot_freq + save_checkpoint(checkpoint_dir, state, step, save_step, is_best) + + ## Generate samples + if config.training.snapshot_sampling: + ema.store(score_model.parameters()) + ema.copy_to(score_model.parameters()) + sample, sample_steps, _ = sampling_fn(score_model, mask) + quantized_sample = quantize(sample) + this_sample_dir = os.path.join(sample_dir, "iter_{}".format(step)) + os.makedirs(this_sample_dir, exist_ok=True) + + ## Evaluate samples + arch_metric = sampling_metrics(arch_list=quantized_sample, this_sample_dir=this_sample_dir) + r_valid, r_unique, r_novel = arch_metric[0][0], arch_metric[0][1], arch_metric[0][2] + logger.update(key="r_valid", v=r_valid) + logger.update(key="r_unique", v=r_unique) + logger.update(key="r_novel", v=r_novel) + logging.info("r_valid: %.5e" % (r_valid)) + logging.info("r_unique: %.5e" % (r_unique)) + logging.info("r_novel: %.5e" % (r_novel)) + + if step % config.training.eval_freq == 0: + logger.write_log(element=element, step=step) + else: + logger.write_log(element={'train': ['training_loss']}, step=step) + + logger.reset() + + logger.save_log() + + +def scorenet_evaluate(config): + """Evaluate trained score network. + Args: + config: Configuration to use. + """ + + ## Set logger + exp_name = set_exp_name(config) + logger = Logger( + log_dir=exp_name, + write_textfile=True) + logger.update_config(config, is_args=True) + logger.write_str(str(vars(config))) + logger.write_str('-' * 100) + + ## Load the config of pre-trained score network + score_config = torch.load(config.scorenet_ckpt_path)['config'] + + ## Setup SDEs + if score_config.training.sde.lower() == 'vpsde': + sde = sde_lib.VPSDE(beta_min=score_config.model.beta_min, beta_max=score_config.model.beta_max, N=score_config.model.num_scales) + sampling_eps = 1e-3 + elif score_config.training.sde.lower() == 'vesde': + sde = sde_lib.VESDE(sigma_min=score_config.model.sigma_min, sigma_max=score_config.model.sigma_max, N=score_config.model.num_scales) + sampling_eps = 1e-5 + else: + raise NotImplementedError(f"SDE {config.training.sde} unknown.") + + ## Creat data normalizer and its inverse + inverse_scaler = datasets_nas.get_data_inverse_scaler(config) + + # Build the sampling function + sampling_shape = (config.eval.batch_size, score_config.data.max_node, score_config.data.n_vocab) + sampling_fn = sampling.get_sampling_fn(config=config, + sde=sde, + shape=sampling_shape, + inverse_scaler=inverse_scaler, + eps=sampling_eps) + + ## Load pre-trained score network + score_model = mutils.create_model(score_config) + ema = ExponentialMovingAverage(score_model.parameters(), decay=score_config.model.ema_rate) + state = dict(model=score_model, ema=ema, step=0, config=score_config) + state = restore_checkpoint(config.scorenet_ckpt_path, state, device=config.device, resume=True) + ema.store(score_model.parameters()) + ema.copy_to(score_model.parameters()) + + ## Build dataset + train_ds, eval_ds, test_ds = datasets_nas.get_dataset(score_config) + + ## Build analysis tools + sampling_metrics = SamplingArchMetrics(config, train_ds, exp_name) + + ## Create directories for experimental logs + sample_dir = os.path.join(exp_name, "samples") + os.makedirs(sample_dir, exist_ok=True) + + ## Start sampling + logging.info("Starting sampling") + element = {'sample': ['r_valid', 'r_unique', 'r_novel']} + + num_sampling_rounds = int(np.ceil(config.eval.num_samples / config.eval.batch_size)) + print(f'>>> Sampling for {num_sampling_rounds} rounds...') + + all_samples = [] + adj = train_ds.adj.to(config.device) + mask = train_ds.mask(algo=score_config.data.aug_mask_algo).to(config.device) + if len(adj.shape) == 2: adj = adj.unsqueeze(0) + if len(mask.shape) == 2: mask = mask.unsqueeze(0) + + for _ in range(num_sampling_rounds): + sample, sample_steps, _ = sampling_fn(score_model, mask) + quantized_sample = quantize(sample) + all_samples += quantized_sample + + ## Evaluate samples + all_samples = all_samples[:config.eval.num_samples] + arch_metric = sampling_metrics(arch_list=all_samples, this_sample_dir=sample_dir) + r_valid, r_unique, r_novel = arch_metric[0][0], arch_metric[0][1], arch_metric[0][2] + logger.update(key="r_valid", v=r_valid) + logger.update(key="r_unique", v=r_unique) + logger.update(key="r_novel", v=r_novel) + logger.write_log(element=element, step=1) + logger.save_log() + + +def meta_surrogate_train(config): + """Runs the meta-predictor model training pipeline. + Args: + config: Configuration to use. + """ + ## Set logger + exp_name = set_exp_name(config) + logger = Logger( + log_dir=exp_name, + write_textfile=True) + logger.update_config(config, is_args=True) + logger.write_str(str(vars(config))) + logger.write_str('-' * 100) + + ## Create directories for experimental logs + sample_dir = os.path.join(exp_name, "samples") + os.makedirs(sample_dir, exist_ok=True) + + ## Initialize model and optimizer + surrogate_model = mutils.create_model(config) + optimizer = losses.get_optimizer(config, surrogate_model.parameters()) + state = dict(optimizer=optimizer, model=surrogate_model, step=0, config=config) + + ## Create checkpoints directory + checkpoint_dir = os.path.join(exp_name, "checkpoints") + + ## Intermediate checkpoints to resume training + checkpoint_meta_dir = os.path.join(exp_name, "checkpoints-meta", "checkpoint.pth") + os.makedirs(checkpoint_dir, exist_ok=True) + os.makedirs(os.path.dirname(checkpoint_meta_dir), exist_ok=True) + + ## Resume training when intermediate checkpoints are detected and resume=True + state = restore_checkpoint(checkpoint_meta_dir, state, config.device, resume=config.resume) + initial_step = int(state['step']) + + ## Build dataloader and iterators + train_ds, eval_ds, test_ds = datasets_nas.get_meta_dataset(config) + train_loader, eval_loader, _ = datasets_nas.get_dataloader(config, train_ds, eval_ds, test_ds) + train_iter = iter(train_loader) + + ## Create data normalizer and its inverse + scaler = datasets_nas.get_data_scaler(config) + inverse_scaler = datasets_nas.get_data_inverse_scaler(config) + + ## Setup SDEs + if config.training.sde.lower() == 'vpsde': + sde = sde_lib.VPSDE(beta_min=config.model.beta_min, beta_max=config.model.beta_max, N=config.model.num_scales) + sampling_eps = 1e-3 + elif config.training.sde.lower() == 'vesde': + sde = sde_lib.VESDE(sigma_min=config.model.sigma_min, sigma_max=config.model.sigma_max, N=config.model.num_scales) + sampling_eps = 1e-5 + else: + raise NotImplementedError(f"SDE {config.training.sde} unknown.") + + ## Build one-step training and evaluation functions + optimize_fn = losses.optimization_manager(config) + continuous = config.training.continuous + reduce_mean = config.training.reduce_mean + likelihood_weighting = config.training.likelihood_weighting + train_step_fn = losses.get_step_fn_predictor(sde=sde, + train=True, + optimize_fn=optimize_fn, + reduce_mean=reduce_mean, + continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name, + label_list=config.data.label_list, + noised=config.training.noised) + eval_step_fn = losses.get_step_fn_predictor(sde, + train=False, + optimize_fn=optimize_fn, + reduce_mean=reduce_mean, + continuous=continuous, + likelihood_weighting=likelihood_weighting, + data=config.data.name, + label_list=config.data.label_list, + noised=config.training.noised) + + ## Build sampling functions + if config.training.snapshot_sampling: + sampling_shape = (config.training.eval_batch_size, config.data.max_node, config.data.n_vocab) + sampling_fn = sampling.get_sampling_fn(config=config, + sde=sde, + shape=sampling_shape, + inverse_scaler=inverse_scaler, + eps=sampling_eps, + conditional=True, + data_name=config.sampling.check_dataname, # for sanity check + num_sample=config.model.num_sample) + ## Load pre-trained score network + score_config = torch.load(config.scorenet_ckpt_path)['config'] + check_config(score_config, config) + score_model = mutils.create_model(score_config) + score_ema = ExponentialMovingAverage(score_model.parameters(), decay=score_config.model.ema_rate) + score_state = dict(model=score_model, ema=score_ema, step=0, config=score_config) + score_state = restore_checkpoint(config.scorenet_ckpt_path, score_state, device=config.device, resume=True) + score_ema.copy_to(score_model.parameters()) + + ## Build analysis tools + sampling_metrics = SamplingArchMetricsMeta(config, train_ds, exp_name) + + ## Start training + logging.info("Starting training loop at step %d." % (initial_step,)) + element = {'train': ['training_loss'], + 'eval': ['eval_loss', 'eval_p_corr', 'eval_s_corr'], + 'sample': ['r_valid', 'r_unique', 'r_novel']} + num_train_steps = config.training.n_iters + is_best = False + max_eval_p_corr = -1 + for step in range(initial_step, num_train_steps + 1): + try: + x, adj, extra, task = next(train_iter) + except StopIteration: + train_iter = train_loader.__iter__() + x, adj, extra, task = next(train_iter) + mask = aug_mask(adj, algo=config.data.aug_mask_algo, data=config.data.name) + x, adj, mask, task = scaler(x.to(config.device)), adj.to(config.device), mask.to(config.device), task.to(config.device) + batch = (x, adj, mask, extra, task) + + ## Execute one training step + loss, pred, labels = train_step_fn(state, batch) + logger.update(key="training_loss", v=loss.item()) + if step % config.training.log_freq == 0: + logging.info("step: %d, training_loss: %.5e" % (step, loss.item())) + + ## Report the loss on evaluation dataset periodically + if step % config.training.eval_freq == 0: + eval_pred_list, eval_labels_list = list(), list() + for eval_x, eval_adj, eval_extra, eval_task in eval_loader: + eval_mask = aug_mask(eval_adj, algo=config.data.aug_mask_algo, data=config.data.name) + eval_x, eval_adj, eval_mask, eval_task = scaler(eval_x.to(config.device)), eval_adj.to(config.device), eval_mask.to(config.device), eval_task.to(config.device) + eval_batch = (eval_x, eval_adj, eval_mask, eval_extra, eval_task) + eval_loss, eval_pred, eval_labels = eval_step_fn(state, eval_batch) + eval_pred_list += [v.detach().item() for v in eval_pred.squeeze()] + eval_labels_list += [v.detach().item() for v in eval_labels.squeeze()] + logging.info("step: %d, eval_loss: %.5e" % (step, eval_loss.item())) + logger.update(key="eval_loss", v=eval_loss.item()) + eval_p_corr = pearsonr(np.array(eval_pred_list), np.array(eval_labels_list))[0] + eval_s_corr = spearmanr(np.array(eval_pred_list), np.array(eval_labels_list))[0] + logging.info("step: %d, eval_p_corr: %.5e" % (step, eval_p_corr)) + logging.info("step: %d, eval_s_corr: %.5e" % (step, eval_s_corr)) + logger.update(key="eval_p_corr", v=eval_p_corr) + logger.update(key="eval_s_corr", v=eval_s_corr) + if eval_p_corr > max_eval_p_corr: + is_best = True + max_eval_p_corr = eval_p_corr + + ## Save a checkpoint periodically and generate samples + if step != 0 and step % config.training.snapshot_freq == 0 or step == num_train_steps: + ## Save the checkpoint. + save_step = step // config.training.snapshot_freq + save_checkpoint(checkpoint_dir, state, step, save_step, is_best) + ## Generate and save samples + if config.training.snapshot_sampling: + score_ema.store(score_model.parameters()) + score_ema.copy_to(score_model.parameters()) + sample = sampling_fn(score_model=score_model, + mask=mask, + classifier=surrogate_model, + classifier_scale=config.sampling.classifier_scale) + quantized_sample = quantize(sample) # quantization + this_sample_dir = os.path.join(sample_dir, "iter_{}".format(step)) + os.makedirs(this_sample_dir, exist_ok=True) + ## Evaluate samples + arch_metric = sampling_metrics(arch_list=quantized_sample, + this_sample_dir=this_sample_dir, + check_dataname=config.sampling.check_dataname) + r_valid, r_unique, r_novel = arch_metric[0][0], arch_metric[0][1], arch_metric[0][2] + logging.info("step: %d, r_valid: %.5e" % (step, r_valid)) + logging.info("step: %d, r_unique: %.5e" % (step, r_unique)) + logging.info("step: %d, r_novel: %.5e" % (step, r_novel)) + logger.update(key="r_valid", v=r_valid) + logger.update(key="r_unique", v=r_unique) + logger.update(key="r_novel", v=r_novel) + + if step % config.training.eval_freq == 0: + logger.write_log(element=element, step=step) + else: + logger.write_log(element={'train': ['training_loss']}, step=step) + + logger.reset() + + +def check_config(config1, config2): + assert config1.model.sigma_min == config2.model.sigma_min + assert config1.model.sigma_max == config2.model.sigma_max + assert config1.training.sde == config2.training.sde + assert config1.training.continuous == config2.training.continuous + assert config1.data.centered == config2.data.centered + assert config1.data.max_node == config2.data.max_node + assert config1.data.n_vocab == config2.data.n_vocab + + +run_train_dict = { + 'scorenet': scorenet_train, + 'meta_surrogate': meta_surrogate_train +} + + +run_eval_dict = { + 'scorenet': scorenet_evaluate, +} + + +def train(config): + run_train_dict[config.model_type](config) + + +def evaluate(config): + run_eval_dict[config.model_type](config) + diff --git a/NAS-Bench-201/sampling.py b/NAS-Bench-201/sampling.py new file mode 100644 index 0000000..8cafc5f --- /dev/null +++ b/NAS-Bench-201/sampling.py @@ -0,0 +1,579 @@ +"""Various sampling methods.""" + +import functools +import torch +import numpy as np +import abc +from tqdm import trange +import sde_lib +from models import utils as mutils +from datasets_nas import MetaTestDataset +from all_path import DATA_PATH + + +_CORRECTORS = {} +_PREDICTORS = {} + + +def register_predictor(cls=None, *, name=None): + """A decorator for registering predictor classes.""" + + def _register(cls): + if name is None: + local_name = cls.__name__ + else: + local_name = name + if local_name in _PREDICTORS: + raise ValueError(f'Already registered predictor with name: {local_name}') + _PREDICTORS[local_name] = cls + return cls + + if cls is None: + return _register + else: + return _register(cls) + + +def register_corrector(cls=None, *, name=None): + """A decorator for registering corrector classes.""" + + def _register(cls): + if name is None: + local_name = cls.__name__ + else: + local_name = name + if local_name in _CORRECTORS: + raise ValueError(f'Already registered corrector with name: {local_name}') + _CORRECTORS[local_name] = cls + return cls + + if cls is None: + return _register + else: + return _register(cls) + + +def get_predictor(name): + return _PREDICTORS[name] + + +def get_corrector(name): + return _CORRECTORS[name] + + +def get_sampling_fn( + config, + sde, + shape, + inverse_scaler, + eps, + conditional=False, + data_name='cifar10', + num_sample=20): + """Create a sampling function. + + Args: + config: A `ml_collections.ConfigDict` object that contains all configuration information. + sde: A `sde_lib.SDE` object that represents the forward SDE. + shape: A sequence of integers representing the expected shape of a single sample. + inverse_scaler: The inverse data normalizer function. + eps: A `float` number. The reverse-time SDE is only integrated to `eps` for numerical stability. + conditional: If `True`, the sampling function is conditional + data_name: A `str` name of the dataset. + num_sample: An `int` number of samples for each class of the dataset. + + Returns: + A function that takes random states and a replicated training state and outputs samples with the + trailing dimensions matching `shape`. + """ + + sampler_name = config.sampling.method + + # Predictor-Corrector sampling. Predictor-only and Corrector-only samplers are special cases. + if sampler_name.lower() == 'pc': + predictor = get_predictor(config.sampling.predictor.lower()) + corrector = get_corrector(config.sampling.corrector.lower()) + + if not conditional: + print('>>> Get pc_sampler...') + sampling_fn = get_pc_sampler_nas(sde=sde, + shape=shape, + predictor=predictor, + corrector=corrector, + inverse_scaler=inverse_scaler, + snr=config.sampling.snr, + n_steps=config.sampling.n_steps_each, + probability_flow=config.sampling.probability_flow, + continuous=config.training.continuous, + denoise=config.sampling.noise_removal, + eps=eps, + device=config.device) + else: + print('>>> Get pc_conditional_sampler...') + sampling_fn = get_pc_conditional_sampler_meta_nas(sde=sde, + shape=shape, + predictor=predictor, + corrector=corrector, + inverse_scaler=inverse_scaler, + snr=config.sampling.snr, + n_steps=config.sampling.n_steps_each, + probability_flow=config.sampling.probability_flow, + continuous=config.training.continuous, + denoise=config.sampling.noise_removal, + eps=eps, + device=config.device, + regress=config.sampling.regress, + labels=config.sampling.labels, + data_name=data_name, + num_sample=num_sample) + + else: + raise NotImplementedError(f"Sampler name {sampler_name} unknown.") + + return sampling_fn + + +class Predictor(abc.ABC): + """The abstract class for a predictor algorithm.""" + + def __init__(self, sde, score_fn, probability_flow=False): + super().__init__() + self.sde = sde + # Compute the reverse SDE/ODE + if isinstance(sde, tuple): + self.rsde = (sde[0].reverse(score_fn, probability_flow), sde[1].reverse(score_fn, probability_flow)) + else: + self.rsde = sde.reverse(score_fn, probability_flow) + self.score_fn = score_fn + + @abc.abstractmethod + def update_fn(self, x, t, *args, **kwargs): + """One update of the predictor. + + Args: + x: A PyTorch tensor representing the current state. + t: A PyTorch tensor representing the current time step. + + Returns: + x: A PyTorch tensor of the next state. + x_mean: A PyTorch tensor. The next state without random noise. Useful for denoising. + """ + pass + + +class Corrector(abc.ABC): + """The abstract class for a corrector algorithm.""" + + def __init__(self, sde, score_fn, snr, n_steps): + super().__init__() + self.sde = sde + self.score_fn = score_fn + self.snr = snr + self.n_steps = n_steps + + @abc.abstractmethod + def update_fn(self, x, t, *args, **kwargs): + """One update of the corrector. + + Args: + x: A PyTorch tensor representing the current state. + t: A PyTorch tensor representing the current time step. + + Returns: + x: A PyTorch tensor of the next state. + x_mean: A PyTorch tensor. The next state without random noise. Useful for denoising. + """ + pass + + +@register_predictor(name='euler_maruyama') +class EulerMaruyamaPredictor(Predictor): + def __init__(self, sde, score_fn, probability_flow=False): + super().__init__(sde, score_fn, probability_flow) + + def update_fn(self, x, t, *args, **kwargs): + dt = -1. / self.rsde.N + z = torch.randn_like(x) + drift, diffusion = self.rsde.sde(x, t, *args, **kwargs) + x_mean = x + drift * dt + x = x_mean + diffusion[:, None, None] * np.sqrt(-dt) * z + return x, x_mean + + +@register_predictor(name='reverse_diffusion') +class ReverseDiffusionPredictor(Predictor): + def __init__(self, sde, score_fn, probability_flow=False): + super().__init__(sde, score_fn, probability_flow) + + def update_fn(self, x, t, *args, **kwargs): + f, G = self.rsde.discretize(x, t, *args, **kwargs) + z = torch.randn_like(x) + x_mean = x - f + x = x_mean + G[:, None, None] * z + return x, x_mean + + +@register_predictor(name='none') +class NonePredictor(Predictor): + """An empty predictor that does nothing.""" + + def __init__(self, sde, score_fn, probability_flow=False): + pass + + def update_fn(self, x, t, *args, **kwargs): + return x, x + + +@register_corrector(name='langevin') +class LangevinCorrector(Corrector): + def __init__(self, sde, score_fn, snr, n_steps): + super().__init__(sde, score_fn, snr, n_steps) + + def update_fn(self, x, t, *args, **kwargs): + sde = self.sde + score_fn = self.score_fn + n_steps = self.n_steps + target_snr = self.snr + if isinstance(sde, sde_lib.VPSDE) or isinstance(sde, sde_lib.subVPSDE): + timestep = (t * (sde.N - 1) / sde.T).long() + # Note: it seems that subVPSDE doesn't set alphas + alpha = sde.alphas.to(t.device)[timestep] + else: + alpha = torch.ones_like(t) + + for i in range(n_steps): + + grad = score_fn(x, t, *args, **kwargs) + noise = torch.randn_like(x) + + grad_norm = torch.norm(grad.reshape(grad.shape[0], -1), dim=-1).mean() + noise_norm = torch.norm(noise.reshape(noise.shape[0], -1), dim=-1).mean() + + step_size = (target_snr * noise_norm / grad_norm) ** 2 * 2 * alpha + x_mean = x + step_size[:, None, None] * grad + x = x_mean + torch.sqrt(step_size * 2)[:, None, None] * noise + + return x, x_mean + + +@register_corrector(name='none') +class NoneCorrector(Corrector): + """An empty corrector that does nothing.""" + + def __init__(self, sde, score_fn, snr, n_steps): + pass + + def update_fn(self, x, t, *args, **kwargs): + return x, x + + +def shared_predictor_update_fn(x, t, sde, model, + predictor, probability_flow, continuous, *args, **kwargs): + """A wrapper that configures and returns the update function of predictors.""" + score_fn = mutils.get_score_fn(sde, model, train=False, continuous=continuous) + if predictor is None: + # Corrector-only sampler + predictor_obj = NonePredictor(sde, score_fn, probability_flow) + else: + predictor_obj = predictor(sde, score_fn, probability_flow) + + return predictor_obj.update_fn(x, t, *args, **kwargs) + + +def shared_corrector_update_fn(x, t, sde, model, + corrector, continuous, snr, n_steps, *args, **kwargs): + """A wrapper that configures and returns the update function of correctors.""" + score_fn = mutils.get_score_fn(sde, model, train=False, continuous=continuous) + + if corrector is None: + # Predictor-only sampler + corrector_obj = NoneCorrector(sde, score_fn, snr, n_steps) + else: + corrector_obj = corrector(sde, score_fn, snr, n_steps) + + return corrector_obj.update_fn(x, t, *args, **kwargs) + + +def get_pc_sampler(sde, + shape, + predictor, + corrector, + inverse_scaler, + snr, + n_steps=1, + probability_flow=False, + continuous=False, + denoise=True, + eps=1e-3, + device='cuda'): + """Create a Predictor-Corrector (PC) sampler. + + Args: + sde: An `sde_lib.SDE` object representing the forward SDE. + shape: A sequence of integers. The expected shape of a single sample. + predictor: A subclass of `sampling.Predictor` representing the predictor algorithm. + corrector: A subclass of `sampling.Corrector` representing the corrector algorithm. + inverse_scaler: The inverse data normalizer. + snr: A `float` number. The signal-to-noise ratio for configuring correctors. + n_steps: An integer. The number of corrector steps per predictor update. + probability_flow: If `True`, solve the reverse-time probability flow ODE when running the predictor. + continuous: `True` indicates that the score model was continuously trained. + denoise: If `True`, add one-step denoising to the final samples. + eps: A `float` number. The reverse-time SDE and ODE are integrated to `epsilon` to avoid numerical issues. + device: PyTorch device. + + Returns: + A sampling function that returns samples and the number of function evaluations during sampling. + """ + # Create predictor & corrector update functions + predictor_update_fn = functools.partial(shared_predictor_update_fn, + sde=sde, + predictor=predictor, + probability_flow=probability_flow, + continuous=continuous) + corrector_update_fn = functools.partial(shared_corrector_update_fn, + sde=sde, + corrector=corrector, + continuous=continuous, + snr=snr, + n_steps=n_steps) + + def pc_sampler(model, n_nodes_pmf): + """The PC sampler function. + + Args: + model: A score model. + n_nodes_pmf: Probability mass function of graph nodes. + + Returns: + Samples, number of function evaluations. + """ + with torch.no_grad(): + # Initial sample + x = sde.prior_sampling(shape).to(device) + timesteps = torch.linspace(sde.T, eps, sde.N, device=device) + + # Sample the number of nodes + n_nodes = torch.multinomial(n_nodes_pmf, shape[0], replacement=True) + mask = torch.zeros((shape[0], shape[-1]), device=device) + for i in range(shape[0]): + mask[i][:n_nodes[i]] = 1. + mask = (mask[:, None, :] * mask[:, :, None]).unsqueeze(1) + mask = torch.tril(mask, -1) + mask = mask + mask.transpose(-1, -2) + + x = x * mask + + for i in range(sde.N): + t = timesteps[i] + vec_t = torch.ones(shape[0], device=t.device) * t + x, x_mean = corrector_update_fn(x, vec_t, model=model, mask=mask) + x = x * mask + x, x_mean = predictor_update_fn(x, vec_t, model=model, mask=mask) + x = x * mask + + return inverse_scaler(x_mean if denoise else x) * mask, sde.N * (n_steps + 1), n_nodes + + return pc_sampler + + +def get_pc_sampler_nas(sde, + shape, + predictor, + corrector, + inverse_scaler, + snr, + n_steps=1, + probability_flow=False, + continuous=False, + denoise=True, + eps=1e-3, + device='cuda'): + """Create a Predictor-Corrector (PC) sampler. + + Args: + sde: An `sde_lib.SDE` object representing the forward SDE. + shape: A sequence of integers. The expected shape of a single sample. + predictor: A subclass of `sampling.Predictor` representing the predictor algorithm. + corrector: A subclass of `sampling.Corrector` representing the corrector algorithm. + inverse_scaler: The inverse data normalizer. + snr: A `float` number. The signal-to-noise ratio for configuring correctors. + n_steps: An integer. The number of corrector steps per predictor update. + probability_flow: If `True`, solve the reverse-time probability flow ODE when running the predictor. + continuous: `True` indicates that the score model was continuously trained. + denoise: If `True`, add one-step denoising to the final samples. + eps: A `float` number. The reverse-time SDE and ODE are integrated to `epsilon` to avoid numerical issues. + device: PyTorch device. + + Returns: + A sampling function that returns samples and the number of function evaluations during sampling. + """ + # Create predictor & corrector update functions + predictor_update_fn = functools.partial(shared_predictor_update_fn, + sde=sde, + predictor=predictor, + probability_flow=probability_flow, + continuous=continuous) + corrector_update_fn = functools.partial(shared_corrector_update_fn, + sde=sde, + corrector=corrector, + continuous=continuous, + snr=snr, + n_steps=n_steps) + + def pc_sampler(model, mask): + """The PC sampler function. + + Args: + model: A score model. + n_nodes_pmf: Probability mass function of graph nodes. + + Returns: + Samples, number of function evaluations. + """ + with torch.no_grad(): + # Initial sample + x = sde.prior_sampling(shape).to(device) + timesteps = torch.linspace(sde.T, eps, sde.N, device=device) + mask = mask[0].unsqueeze(0).repeat(x.size(0), 1, 1) + + for i in trange(sde.N, desc='[PC sampling]', position=1, leave=False): + t = timesteps[i] + vec_t = torch.ones(shape[0], device=t.device) * t + x, x_mean = corrector_update_fn(x, vec_t, model=model, maskX=mask) + x, x_mean = predictor_update_fn(x, vec_t, model=model, maskX=mask) + return inverse_scaler(x_mean if denoise else x), sde.N * (n_steps + 1), None + + return pc_sampler + + +def get_pc_conditional_sampler_meta_nas( + sde, + shape, + predictor, + corrector, + inverse_scaler, + snr, + n_steps=1, + probability_flow=False, + continuous=False, + denoise=True, + eps=1e-5, + device='cuda', + regress=True, + labels='max', + data_name='cifar10', + num_sample=20): + + """Class-conditional sampling with Predictor-Corrector (PC) samplers. + + Args: + sde: An `sde_lib.SDE` object that represents the forward SDE. + score_model: A `torch.nn.Module` object that represents the architecture of the score-based model. + classifier: A `torch.nn.Module` object that represents the architecture of the noise-dependent classifier. + # classifier_params: A dictionary that contains the weights of the classifier. + shape: A sequence of integers. The expected shape of a single sample. + predictor: A subclass of `sampling.predictor` that represents a predictor algorithm. + corrector: A subclass of `sampling.corrector` that represents a corrector algorithm. + inverse_scaler: The inverse data normalizer. + snr: A `float` number. The signal-to-noise ratio for correctors. + n_steps: An integer. The number of corrector steps per update of the predictor. + probability_flow: If `True`, solve the probability flow ODE for sampling with the predictor. + continuous: `True` indicates the score-based model was trained with continuous time. + denoise: If `True`, add one-step denoising to final samples. + eps: A `float` number. The SDE/ODE will be integrated to `eps` to avoid numerical issues. + + Returns: A pmapped class-conditional image sampler. + """ + + # --------- Meta-NAS ---------- # + test_dataset = MetaTestDataset( + data_path=DATA_PATH, + data_name=data_name, + num_sample=num_sample) + + + def conditional_predictor_update_fn(score_model, classifier, x, t, labels, maskX, classifier_scale, *args, **kwargs): + """The predictor update function for class-conditional sampling.""" + score_fn = mutils.get_score_fn(sde, score_model, train=False, continuous=continuous) + classifier_grad_fn = mutils.get_classifier_grad_fn(sde, classifier, train=False, continuous=continuous, + regress=regress, labels=labels) + + def total_grad_fn(x, t, *args, **kwargs): + score = score_fn(x, t, maskX) + classifier_grad = classifier_grad_fn(x, t, maskX, *args, **kwargs) + return score + classifier_scale * classifier_grad + + if predictor is None: + predictor_obj = NonePredictor(sde, total_grad_fn, probability_flow) + else: + predictor_obj = predictor(sde, total_grad_fn, probability_flow) + + return predictor_obj.update_fn(x, t, *args, **kwargs) + + + def conditional_corrector_update_fn(score_model, classifier, x, t, labels, maskX, classifier_scale, *args, **kwargs): + """The corrector update function for class-conditional sampling.""" + score_fn = mutils.get_score_fn(sde, score_model, train=False, continuous=continuous) + classifier_grad_fn = mutils.get_classifier_grad_fn(sde, classifier, train=False, continuous=continuous, + regress=regress, labels=labels) + + def total_grad_fn(x, t, *args, **kwargs): + score = score_fn(x, t, maskX) + classifier_grad = classifier_grad_fn(x, t, maskX, *args, **kwargs) + return score + classifier_scale * classifier_grad + + if corrector is None: + corrector_obj = NoneCorrector(sde, total_grad_fn, snr, n_steps) + else: + corrector_obj = corrector(sde, total_grad_fn, snr, n_steps) + + return corrector_obj.update_fn(x, t, *args, **kwargs) + + + def pc_conditional_sampler( + score_model, + mask, + classifier, + classifier_scale=None, + task=None): + + """Generate class-conditional samples with Predictor-Corrector (PC) samplers. + + Args: + score_model: A `torch.nn.Module` object that represents the training state + of the score-based model. + labels: A JAX array of integers that represent the target label of each sample. + + Returns: + Class-conditional samples. + """ + + # to accerlerating sampling + with torch.no_grad(): + if task is None: + task = test_dataset[0] + task = task.repeat(shape[0], 1, 1) + task = task.to(device) + else: + task = task.repeat(shape[0], 1, 1) + task = task.to(device) + classifier.sample_state = True + classifier.D_mu = None + + # initial sample + x = sde.prior_sampling(shape).to(device) + timesteps = torch.linspace(sde.T, eps, sde.N, device=device) + + if len(mask.shape) == 3: mask = mask[0] + mask = mask.unsqueeze(0).repeat(x.size(0), 1, 1) # adj + + for i in trange(sde.N, desc='[PC conditional sampling]', position=1, leave=False): + t = timesteps[i] + vec_t = torch.ones(shape[0], device=t.device) * t + x, x_mean = conditional_corrector_update_fn(score_model, classifier, x, vec_t, labels=labels, maskX=mask, task=task, classifier_scale=classifier_scale) + x, x_mean = conditional_predictor_update_fn(score_model, classifier, x, vec_t, labels=labels, maskX=mask, task=task, classifier_scale=classifier_scale) + classifier.sample_state = False + return inverse_scaler(x_mean if denoise else x) + + return pc_conditional_sampler \ No newline at end of file diff --git a/NAS-Bench-201/script/download_preprocessed_dataset.sh b/NAS-Bench-201/script/download_preprocessed_dataset.sh new file mode 100644 index 0000000..23c16e2 --- /dev/null +++ b/NAS-Bench-201/script/download_preprocessed_dataset.sh @@ -0,0 +1,4 @@ +export LD_LIBRARY_PATH=/opt/conda/envs/gtctnz_2/lib/python3.7/site-packages/nvidia/cublas/lib/ + +echo '[Downloading processed]' +python main_exp/transfer_nag/get_files/get_preprocessed_data.py diff --git a/NAS-Bench-201/script/download_raw_dataset.sh b/NAS-Bench-201/script/download_raw_dataset.sh new file mode 100644 index 0000000..a04447e --- /dev/null +++ b/NAS-Bench-201/script/download_raw_dataset.sh @@ -0,0 +1,15 @@ +export LD_LIBRARY_PATH=/opt/conda/envs/gtctnz_2/lib/python3.7/site-packages/nvidia/cublas/lib/ + +DATANAME=$1 + +if [[ $DATANAME = 'aircraft' ]]; then + echo '[Downloading aircraft]' + python main_exp/transfer_nag/get_files/get_aircraft.py + +elif [[ $DATANAME = 'pets' ]]; then + echo '[Downloading pets]' + python main_exp/transfer_nag/get_files/get_pets.py + +else + echo 'Not Implemeted' +fi \ No newline at end of file diff --git a/NAS-Bench-201/script/tr_meta_surrogate.sh b/NAS-Bench-201/script/tr_meta_surrogate.sh new file mode 100644 index 0000000..2a1756f --- /dev/null +++ b/NAS-Bench-201/script/tr_meta_surrogate.sh @@ -0,0 +1,6 @@ +FOLDER_NAME='tr_meta_surrogate_nb201' + +CUDA_VISIBLE_DEVICES=$1 python main.py --config configs/tr_meta_surrogate.py \ + --mode train \ + --config.folder_name $FOLDER_NAME + diff --git a/NAS-Bench-201/script/tr_scorenet.sh b/NAS-Bench-201/script/tr_scorenet.sh new file mode 100644 index 0000000..bad2e73 --- /dev/null +++ b/NAS-Bench-201/script/tr_scorenet.sh @@ -0,0 +1,5 @@ +FOLDER_NAME='tr_scorenet_nb201' + +CUDA_VISIBLE_DEVICES=$1 python main.py --config configs/tr_scorenet.py \ + --mode train \ + --config.folder_name $FOLDER_NAME diff --git a/NAS-Bench-201/script/transfer_nag.sh b/NAS-Bench-201/script/transfer_nag.sh new file mode 100644 index 0000000..0c67f26 --- /dev/null +++ b/NAS-Bench-201/script/transfer_nag.sh @@ -0,0 +1,10 @@ +FOLDER_NAME='transfer_nag_nb201' + +GPU=$1 +DATANAME=$2 + +CUDA_VISIBLE_DEVICES=$GPU python main_exp/transfer_nag/main.py \ + --gpu $GPU \ + --test \ + --folder_name $FOLDER_NAME \ + --data-name $DATANAME diff --git a/NAS-Bench-201/sde_lib.py b/NAS-Bench-201/sde_lib.py new file mode 100644 index 0000000..54994b5 --- /dev/null +++ b/NAS-Bench-201/sde_lib.py @@ -0,0 +1,300 @@ +"""Abstract SDE classes, Reverse SDE, and VP SDEs.""" + +import abc +import torch +import numpy as np + + +class SDE(abc.ABC): + """SDE abstract class. Functions are designed for a mini-batch of inputs.""" + + def __init__(self, N): + """Construct an SDE. + + Args: + N: number of discretization time steps. + """ + super().__init__() + self.N = N + + @property + @abc.abstractmethod + def T(self): + """End time of the SDE.""" + pass + + @abc.abstractmethod + def sde(self, x, t): + pass + + @abc.abstractmethod + def marginal_prob(self, x, t): + """Parameters to determine the marginal distribution of the SDE, $p_t(x)$""" + pass + + @abc.abstractmethod + def prior_sampling(self, shape): + """Generate one sample from the prior distribution, $p_T(x)$.""" + pass + + @abc.abstractmethod + def prior_logp(self, z, mask): + """Compute log-density of the prior distribution. + + Useful for computing the log-likelihood via probability flow ODE. + + Args: + z: latent code + Returns: + log probability density + """ + pass + + def discretize(self, x, t): + """Discretize the SDE in the form: x_{i+1} = x_i + f_i(x_i) + G_i z_i. + + Useful for reverse diffusion sampling and probability flow sampling. + Defaults to Euler-Maruyama discretization. + + Args: + x: a torch tensor + t: a torch float representing the time step (from 0 to `self.T`) + + Returns: + f, G + """ + dt = 1 / self.N + drift, diffusion = self.sde(x, t) + f = drift * dt + G = diffusion * torch.sqrt(torch.tensor(dt, device=t.device)) + return f, G + + def reverse(self, score_fn, probability_flow=False): + """Create the reverse-time SDE/ODE. + + Args: + score_fn: A time-dependent score-based model that takes x and t and returns the score. + probability_flow: If `True`, create the reverse-time ODE used for probability flow sampling. + """ + + N = self.N + T = self.T + sde_fn = self.sde + discretize_fn = self.discretize + + # Build the class for reverse-time SDE. + class RSDE(self.__class__): + def __init__(self): + self.N = N + self.probability_flow = probability_flow + + @property + def T(self): + return T + + def sde(self, x, t, *args, **kwargs): + """Create the drift and diffusion functions for the reverse SDE/ODE.""" + + drift, diffusion = sde_fn(x, t) + score = score_fn(x, t, *args, **kwargs) + drift = drift - diffusion[:, None, None] ** 2 * score * (0.5 if self.probability_flow else 1.) + # Set the diffusion function to zero for ODEs. + diffusion = 0. if self.probability_flow else diffusion + return drift, diffusion + + ''' + def sde_score(self, x, t, score): + """Create the drift and diffusion functions for the reverse SDE/ODE, given score values.""" + drift, diffusion = sde_fn(x, t) + if len(score.shape) == 4: + drift = drift - diffusion[:, None, None, None] ** 2 * score * (0.5 if self.probability_flow else 1.) + elif len(score.shape) == 3: + drift = drift - diffusion[:, None, None] ** 2 * score * (0.5 if self.probability_flow else 1.) + else: + raise ValueError + diffusion = 0. if self.probability_flow else diffusion + return drift, diffusion + ''' + + def discretize(self, x, t, *args, **kwargs): + """Create discretized iteration rules for the reverse diffusion sampler.""" + f, G = discretize_fn(x, t) + rev_f = f - G[:, None, None] ** 2 * score_fn(x, t, *args, **kwargs) * \ + (0.5 if self.probability_flow else 1.) + rev_G = torch.zeros_like(G) if self.probability_flow else G + return rev_f, rev_G + + ''' + def discretize_score(self, x, t, score): + """Create discretized iteration rules for the reverse diffusion sampler, given score values.""" + f, G = discretize_fn(x, t) + if len(score.shape) == 4: + rev_f = f - G[:, None, None, None] ** 2 * score * \ + (0.5 if self.probability_flow else 1.) + elif len(score.shape) == 3: + rev_f = f - G[:, None, None] ** 2 * score * (0.5 if self.probability_flow else 1.) + else: + raise ValueError + rev_G = torch.zeros_like(G) if self.probability_flow else G + return rev_f, rev_G + ''' + + return RSDE() + + +class VPSDE(SDE): + def __init__(self, beta_min=0.1, beta_max=20, N=1000): + """Construct a Variance Preserving SDE. + + Args: + beta_min: value of beta(0) + beta_max: value of beta(1) + N: number of discretization steps + """ + super().__init__(N) + self.beta_0 = beta_min + self.beta_1 = beta_max + self.N = N + self.discrete_betas = torch.linspace(beta_min / N, beta_max / N, N) + self.alphas = 1. - self.discrete_betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + self.sqrt_alphas_cumprod = torch.sqrt(self.alphas_cumprod) + self.sqrt_1m_alphas_cumprod = torch.sqrt(1. - self.alphas_cumprod) + + @property + def T(self): + return 1 + + def sde(self, x, t): + beta_t = self.beta_0 + t * (self.beta_1 - self.beta_0) + if len(x.shape) == 4: + drift = -0.5 * beta_t[:, None, None, None] * x + elif len(x.shape) == 3: + drift = -0.5 * beta_t[:, None, None] * x + else: + raise NotImplementedError + diffusion = torch.sqrt(beta_t) + return drift, diffusion + + def marginal_prob(self, x, t): + log_mean_coeff = -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0 + if len(x.shape) == 4: + mean = torch.exp(log_mean_coeff[:, None, None, None]) * x + elif len(x.shape) == 3: + mean = torch.exp(log_mean_coeff[:, None, None]) * x + else: + raise ValueError("The shape of x in marginal_prob is not correct.") + std = torch.sqrt(1. - torch.exp(2. * log_mean_coeff)) + return mean, std + + def prior_sampling(self, shape): + return torch.randn(*shape) + + def prior_logp(self, z, mask): + N = torch.sum(mask, dim=tuple(range(1, len(mask.shape)))) + logps = -N / 2. * np.log(2 * np.pi) - torch.sum((z * mask) ** 2, dim=(1, 2, 3)) / 2. + return logps + + def discretize(self, x, t): + """DDPM discretization.""" + timestep = (t * (self.N - 1) / self.T).long() + beta = self.discrete_betas.to(x.device)[timestep] + alpha = self.alphas.to(x.device)[timestep] + sqrt_beta = torch.sqrt(beta) + if len(x.shape) == 4: + f = torch.sqrt(alpha)[:, None, None, None] * x - x + elif len(x.shape) == 3: + f = torch.sqrt(alpha)[:, None, None] * x - x + else: + NotImplementedError + G = sqrt_beta + return f, G + + +class subVPSDE(SDE): + def __init__(self, beta_min=0.1, beta_max=20, N=1000): + """Construct the sub-VP SDE that excels at likelihoods. + Args: + beta_min: value of beta(0) + beta_max: value of beta(1) + N: number of discretization steps + """ + super().__init__(N) + self.beta_0 = beta_min + self.beta_1 = beta_max + self.N = N + + @property + def T(self): + return 1 + + def sde(self, x, t): + beta_t = self.beta_0 + t * (self.beta_1 - self.beta_0) + drift = -0.5 * beta_t[:, None, None] * x + discount = 1. - torch.exp(-2 * self.beta_0 * t - (self.beta_1 - self.beta_0) * t ** 2) + diffusion = torch.sqrt(beta_t * discount) + return drift, diffusion + + def marginal_prob(self, x, t): + log_mean_coeff = -0.25 * t ** 2 * (self.beta_1 - self.beta_0) - 0.5 * t * self.beta_0 + mean = torch.exp(log_mean_coeff)[:, None, None] * x + std = 1 - torch.exp(2. * log_mean_coeff) + return mean, std + + def prior_sampling(self, shape): + return torch.randn(*shape) + + def prior_logp(self, z): + shape = z.shape + N = np.prod(shape[1:]) + return -N / 2. * np.log(2 * np.pi) - torch.sum(z ** 2, dim=(1, 2, 3)) / 2. + + +class VESDE(SDE): + def __init__(self, sigma_min=0.01, sigma_max=50, N=1000): + """Construct a Variance Exploding SDE. + + Args: + sigma_min: smallest sigma. + sigma_max: largest sigma. + N: number of discretization steps + """ + super().__init__(N) + self.sigma_min = sigma_min + self.sigma_max = sigma_max + self.discrete_sigmas = torch.exp(torch.linspace(np.log(self.sigma_min), np.log(self.sigma_max), N)) + self.N = N + + @property + def T(self): + return 1 + + def sde(self, x, t): + sigma = self.sigma_min * (self.sigma_max / self.sigma_min) ** t + drift = torch.zeros_like(x) + diffusion = sigma * torch.sqrt(torch.tensor(2 * (np.log(self.sigma_max) - np.log(self.sigma_min)), + device=t.device)) + return drift, diffusion + + def marginal_prob(self, x, t): + std = self.sigma_min * (self.sigma_max / self.sigma_min) ** t + mean = x + return mean, std + + def prior_sampling(self, shape): + return torch.randn(*shape) * self.sigma_max + + def prior_logp(self, z): + shape = z.shape + N = np.prod(shape[1:]) + return -N / 2. * np.log(2 * np.pi * self.sigma_max ** 2) - torch.sum(z ** 2, dim=(1, 2, 3)) / (2 * self.sigma_max ** 2) + + def discretize(self, x, t): + """SMLD(NCSN) discretization.""" + timestep = (t * (self.N - 1) / self.T).long() + sigma = self.discrete_sigmas.to(t.device)[timestep] + adjacent_sigma = torch.where(timestep == 0, torch.zeros_like(t), + self.discrete_sigmas[timestep.cpu() - 1].to(t.device)) + f = torch.zeros_like(x) + G = torch.sqrt(sigma ** 2 - adjacent_sigma ** 2) + return f, G \ No newline at end of file diff --git a/NAS-Bench-201/utils.py b/NAS-Bench-201/utils.py new file mode 100644 index 0000000..84f26da --- /dev/null +++ b/NAS-Bench-201/utils.py @@ -0,0 +1,262 @@ +import os +import logging +import torch +from torch_scatter import scatter +import shutil + + +@torch.no_grad() +def to_dense_adj(edge_index, batch=None, edge_attr=None, max_num_nodes=None): + """Converts batched sparse adjacency matrices given by edge indices and + edge attributes to a single dense batched adjacency matrix. + + Args: + edge_index (LongTensor): The edge indices. + batch (LongTensor, optional): Batch vector + :math:`\mathbf{b} \in {\{ 0, \ldots, B-1\}}^N`, which assigns each + node to a specific example. (default: :obj:`None`) + edge_attr (Tensor, optional): Edge weights or multi-dimensional edge + features. (default: :obj:`None`) + max_num_nodes (int, optional): The size of the output node dimension. + (default: :obj:`None`) + + Returns: + adj: [batch_size, max_num_nodes, max_num_nodes] Dense adjacency matrices. + mask: Mask for dense adjacency matrices. + """ + if batch is None: + batch = edge_index.new_zeros(edge_index.max().item() + 1) + + batch_size = batch.max().item() + 1 + one = batch.new_ones(batch.size(0)) + num_nodes = scatter(one, batch, dim=0, dim_size=batch_size, reduce='add') + cum_nodes = torch.cat([batch.new_zeros(1), num_nodes.cumsum(dim=0)]) + + idx0 = batch[edge_index[0]] + idx1 = edge_index[0] - cum_nodes[batch][edge_index[0]] + idx2 = edge_index[1] - cum_nodes[batch][edge_index[1]] + + if max_num_nodes is None: + max_num_nodes = num_nodes.max().item() + + elif idx1.max() >= max_num_nodes or idx2.max() >= max_num_nodes: + mask = (idx1 < max_num_nodes) & (idx2 < max_num_nodes) + idx0 = idx0[mask] + idx1 = idx1[mask] + idx2 = idx2[mask] + edge_attr = None if edge_attr is None else edge_attr[mask] + + if edge_attr is None: + edge_attr = torch.ones(idx0.numel(), device=edge_index.device) + + size = [batch_size, max_num_nodes, max_num_nodes] + size += list(edge_attr.size())[1:] + adj = torch.zeros(size, dtype=edge_attr.dtype, device=edge_index.device) + + flattened_size = batch_size * max_num_nodes * max_num_nodes + adj = adj.view([flattened_size] + list(adj.size())[3:]) + idx = idx0 * max_num_nodes * max_num_nodes + idx1 * max_num_nodes + idx2 + scatter(edge_attr, idx, dim=0, out=adj, reduce='add') + adj = adj.view(size) + + node_idx = torch.arange(batch.size(0), dtype=torch.long, device=edge_index.device) + node_idx = (node_idx - cum_nodes[batch]) + (batch * max_num_nodes) + mask = torch.zeros(batch_size * max_num_nodes, dtype=adj.dtype, device=adj.device) + mask[node_idx] = 1 + mask = mask.view(batch_size, max_num_nodes) + + mask = mask[:, None, :] * mask[:, :, None] + + return adj, mask + + +def restore_checkpoint_partial(model, pretrained_stdict): + model_dict = model.state_dict() + # 1. filter out unnecessary keys + pretrained_dict = {k: v for k, v in pretrained_stdict.items() if k in model_dict} + # 2. overwrite entries in the existing state dict + model_dict.update(pretrained_dict) + # 3. load the new state dict + model.load_state_dict(model_dict) + return model + + +def restore_checkpoint(ckpt_dir, state, device, resume=False): + if not resume: + os.makedirs(os.path.dirname(ckpt_dir), exist_ok=True) + return state + elif not os.path.exists(ckpt_dir): + if not os.path.exists(os.path.dirname(ckpt_dir)): + os.makedirs(os.path.dirname(ckpt_dir)) + logging.warning(f"No checkpoint found at {ckpt_dir}. " + f"Returned the same state as input") + return state + else: + loaded_state = torch.load(ckpt_dir, map_location=device) + for k in state: + if k in ['optimizer', 'model', 'ema']: + state[k].load_state_dict(loaded_state[k]) + else: + state[k] = loaded_state[k] + return state + + +def save_checkpoint(ckpt_dir, state, step, save_step, is_best, remove_except_best=False): + saved_state = {} + for k in state: + if k in ['optimizer', 'model', 'ema']: + saved_state.update({k: state[k].state_dict()}) + else: + saved_state.update({k: state[k]}) + os.makedirs(ckpt_dir, exist_ok=True) + torch.save(saved_state, os.path.join(ckpt_dir, f'checkpoint_{step}_{save_step}.pth.tar')) + if is_best: + shutil.copy(os.path.join(ckpt_dir, f'checkpoint_{step}_{save_step}.pth.tar'), os.path.join(ckpt_dir, 'model_best.pth.tar')) + # remove the ckpt except is_best state + if remove_except_best: + for ckpt_file in sorted(os.listdir(ckpt_dir)): + if not ckpt_file.startswith('checkpoint'): + continue + if os.path.join(ckpt_dir, ckpt_file) != os.path.join(ckpt_dir, 'model_best.pth.tar'): + os.remove(os.path.join(ckpt_dir, ckpt_file)) + + +def floyed(r): + """ + :param r: a numpy NxN matrix with float 0,1 + :return: a numpy NxN matrix with float 0,1 + """ + # r = np.array(r) + if type(r) == torch.Tensor: + r = r.cpu().numpy() + N = r.shape[0] + # import pdb; pdb.set_trace() + for k in range(N): + for i in range(N): + for j in range(N): + if r[i, k] > 0 and r[k, j] > 0: + r[i, j] = 1 + return r + + +def aug_mask(adj, algo='floyed', data='NASBench201'): + if len(adj.shape) == 2: + adj = adj.unsqueeze(0) + + if data.lower() in ['nasbench201', 'ofa']: + assert len(adj.shape) == 3 + r = adj[0].clone().detach() + if algo == 'long_range': + mask_i = torch.from_numpy(long_range(r)).float().to(adj.device) + elif algo == 'floyed': + mask_i = torch.from_numpy(floyed(r)).float().to(adj.device) + else: + mask_i = r + masks = [mask_i] * adj.size(0) + return torch.stack(masks) + else: + masks = [] + for r in adj: + if algo == 'long_range': + mask_i = torch.from_numpy(long_range(r)).float().to(adj.device) + elif algo == 'floyed': + mask_i = torch.from_numpy(floyed(r)).float().to(adj.device) + else: + mask_i = r + masks.append(mask_i) + return torch.stack(masks) + + +def long_range(r): + """ + :param r: a numpy NxN matrix with float 0,1 + :return: a numpy NxN matrix with float 0,1 + """ + # r = np.array(r) + if type(r) == torch.Tensor: + r = r.cpu().numpy() + N = r.shape[0] + for j in range(1, N): + col_j = r[:, j][:j] + in_to_j = [i for i, val in enumerate(col_j) if val > 0] + if len(in_to_j) > 0: + for i in in_to_j: + col_i = r[:, i][:i] + in_to_i = [i for i, val in enumerate(col_i) if val > 0] + if len(in_to_i) > 0: + for k in in_to_i: + r[k, j] = 1 + return r + + +def dense_adj(graph_data, max_num_nodes, scaler=None, dequantization=False): + """Convert PyG DataBatch to dense adjacency matrices. + + Args: + graph_data: DataBatch object. + max_num_nodes: The size of the output node dimension. + scaler: Data normalizer. + dequantization: uniform dequantization. + + Returns: + adj: Dense adjacency matrices. + mask: Mask for adjacency matrices. + """ + + adj, adj_mask = to_dense_adj(graph_data.edge_index, graph_data.batch, max_num_nodes=max_num_nodes) # [B, N, N] + # adj: [32, 20, 20] / adj_mask: [32, 20, 20] + if dequantization: + noise = torch.rand_like(adj) + noise = torch.tril(noise, -1) + noise = noise + noise.transpose(1, 2) + adj = (noise + adj) / 2. + adj = scaler(adj[:, None, :, :]) # [32, 1, 20, 20] + # set diag = 0 in adj_mask + adj_mask = torch.tril(adj_mask, -1) # [32, 20, 20] + adj_mask = adj_mask + adj_mask.transpose(1, 2) + + return adj, adj_mask[:, None, :, :] + + +def adj2graph(adj, sample_nodes): + """Covert the PyTorch tensor adjacency matrices to numpy array. + + Args: + adj: [Batch_size, channel, Max_node, Max_node], assume channel=1 + sample_nodes: [Batch_size] + """ + adj_list = [] + # discretization + adj[adj >= 0.5] = 1. + adj[adj < 0.5] = 0. + for i in range(adj.shape[0]): + adj_tmp = adj[i, 0] + # symmetric + adj_tmp = torch.tril(adj_tmp, -1) + adj_tmp = adj_tmp + adj_tmp.transpose(0, 1) + # truncate + adj_tmp = adj_tmp.cpu().numpy()[:sample_nodes[i], :sample_nodes[i]] + adj_list.append(adj_tmp) + + return adj_list + + +def quantize(x): + """Covert the PyTorch tensor x, adj matrices to numpy array. + + Args: + x: [Batch_size, Max_node, N_vocab] + adj: [Batch_size, Max_node, Max_node] + """ + x_list = [] + + # discretization + x[x >= 0.5] = 1. + x[x < 0.5] = 0. + + for i in range(x.shape[0]): + x_tmp = x[i] + x_tmp = x_tmp.cpu().numpy() + x_list.append(x_tmp) + + return x_list \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..54b3796 --- /dev/null +++ b/README.md @@ -0,0 +1,144 @@ +# DiffusionNAG: Predictor-guided Neural Architecture Generation with Diffusion Models + +Official Code Repository for the paper [DiffusionNAG: Predictor-guided Neural Architecture Generation with Diffusion Models](https://arxiv.org/abs/2305.16943). + + + + +## Why DiffusionNAG? + ++ Existing NAS approaches still result in large waste of time as they need to explore an extensive search space and the property predictors mostly play a passive role such as the evaluators that rank architecture candidates provided by a search strategy to simply filter them out during the search process. + ++ We introduce a novel predictor-guided **Diffusion**-based **N**eural **A**rchitecture **G**enerative framework called **DiffusionNAG**, which explicitly incorporates the predictors into generating architectures that satisfy the objectives. + ++ DiffusionNAG offers several advantages compared with conventional NAS methods, including efficient and effective search, superior utilization of predictors for both NAG and evaluation purposes, and easy adaptability across diverse tasks. + + +## Environment Setup + +Create environment with **Python 3.7.2** and **Pytorch 1.13.1**. +Use the following command to install the requirements: + +``` +cd setup +conda create -n diffusionnag python==3.7.2 +conda activate diffusionnag +bash install.sh +``` + +## Run - NAS-Bench-201 +### Download datasets, preprocessed datasets, and checkpoints +``` +cd NAS-Bench-201 +``` + +If you want to run experiments on datasets that are not included in the benchmark, such as "aircraft" or "pets", you will need to download the raw dataset for actually trainining neural architectures on it: +``` +## Download the raw dataset +bash script/download_raw_dataset.sh [DATASET] +``` + +We use dataset encoding when training the meta-predictor or performing conditional sampling with the target dataset. To obtain dataset encoding, you need to download the preprocessed dataset: +``` +## Download the preprocessed dataset +bash script/download_preprocessed_dataset.sh +``` + +If you want to use the pre-trained score network or meta-predictor, download the checkpoints from the following links. + +Download the pre-trained score network and move the checkpoint to ```checkpoints/scorenet``` directory: ++ https://drive.google.com/file/d/1-GnItyf03-2r_KbYV3PCHS1FNVFXlNR3/view?usp=sharing + +Download the pre-trained meta-predictor and move the checkpoint to ```checkpoints/meta_surrogate``` directory: ++ https://drive.google.com/file/d/1oFXSLBPvorO_Ar-1JQQB49x7L1BX79gd/view?usp=sharing ++ https://drive.google.com/file/d/1S2IV6L9t6Hlhh6vGsQkyqMJGt5NnJ8pj/view?usp=sharing + +### Transfer NAG +``` +bash script/transfer_nag.sh [GPU] [DATASET] +## Examples +bash script/transfer_nag.sh 0 cifar10 +bash script/transfer_nag.sh 0 cifar100 +bash script/transfer_nag.sh 0 aircraft +bash script/transfer_nag.sh 0 pets +``` + +### Train score network +``` +bash script/tr_scorenet.sh [GPU] +``` + +### Train meta-predictor +``` +bash script/tr_meta_surrogate.sh [GPU] +``` + + +## Run - MobileNetV3 +### Download datasets, preprocessed datasets, and checkpoints +``` +cd MobileNetV3 +``` + +If you want to run experiments on datasets that are not included in the benchmark, such as "aircraft" or "pets", you will need to download the raw dataset for actually trainining neural architectures on it: +``` +## Download the raw dataset +bash script/download_raw_dataset.sh [DATASET] +``` + +We use dataset encoding when training the meta-predictor or performing conditional sampling with the target dataset. To obtain dataset encoding, you need to download the preprocessed dataset: +``` +## Download the preprocessed dataset +bash script/download_preprocessed_dataset.sh +``` + +If you want to use the pre-trained score network or meta-predictor, download the checkpoints from the following links. + +Download the pre-trained score network and move the checkpoint to ```checkpoints/ofa/score_model``` directory: ++ https://www.dropbox.com/scl/fi/r47svpl1tvpm9tos3vtsd/model_best.pth.tar?rlkey=5wpa6zh8cpp4gctuol25wxj0u&dl=0 + +Download the first pre-trained meta-predictor and move the checkpoint to ```checkpoints/ofa/noise_aware_meta_surrogate``` directory: ++ https://www.dropbox.com/scl/fi/k896bi61pu0rq87p5argx/model_best.pth.tar?rlkey=qo4ga96c5a3fu4228nnvift6v&dl=0 + +Download the second pre-trained meta-predictor and move the checkpoint to ```checkpoints/ofa/unnoised_meta_surrogate_from_metad2a``` directory: ++ https://www.dropbox.com/scl/fi/zfdis3njlfa1g5nsje3h8/ckpt_max_corr.pt?rlkey=1vplo2oiilljv6991ub0a50sb&dl=0 + +Download the config file for TransferNAG experiments and move the checkpoint to ```configs``` directory: ++ https://www.dropbox.com/scl/fi/psv7lh4bijwapj5jkgaq3/transfer_nag_ofa.pt?rlkey=wi15mjvme2pmep7p12auvm1ie&dl=0 + +### Transfer NAG +``` +bash script/transfer_nag.sh [GPU] [DATASET] +## Examples +bash script/transfer_nag.sh 0,1 cifar10 +bash script/transfer_nag.sh 0,1 cifar100 +bash script/transfer_nag.sh 0,1 aircraft +bash script/transfer_nag.sh 0,1 pets +``` + +### Train score network +``` +bash script/tr_scorenet_ofa.sh [GPU] +``` + +### Train meta-predictor +``` +bash script/tr_meta_surrogate_ofa.sh [GPU] +``` + + + +## Citation + +If you have found our work helpful for your research, we would appreciate it if you could acknowledge it by citing our work. + +```BibTex +@inproceedings{ +an2024diffusionnag, +title={Diffusion{NAG}: Predictor-guided Neural Architecture Generation with Diffusion Models}, +author={Sohyun An and Hayeon Lee and Jaehyeong Jo and Seanie Lee and Sung Ju Hwang}, +booktitle={The Twelfth International Conference on Learning Representations}, +year={2024}, +url={https://openreview.net/forum?id=dyG2oLJYyX} +} +``` diff --git a/assets/DiffusionNAG-illustration.png b/assets/DiffusionNAG-illustration.png new file mode 100644 index 0000000..7ba19bb Binary files /dev/null and b/assets/DiffusionNAG-illustration.png differ diff --git a/setup/diffusionnag_env.yaml b/setup/diffusionnag_env.yaml new file mode 100644 index 0000000..0f4dac1 --- /dev/null +++ b/setup/diffusionnag_env.yaml @@ -0,0 +1,120 @@ +name: diffusionnag +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - absl-py=1.4.0=pyhd8ed1ab_0 + - c-ares=1.18.1=h7f98852_0 + - ca-certificates=2022.12.7=ha878542_0 + - certifi=2022.12.7=pyhd8ed1ab_0 + - freetype=2.10.4=h0708190_1 + - giflib=5.2.1=h36c2ea0_2 + - grpcio=1.16.0=py37h4f00d22_1000 + - imageio=2.27.0=pyh24c5eb1_0 + - jpeg=9e=h166bdaf_1 + - lcms2=2.12=h3be6417_0 + - lerc=3.0=h295c915_0 + - libblas=3.9.0=15_linux64_openblas + - libcblas=3.9.0=15_linux64_openblas + - libdeflate=1.8=h7f8727e_5 + - libedit=3.1.20221030=h5eee18b_0 + - libffi=3.2.1=hf484d3e_1007 + - libgcc-ng=11.2.0=h1234567_1 + - libgfortran-ng=12.2.0=h69a702a_19 + - libgfortran5=12.2.0=h337968e_19 + - libgomp=11.2.0=h1234567_1 + - liblapack=3.9.0=15_linux64_openblas + - libopenblas=0.3.20=pthreads_h78a6416_0 + - libpng=1.6.39=h5eee18b_0 + - libprotobuf=3.18.0=h780b84a_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - libtiff=4.5.0=hecacb30_0 + - libwebp=1.2.4=h11a3e52_1 + - libwebp-base=1.2.4=h5eee18b_1 + - lz4-c=1.9.3=h9c3ff4c_1 + - markdown=3.4.3=pyhd8ed1ab_0 + - markupsafe=2.1.1=py37h540881e_1 + - ncurses=6.3=h5eee18b_3 + - openssl=1.0.2u=h516909a_0 + - pip=22.3.1=py37h06a4308_0 + - readline=7.0=h7b6447c_5 + - setuptools=65.6.3=py37h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.33.0=h62c20be_0 + - tensorboard=1.15.0=py37_0 + - tk=8.6.12=h1ccaba5_0 + - typing_extensions=4.5.0=pyha770c72_0 + - werkzeug=2.2.3=pyhd8ed1ab_0 + - wheel=0.37.1=pyhd3eb1b0_0 + - xz=5.2.8=h5eee18b_0 + - zlib=1.2.13=h5eee18b_0 + - zstd=1.5.2=ha4553b6_0 + - pip: + - appdirs==1.4.4 + - blessed==1.19.1 + - charset-normalizer==3.0.1 + - click==8.1.3 + - cycler==0.10.0 + - docker-pycreds==0.4.0 + - easydict==1.9 + - einops==0.6.0 + - fcd-torch==1.0.7 + - flatbuffers==23.1.4 + - gitdb==4.0.10 + - gitpython==3.1.30 + - gpustat==1.0.0 + - gpytorch==1.8.1 + - idna==3.4 + - igraph==0.10.3 + - importlib-metadata==6.0.0 + - joblib==1.2.0 + - kiwisolver==1.3.2 + - libclang==15.0.6.1 + - matplotlib==3.4.2 + - molsets==0.3.1 + - networkx==2.6.3 + - numpy==1.20.3 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-ml-py==11.495.46 + - pandas==1.1.5 + - pathtools==0.1.2 + - pillow==9.4.0 + - pomegranate==0.12.0 + - protobuf==3.19.6 + - psutil==5.9.4 + - pyasn1==0.4.8 + - pyasn1-modules==0.2.8 + - pyemd==0.5.1 + - pyparsing==2.4.7 + - python-dateutil==2.8.2 + - pytz==2022.7.1 + - pyyaml==6.0 + - requests==2.28.2 + - rsa==4.9 + - scikit-learn==1.0.2 + - scipy==1.7.1 + - seaborn==0.12.2 + - sentry-sdk==1.13.0 + - setproctitle==1.3.2 + - sklearn==0.0.post1 + - smmap==5.0.0 + - tensorboard-data-server==0.6.1 + - tensorboard-plugin-wit==1.8.1 + - tensorflow-estimator==2.11.0 + - tensorflow-io-gcs-filesystem==0.29.0 + - termcolor==2.2.0 + - texttable==1.6.7 + - threadpoolctl==3.1.0 + - torch==1.13.1 + - torchdiffeq==0.2.3 + - tqdm==4.64.1 + - urllib3==1.26.14 + - wandb==0.13.9 + - wcwidth==0.2.6 + - wrapt==1.14.1 + - zipp==3.11.0 +prefix: /opt/conda/envs/diffuisonnag diff --git a/setup/install.sh b/setup/install.sh new file mode 100644 index 0000000..81b50ee --- /dev/null +++ b/setup/install.sh @@ -0,0 +1,21 @@ +conda env update --file diffusionnag_env.yaml +pip install nas-bench-201==1.3 +pip install pyg_lib torch_scatter torch_sparse torch_cluster torch_spline_conv -f https://data.pyg.org/whl/torch-1.13.1+cu117.html +pip install torch_geometric +pip install tensorflow==1.14.0 +pip install pybnn +pip install ml_collections +pip install igraph +pip install gpytorch +pip install pandas +pip uninstall protobuf +pip install protobuf==3.19.0 +pip install torchdiffeq +pip install wandb +pip install einops +pip install networkx +pip install matplotlib +pip install timm +pip install ofa==0.0.4-2007200808 +pip install torchprofile +pip uninstall -y nvidia_cublas_cu11