From ab227586b46ad48138303a611ae445cd460f0ea9 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 02:54:36 +0000 Subject: [PATCH 01/59] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20Transformer-xl=5Ffor?= =?UTF-8?q?=5FPyTorch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/.keep diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/.keep b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/.keep new file mode 100644 index 0000000000..e69de29bb2 -- Gitee From eaf2f1f5f0cc839bd6ae100f1c4f9ddb9d4b7255 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 02:55:19 +0000 Subject: [PATCH 02/59] my first commit --- .../nlp/Transformer-xl_for_PyTorch/Dockerfile | 5 + .../nlp/Transformer-xl_for_PyTorch/Readme.md | 62 ++ .../adaptive_softmax.py | 105 +++ .../Transformer-xl_for_PyTorch/data_utils.py | 270 ++++++ .../Transformer-xl_for_PyTorch/eval_npu.py | 376 ++++++++ .../mem_transformer.py | 851 ++++++++++++++++++ .../modelzoo_level.txt | 3 + .../requirements.txt | 5 + .../train_1p_npu.py | 542 +++++++++++ .../train_8p_npu.py | 648 +++++++++++++ 10 files changed, 2867 insertions(+) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Dockerfile create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/mem_transformer.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/requirements.txt create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_1p_npu.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Dockerfile b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Dockerfile new file mode 100644 index 0000000000..360861ede1 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Dockerfile @@ -0,0 +1,5 @@ +ARG FROM_IMAGE_NAME +FROM ${FROM_IMAGE_NAME} + +COPY requirements.txt . +RUN pip3.7 install -r requirements.txt diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md new file mode 100644 index 0000000000..0412217fa7 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md @@ -0,0 +1,62 @@ +# Transformer-xl + +This implements training of transformer-xl on the enwik8 dataset, mainly modified from [pytorch/examples](https://github.com/kimiyoung/transformer-xl/tree/master/pytorch). + +## Transformer-xl Detail + +As of the current date, Ascend-Pytorch is still inefficient for contiguous operations.Therefore, Transformer-xl is re-implemented using semantics such as custom OP. + + +## Requirements + +- Install PyTorch ([pytorch.org](http://pytorch.org)) +- `pip install -r requirements.txt` + +## Data Prepration +- `bash getdata.sh` + +## Training and Evaluation + +To train a model, run `bash test/train_full_8p.sh` with the desired model architecture and the path to the enwik8 dataset: + + +```bash +#env +cd transformer-xl +dos2unix ./test/*.sh + +# 1p train perf +bash test/train_performance_1p.sh + +# 8p train perf +bash test/train_performance_8p.sh + +# 8p train full +bash test/train_full_8p.sh + +# 1p eval +bash test/eval_1p.sh + +``` + +- 参数说明： +```bash +#--data //数据集路径,可自行修改为对应路径的数据集 +#--restart_dir //加载模型checkpoint路径，可自行修改为对应路径的模型文件 +#--addr //主机地址 +#--max_step //最大训练步数 +#--batch-size //训练批次大小 +#--lr //初始学习率，默认：0.00025 +#--device-list //多卡训练指定训练用卡 ,8卡：'0,1,2,3,4,5,6,7' +#--amp //是否使用混合精度 +#--loss-scale //lossscale大小 +#--opt-level //混合精度类型 +``` + + +## Transformer-xl training result + +| bpc | FPS | Npu_nums | Epochs | AMP_Type | +| :------: | :------: | :------: | :------: | :------: | +| - | 8300 | 1 | 1 | O2 | +| 1.09 | 44500 | 8 | 50 | O2 | \ No newline at end of file diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py new file mode 100644 index 0000000000..8360772d99 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py @@ -0,0 +1,105 @@ +from collections import defaultdict + +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class AdaptiveLogSoftmax(nn.Module): + def __init__(self, in_features, n_classes, cutoffs, keep_order=False): + super(AdaptiveLogSoftmax, self).__init__() + + cutoffs = list(cutoffs) + + if (cutoffs != sorted(cutoffs)) \ + or (min(cutoffs) <= 0) \ + or (max(cutoffs) >= (n_classes - 1)) \ + or (len(set(cutoffs)) != len(cutoffs)) \ + or any([int(c) != c for c in cutoffs]): + + raise ValueError("cutoffs should be a sequence of unique, positive " + "integers sorted in an increasing order, where " + "each value is between 1 and n_classes-1") + + self.in_features = in_features + self.n_classes = n_classes + self.cutoffs = cutoffs + [n_classes] + + self.shortlist_size = self.cutoffs[0] + self.n_clusters = len(self.cutoffs) - 1 + self.head_size = self.shortlist_size + self.n_clusters + + self.cluster_weight = nn.Parameter(torch.zeros(self.n_clusters, self.in_features)) + self.cluster_bias = nn.Parameter(torch.zeros(self.n_clusters)) + + self.keep_order = keep_order + + + def forward(self, hidden, target, weight, bias, keep_order=False): + if hidden.size(0) != target.size(0): + raise RuntimeError('Input and target should have the same size ' + 'in the batch dimension.') + + head_weight = torch.cat( + [weight[:self.shortlist_size], self.cluster_weight], dim=0) + head_bias = torch.cat( + [bias[:self.shortlist_size], self.cluster_bias], dim=0) + + head_logit = F.linear(hidden, head_weight, bias=head_bias) + head_logprob = F.log_softmax(head_logit, dim=1) + + nll = torch.zeros_like(target, + dtype=hidden.dtype, device=hidden.device) + + offset = 0 + cutoff_values = [0] + self.cutoffs + for i in range(len(cutoff_values) - 1): + l_idx, h_idx = cutoff_values[i], cutoff_values[i + 1] + + mask_i = (target >= l_idx) & (target < h_idx) + indices_i = mask_i.nonzero().squeeze() + + if indices_i.numel() == 0: + continue + + target_i = target.index_select(0, indices_i) - l_idx + head_logprob_i = head_logprob.index_select(0, indices_i) + + if i == 0: + + # aa=target_i[:, None] + # aa=aa.to('cpu') + # logprob_i = head_logprob_i.gather(1, aa).squeeze(1) + print(f'target_i[:,None]: {target_i[:,None]}') + print(f'target_i[:,None].shape: {target_i[:, None].shape}') + + + logprob_i = head_logprob_i.gather(1, target_i[:,None]).squeeze(1) + else: + weight_i = weight[l_idx:h_idx] + bias_i = bias[l_idx:h_idx] + + hidden_i = hidden.index_select(0, indices_i) + + tail_logit_i = F.linear(hidden_i, weight_i, bias=bias_i) + tail_logprob_i = F.log_softmax(tail_logit_i, dim=1) + + # aa = target_i[:, None] + # aa = aa.to('cpu') + # logprob_i = head_logprob_i[:, -i] \ + # + tail_logprob_i.gather(1, aa).squeeze(1) + + print(f'target_i[:,None]: {target_i[:, None]}') + print(f'target_i[:,None].shape: {target_i[:, None].shape}') + logprob_i = head_logprob_i[:, -i] \ + + tail_logprob_i.gather(1, target_i[:,None]).squeeze(1) + + if (hasattr(self, 'keep_order') and self.keep_order) or keep_order: + nll.index_copy_(0, indices_i, -logprob_i) + else: + nll[offset:offset+logprob_i.size(0)].copy_(-logprob_i) + + offset += logprob_i.size(0) + + return nll diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py new file mode 100644 index 0000000000..554739ebea --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py @@ -0,0 +1,270 @@ +import os, sys +import glob +import numpy as np +import torch + +from utils.vocabulary import Vocab + + +class LMOrderedIterator(object): # 有序数据集 + def __init__(self, data, bsz, bptt, device='npu:0', ext_len=None): + """ + data -- LongTensor -- the LongTensor is strictly ordered + """ + self.bsz = bsz # batch_size + self.bptt = bptt # 步长？ + self.ext_len = ext_len if ext_len is not None else 0 + self.device = device + + # Work out how cleanly we can divide the dataset into bsz parts. + self.n_step = data.size(0) // bsz + + # Trim off any extra elements that wouldn't cleanly fit (remainders). + data = data.narrow(0, 0, self.n_step * bsz) + + # Evenly divide the data across the bsz batches. + self.data = data.view(bsz, -1).t().contiguous().to(device) + + # Number of mini-batches + self.n_batch = (self.n_step + self.bptt - 1) // self.bptt + + def get_batch(self, i, bptt=None): # 一个batch的data和index范围 + if bptt is None: bptt = self.bptt + seq_len = min(bptt, self.data.size(0) - 1 - i) + + end_idx = i + seq_len # 结束index + beg_idx = max(0, i - self.ext_len) # 开始index + + data = self.data[beg_idx:end_idx] + target = self.data[i+1:i+1+seq_len] + + return data, target, seq_len # 按照batch形式返回，data和target是两个列表 + + def get_fixlen_iter(self, start=0): + for i in range(start, self.data.size(0) - 1, self.bptt): + yield self.get_batch(i) # yield的作用是把一个函数变成一个 generator + + def get_varlen_iter(self, start=0, std=5, min_len=5, max_deviation=3): + max_len = self.bptt + max_deviation * std + i = start + while True: + bptt = self.bptt if np.random.random() < 0.95 else self.bptt / 2. + bptt = min(max_len, max(min_len, int(np.random.normal(bptt, std)))) + data, target, seq_len = self.get_batch(i, bptt) + i += seq_len + yield data, target, seq_len + if i >= self.data.size(0) - 2: + break + + def __iter__(self): + return self.get_fixlen_iter() + + +class LMShuffledIterator(object): # 乱序数据集 + def __init__(self, data, bsz, bptt, device='npu:0', ext_len=None, shuffle=False): + """ + data -- list[LongTensor] -- there is no order among the LongTensors + """ + self.data = data + + self.bsz = bsz + self.bptt = bptt + self.ext_len = ext_len if ext_len is not None else 0 + + self.device = device + self.shuffle = shuffle + + def get_sent_stream(self): + # index iterator + epoch_indices = np.random.permutation(len(self.data)) if self.shuffle \ + else np.array(range(len(self.data))) + + # sentence iterator + for idx in epoch_indices: + yield self.data[idx] + + def stream_iterator(self, sent_stream): + # streams for each data in the batch + streams = [None] * self.bsz + + data = torch.LongTensor(self.bptt, self.bsz) + target = torch.LongTensor(self.bptt, self.bsz) + + n_retain = 0 + + while True: + # data : [n_retain+bptt x bsz] + # target : [bptt x bsz] + data[n_retain:].fill_(-1) + target.fill_(-1) + + valid_batch = True + + for i in range(self.bsz): + n_filled = 0 + try: + while n_filled < self.bptt: + if streams[i] is None or len(streams[i]) <= 1: + streams[i] = next(sent_stream) + # number of new tokens to fill in + n_new = min(len(streams[i]) - 1, self.bptt - n_filled) + # first n_retain tokens are retained from last batch + data[n_retain+n_filled:n_retain+n_filled+n_new, i] = \ + streams[i][:n_new] + target[n_filled:n_filled+n_new, i] = \ + streams[i][1:n_new+1] + streams[i] = streams[i][n_new:] + n_filled += n_new + except StopIteration: + valid_batch = False + break + + if not valid_batch: + return + + data = data.to(self.device) + target = target.to(self.device) + + yield data, target, self.bptt + + n_retain = min(data.size(0), self.ext_len) + if n_retain > 0: + data[:n_retain] = data[-n_retain:] + data.resize_(n_retain + self.bptt, data.size(1)) + + def __iter__(self): + # sent_stream is an iterator + sent_stream = self.get_sent_stream() + + for batch in self.stream_iterator(sent_stream): + yield batch + + +class LMMultiFileIterator(LMShuffledIterator): + def __init__(self, paths, vocab, bsz, bptt, device='npu:0', ext_len=None, + shuffle=False): + + self.paths = paths + self.vocab = vocab + + self.bsz = bsz + self.bptt = bptt + self.ext_len = ext_len if ext_len is not None else 0 + + self.device = device + self.shuffle = shuffle + + def get_sent_stream(self, path): + sents = self.vocab.encode_file(path, add_double_eos=True) + if self.shuffle: + np.random.shuffle(sents) + sent_stream = iter(sents) + + return sent_stream + + def __iter__(self): + if self.shuffle: + np.random.shuffle(self.paths) + + for path in self.paths: + # sent_stream is an iterator + sent_stream = self.get_sent_stream(path) + for batch in self.stream_iterator(sent_stream): + yield batch + + +class Corpus(object): # 语料库 + def __init__(self, path, dataset, *args, **kwargs): + self.dataset = dataset + self.vocab = Vocab(*args, **kwargs) + + if self.dataset in ['ptb', 'wt2', 'enwik8', 'text8']: + self.vocab.count_file(os.path.join(path, 'train.txt')) + self.vocab.count_file(os.path.join(path, 'valid.txt')) + self.vocab.count_file(os.path.join(path, 'test.txt')) + elif self.dataset == 'wt103': + self.vocab.count_file(os.path.join(path, 'train.txt')) # path='../data/wikitext-103',对train.txt的每一行看做整体，创建列表 + elif self.dataset == 'lm1b': + train_path_pattern = os.path.join( + path, '1-billion-word-language-modeling-benchmark-r13output', + 'training-monolingual.tokenized.shuffled', 'news.en-*') + train_paths = glob.glob(train_path_pattern) + # the vocab will load from file when build_vocab() is called + + self.vocab.build_vocab() + + if self.dataset in ['ptb', 'wt2', 'wt103']: + self.train = self.vocab.encode_file( + os.path.join(path, 'train.txt'), ordered=True) # 按照行转换为tensor + self.valid = self.vocab.encode_file( + os.path.join(path, 'valid.txt'), ordered=True) + self.test = self.vocab.encode_file( + os.path.join(path, 'test.txt'), ordered=True) + elif self.dataset in ['enwik8', 'text8']: + self.train = self.vocab.encode_file( + os.path.join(path, 'train.txt'), ordered=True, add_eos=False) + self.valid = self.vocab.encode_file( + os.path.join(path, 'valid.txt'), ordered=True, add_eos=False) + self.test = self.vocab.encode_file( + os.path.join(path, 'test.txt'), ordered=True, add_eos=False) + elif self.dataset == 'lm1b': + self.train = train_paths + self.valid = self.vocab.encode_file( + os.path.join(path, 'valid.txt'), ordered=False, add_double_eos=True) + self.test = self.vocab.encode_file( + os.path.join(path, 'test.txt'), ordered=False, add_double_eos=True) + + def get_iterator(self, split, *args, **kwargs): + if split == 'train': + if self.dataset in ['ptb', 'wt2', 'wt103', 'enwik8', 'text8']: + data_iter = LMOrderedIterator(self.train, *args, **kwargs) + elif self.dataset == 'lm1b': + kwargs['shuffle'] = True + data_iter = LMMultiFileIterator(self.train, self.vocab, *args, **kwargs) + elif split in ['valid', 'test.py']: + data = self.valid if split == 'valid' else self.test + if self.dataset in ['ptb', 'wt2', 'wt103', 'enwik8', 'text8']: + data_iter = LMOrderedIterator(data, *args, **kwargs) + elif self.dataset == 'lm1b': + data_iter = LMShuffledIterator(data, *args, **kwargs) + + return data_iter + + +def get_lm_corpus(datadir, dataset): + fn = os.path.join(datadir, 'cache.pt') + if os.path.exists(fn): + print('Loading cached dataset...') + corpus = torch.load(fn) + else: + print('Producing dataset {}...'.format(dataset)) # 产生数据集 + kwargs = {} + if dataset in ['wt103', 'wt2']: + kwargs['special'] = [''] + kwargs['lower_case'] = False # 不转为小写字母 + elif dataset == 'ptb': + kwargs['special'] = [''] + kwargs['lower_case'] = True + elif dataset == 'lm1b': + kwargs['special'] = [] + kwargs['lower_case'] = False + kwargs['vocab_file'] = os.path.join(datadir, '1b_word_vocab.txt') + elif dataset in ['enwik8', 'text8']: + pass + corpus = Corpus(datadir, dataset, **kwargs) # data_iter,字典迭代器 + torch.save(corpus, fn) # fn是路径 + return corpus + + +# if __name__ == '__main__': +# import argparse +# parser = argparse.ArgumentParser(description='unit test') +# parser.add_argument('--datadir', type=str, default='../data/enwik8', +# help='location of the data corpus') +# parser.add_argument('--dataset', type=str, default='enwik8', +# choices=['ptb', 'wt2', 'wt103', 'lm1b', 'enwik8', 'text8'], +# help='dataset name') +# args = parser.parse_args() +# +# corpus = get_lm_corpus(args.datadir, args.dataset) # corpus是字典iter +# print('Vocab size : {}'.format(len(corpus.vocab.idx2sym))) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py new file mode 100644 index 0000000000..29e80f07e2 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py @@ -0,0 +1,376 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import time +import math +import os +import torch +import torch.nn as nn +import torch.optim as optim + +from data_utils import get_lm_corpus +from mem_transformer import MemTransformerLM +from utils.exp_utils import create_exp_dir +from apex import amp +import apex +from utils.exp_utils import get_logger + +parser = argparse.ArgumentParser(description='PyTorch Transformer Language Model') +parser.add_argument('--data', type=str, default='../data/enwik8', + help='location of the data corpus') +parser.add_argument('--dataset', type=str, default='enwik8', + choices=['wt103', 'lm1b', 'enwik8', 'text8'], + help='dataset name') +parser.add_argument('--n_layer', type=int, default=12, + help='number of total layers') +parser.add_argument('--n_head', type=int, default=8, + help='number of heads') +parser.add_argument('--d_head', type=int, default=64, + help='head dimension') +parser.add_argument('--d_embed', type=int, default=-1, + help='embedding dimension') +parser.add_argument('--d_model', type=int, default=512, + help='model dimension') +parser.add_argument('--d_inner', type=int, default=2048, + help='inner dimension in FF') +parser.add_argument('--dropout', type=float, default=0.1, + help='global dropout rate') +parser.add_argument('--dropatt', type=float, default=0.0, + help='attention probability dropout rate') +parser.add_argument('--init', default='normal', type=str, + help='parameter initializer to use.') +parser.add_argument('--emb_init', default='normal', type=str, + help='parameter initializer to use.') +parser.add_argument('--init_range', type=float, default=0.1, + help='parameters initialized by U(-init_range, init_range)') +parser.add_argument('--emb_init_range', type=float, default=0.01, + help='parameters initialized by U(-init_range, init_range)') +parser.add_argument('--init_std', type=float, default=0.02, + help='parameters initialized by N(0, init_std)') +parser.add_argument('--proj_init_std', type=float, default=0.01, + help='parameters initialized by N(0, init_std)') +parser.add_argument('--optim', default='adam', type=str, + choices=['adam', 'sgd', 'adagrad'], + help='optimizer to use.') +parser.add_argument('--lr', type=float, default=0.00025, + help='initial learning rate (0.00025|5 for adam|sgd)') +parser.add_argument('--mom', type=float, default=0.0, + help='momentum for sgd') +parser.add_argument('--scheduler', default='cosine', type=str, + choices=['cosine', 'inv_sqrt', 'dev_perf', 'constant'], + help='lr scheduler to use.') +parser.add_argument('--warmup_step', type=int, default=0, + help='upper epoch limit') +parser.add_argument('--decay_rate', type=float, default=0.5, + help='decay factor when ReduceLROnPlateau is used') +parser.add_argument('--lr_min', type=float, default=0.0, + help='minimum learning rate during annealing') +parser.add_argument('--clip', type=float, default=0.25, + help='gradient clipping') +parser.add_argument('--clip_nonemb', action='store_true', + help='only clip the gradient of non-embedding params') +parser.add_argument('--max_step', type=int, default=100000, + help='upper epoch limit') +parser.add_argument('--batch_size', type=int, default=10, + help='batch size') +parser.add_argument('--batch_chunk', type=int, default=1, + help='split batch into chunks to save memory') +parser.add_argument('--tgt_len', type=int, default=512, + help='number of tokens to predict') +parser.add_argument('--eval_tgt_len', type=int, default=128, + help='number of tokens to predict for evaluation') +parser.add_argument('--ext_len', type=int, default=0, + help='length of the extended context') +parser.add_argument('--mem_len', type=int, default=512, + help='length of the retained previous heads') +parser.add_argument('--not_tied', action='store_true', + help='do not tie the word embedding and softmax weights') +parser.add_argument('--seed', type=int, default=1111, + help='random seed') +parser.add_argument('--npu', default=True, help='use NPU') +parser.add_argument('--adaptive', action='store_true', + help='use adaptive softmax') +parser.add_argument('--div_val', type=int, default=1, + help='divident value for adapative input and softmax') +parser.add_argument('--pre_lnorm', action='store_true', + help='apply LayerNorm to the input instead of the output') +parser.add_argument('--varlen', action='store_true', + help='use variable length') +parser.add_argument('--multi_gpu', action='store_true', + help='use multiple GPU') +parser.add_argument('--log-interval', type=int, default=200, + help='report interval') +parser.add_argument('--eval-interval', type=int, default=4000, + help='evaluation interval') +parser.add_argument('--work_dir', default='LM-TFM', type=str, + help='experiment directory.') +parser.add_argument('--restart', action='store_true', + help='restart training from the saved checkpoint') +parser.add_argument('--restart_dir', type=str, default='', + help='restart dir') +parser.add_argument('--debug', action='store_true', + help='run in debug mode (do not create exp dir)') +parser.add_argument('--same_length', action='store_true', + help='use the same attn length for all tokens') +parser.add_argument('--attn_type', type=int, default=0, + help='attention type. 0 for ours, 1 for Shaw et al,' + '2 for Vaswani et al, 3 for Al Rfou et al.') +parser.add_argument('--clamp_len', type=int, default=-1, + help='use the same pos embeddings after clamp_len') +parser.add_argument('--eta_min', type=float, default=0.0, + help='min learning rate for cosine scheduler') +parser.add_argument('--gpu0_bsz', type=int, default=-1, + help='batch size on gpu 0') +parser.add_argument('--max_eval_steps', type=int, default=-1, + help='max eval steps') +parser.add_argument('--sample_softmax', type=int, default=-1, + help='number of samples in sampled softmax') +parser.add_argument('--patience', type=int, default=0, + help='patience') +parser.add_argument('--finetune_v2', action='store_true', + help='finetune v2') +parser.add_argument('--finetune_v3', action='store_true', + help='finetune v3') +parser.add_argument('--static-loss-scale', type=float, default=128.0, + help='Static loss scale, positive power of 2 values can ' + 'improve fp16 convergence.') +parser.add_argument('--dynamic-loss-scale', action='store_true', + help='Use dynamic loss scaling. If supplied, this argument' + ' supersedes --static-loss-scale.') +parser.add_argument('--no_log', action='store_true', + help='do not log the eval result') +parser.add_argument('--split', default='valid', + choices=['all','valid','test']) + +args = parser.parse_args() +args.tied = not args.not_tied + +if args.d_embed < 0: + args.d_embed = args.d_model + +assert args.ext_len >= 0, 'extended context length must be non-negative' +assert args.batch_size % args.batch_chunk == 0 + +args.work_dir = '{}-{}'.format(args.work_dir, args.dataset) +args.work_dir = os.path.join(args.work_dir, time.strftime('%Y%m%d-%H%M%S')) + +# Get logger +logging = create_exp_dir(args.work_dir, + scripts_to_save=['train.py', 'mem_transformer.py'], debug=args.debug) +logging = get_logger('log.txt', log_=not args.no_log) + +loc = "npu:0" +torch.npu.set_device(loc) + +############################################################################### +# Load data +############################################################################### +corpus = get_lm_corpus(args.data, args.dataset) +ntokens = len(corpus.vocab) +args.n_token = ntokens + +va_iter = corpus.get_iterator('valid', args.batch_size, args.eval_tgt_len, + device=loc, ext_len=args.ext_len) +te_iter = corpus.get_iterator('test.py', args.batch_size, args.eval_tgt_len, + device=loc, ext_len=args.ext_len) + +# adaptive softmax / embedding +cutoffs, tie_projs = [], [False] +if args.adaptive: + assert args.dataset in ['wt103', 'lm1b'] + if args.dataset == 'wt103': + cutoffs = [20000, 40000, 200000] + tie_projs += [True] * len(cutoffs) + elif args.dataset == 'lm1b': + cutoffs = [60000, 100000, 640000] + tie_projs += [False] * len(cutoffs) + +############################################################################### +# Build the model +############################################################################### +def init_weight(weight): + if args.init == 'uniform': + nn.init.uniform_(weight, -args.init_range, args.init_range) + elif args.init == 'normal': + nn.init.normal_(weight, 0.0, args.init_std) + +def init_bias(bias): + nn.init.constant_(bias, 0.0) + +def weights_init(m): + classname = m.__class__.__name__ + if classname.find('Linear') != -1: + if hasattr(m, 'weight') and m.weight is not None: + init_weight(m.weight) + if hasattr(m, 'bias') and m.bias is not None: + init_bias(m.bias) + elif classname.find('AdaptiveEmbedding') != -1: + if hasattr(m, 'emb_projs'): + for i in range(len(m.emb_projs)): + if m.emb_projs[i] is not None: + nn.init.normal_(m.emb_projs[i], 0.0, args.proj_init_std) + elif classname.find('Embedding') != -1: + if hasattr(m, 'weight'): + init_weight(m.weight) + elif classname.find('ProjectedAdaptiveLogSoftmax') != -1: + if hasattr(m, 'cluster_weight') and m.cluster_weight is not None: + init_weight(m.cluster_weight) + if hasattr(m, 'cluster_bias') and m.cluster_bias is not None: + init_bias(m.cluster_bias) + if hasattr(m, 'out_projs'): + for i in range(len(m.out_projs)): + if m.out_projs[i] is not None: + nn.init.normal_(m.out_projs[i], 0.0, args.proj_init_std) + elif classname.find('LayerNorm') != -1: + if hasattr(m, 'weight'): + nn.init.normal_(m.weight, 1.0, args.init_std) + if hasattr(m, 'bias') and m.bias is not None: + init_bias(m.bias) + elif classname.find('TransformerLM') != -1: + if hasattr(m, 'r_emb'): + init_weight(m.r_emb) + if hasattr(m, 'r_w_bias'): + init_weight(m.r_w_bias) + if hasattr(m, 'r_r_bias'): + init_weight(m.r_r_bias) + if hasattr(m, 'r_bias'): + init_bias(m.r_bias) + +def update_dropout(m): + classname = m.__class__.__name__ + if classname.find('Dropout') != -1: + if hasattr(m, 'p'): + m.p = args.dropout + +def update_dropatt(m): + if hasattr(m, 'dropatt'): + m.dropatt.p = args.dropatt + +model = MemTransformerLM(ntokens, args.n_layer, args.n_head, args.d_model, + args.d_head, args.d_inner, args.dropout, args.dropatt, + tie_weight=args.tied, d_embed=args.d_embed, div_val=args.div_val, + tie_projs=tie_projs, pre_lnorm=args.pre_lnorm, tgt_len=args.tgt_len, + ext_len=args.ext_len, mem_len=args.mem_len, cutoffs=cutoffs, + same_length=args.same_length, attn_type=args.attn_type, + clamp_len=args.clamp_len, sample_softmax=args.sample_softmax) +model.apply(weights_init) +model.word_emb.apply(weights_init) +args.n_all_param = sum([p.nelement() for p in model.parameters()]) +args.n_nonemb_param = sum([p.nelement() for p in model.layers.parameters()]) + +model = model.to(loc) + +#### optimizer +if args.optim.lower() == 'sgd': + if args.sample_softmax > 0: + dense_params, sparse_params = [], [] + for param in model.parameters(): + if param.size() == model.word_emb.weight.size(): + sparse_params.append(param) + else: + dense_params.append(param) + optimizer_sparse = optim.SGD(sparse_params, lr=args.lr * 2) + optimizer = optim.SGD(dense_params, lr=args.lr, momentum=args.mom) + else: + optimizer = optim.SGD(model.parameters(), lr=args.lr, + momentum=args.mom) +elif args.optim.lower() == 'adam': + if args.sample_softmax > 0: + dense_params, sparse_params = [], [] + for param in model.parameters(): + if param.size() == model.word_emb.weight.size(): + sparse_params.append(param) + else: + dense_params.append(param) + optimizer_sparse = optim.SparseAdam(sparse_params, lr=args.lr) + optimizer = optim.Adam(dense_params, lr=args.lr) + else: + optimizer = apex.optimizers.NpuFusedAdam(model.parameters(), lr=args.lr) +elif args.optim.lower() == 'adagrad': + optimizer = optim.Adagrad(model.parameters(), lr=args.lr) + + + +logging('=' * 100) +logging('#params = {}'.format(args.n_all_param)) +logging('#non emb params = {}'.format(args.n_nonemb_param)) + +# Load the best saved model. +with open('model_best_bpc.pt', 'rb') as f: + model.load_state_dict(torch.load(f, map_location=loc)) + +logging('Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'.format( + args.batch_size, args.tgt_len, args.ext_len, args.mem_len, args.clamp_len)) + +model.reset_length(args.tgt_len, args.ext_len, args.mem_len) +if args.clamp_len > 0: + model.clamp_len = args.clamp_len +if args.same_length: + model.same_length = True + +############################################################################### +# Evaluation code +############################################################################### + +def evaluate(eval_iter): + model.eval() + total_len, total_loss = 0, 0. + start_time = time.time() + with torch.no_grad(): + mems = tuple() + for idx, (data, target, seq_len) in enumerate(eval_iter): + ts = time.time() + ret = model(data,target,*mems) + loss, mems = ret[0], ret[1:] + loss = loss.mean() + total_loss += seq_len * loss.item() + total_len += seq_len + #print('eval_batch id: {} use time: {:.2f} ms '.format(idx, (time.time()-ts)*1000)) + total_time = time.time() - start_time + logging('Time : {:.2f}s, FPS: {:.2f} characters/s'.format( + total_time, total_len*args.batch_size*args.eval_tgt_len/total_time)) + return total_loss / total_len + + +# Run on test.py data. +if args.split == 'all': + test_loss = evaluate(te_iter) + valid_loss = evaluate(va_iter) +elif args.split == 'valid': + valid_loss = evaluate(va_iter) + test_loss = None +elif args.split == 'test': + test_loss = evaluate(te_iter) + valid_loss = None + +def format_log(loss, split): + if args.dataset in ['enwik8', 'text8']: + log_str = '| {0} loss {1:5.2f} | {0} bpc {2:9.5f} '.format( + split, loss, loss / math.log(2)) + else: + log_str = '| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '.format( + split, loss, math.exp(loss)) + return log_str + +log_str = '' +if valid_loss is not None: + log_str += format_log(valid_loss, 'valid') +if test_loss is not None: + log_str += format_log(test_loss, 'test.py') + +logging('=' * 100) +logging(log_str) +logging('=' * 100) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/mem_transformer.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/mem_transformer.py new file mode 100644 index 0000000000..92a55122f9 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/mem_transformer.py @@ -0,0 +1,851 @@ +import sys +import math +import functools + +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F + +sys.path.append('utils') +from utils.proj_adaptive_softmax import ProjectedAdaptiveLogSoftmax +from utils.log_uniform_sampler import LogUniformSampler, sample_logits + +class PositionalEmbedding(nn.Module): + def __init__(self, demb): + super(PositionalEmbedding, self).__init__() + + self.demb = demb + + inv_freq = 1 / (10000 ** (torch.arange(0.0, demb, 2.0) / demb)) + self.register_buffer('inv_freq', inv_freq) + + def forward(self, pos_seq, bsz=None): + sinusoid_inp = torch.ger(pos_seq, self.inv_freq) + pos_emb = torch.cat([sinusoid_inp.sin(), sinusoid_inp.cos()], dim=-1) + + if bsz is not None: + return pos_emb[:,None,:].expand(-1, bsz, -1) + else: + return pos_emb[:,None,:] + + +class PositionwiseFF(nn.Module): + def __init__(self, d_model, d_inner, dropout, pre_lnorm=False): + super(PositionwiseFF, self).__init__() + + self.d_model = d_model + self.d_inner = d_inner + self.dropout = dropout + + self.CoreNet = nn.Sequential( + nn.Linear(d_model, d_inner), nn.ReLU(inplace=True), + nn.Dropout(dropout), + nn.Linear(d_inner, d_model), + nn.Dropout(dropout), + ) + + self.layer_norm = nn.LayerNorm(d_model) + + self.pre_lnorm = pre_lnorm + + def forward(self, inp): + if self.pre_lnorm: + ##### layer normalization + positionwise feed-forward + core_out = self.CoreNet(self.layer_norm(inp)) + + ##### residual connection + output = core_out + inp + else: + ##### positionwise feed-forward + core_out = self.CoreNet(inp) + + ##### residual connection + layer normalization + output = self.layer_norm(inp + core_out) + + # output = self.layer_norm((inp + core_out).squeeze()) + # output = output.unsqueeze(1) + + return output + +class MultiHeadAttn(nn.Module): + def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, + pre_lnorm=False): + super(MultiHeadAttn, self).__init__() + + self.n_head = n_head + self.d_model = d_model + self.d_head = d_head + self.dropout = dropout + + self.q_net = nn.Linear(d_model, n_head * d_head, bias=False) + self.kv_net = nn.Linear(d_model, 2 * n_head * d_head, bias=False) + + self.drop = nn.Dropout(dropout) + self.dropatt = nn.Dropout(dropatt) + self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) + + self.layer_norm = nn.LayerNorm(d_model) + + self.scale = 1 / (d_head ** 0.5) + + self.pre_lnorm = pre_lnorm + + def forward(self, h, attn_mask=None, mems=None): + ##### multihead attention + # [hlen x bsz x n_head x d_head] + + if mems is not None: + c = torch.cat([mems, h], 0) + else: + c = h + + if self.pre_lnorm: + ##### layer normalization + c = self.layer_norm(c) + + head_q = self.q_net(h) + head_k, head_v = torch.chunk(self.kv_net(c), 2, -1) + + head_q = head_q.view(h.size(0), h.size(1), self.n_head, self.d_head) + head_k = head_k.view(c.size(0), c.size(1), self.n_head, self.d_head) + head_v = head_v.view(c.size(0), c.size(1), self.n_head, self.d_head) + + # [qlen x klen x bsz x n_head] + attn_score = torch.einsum('ibnd,jbnd->ijbn', (head_q, head_k)) + attn_score.mul_(self.scale) + if attn_mask is not None and attn_mask.any().item(): + if attn_mask.dim() == 2: + attn_score.masked_fill_(attn_mask[None,:,:,None], -float('inf')) + elif attn_mask.dim() == 3: + attn_score.masked_fill_(attn_mask[:,:,:,None], -float('inf')) + + # [qlen x klen x bsz x n_head] + attn_prob = F.softmax(attn_score, dim=1) + attn_prob = self.dropatt(attn_prob) + + # [qlen x klen x bsz x n_head] + [klen x bsz x n_head x d_head] -> [qlen x bsz x n_head x d_head] + attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, head_v)) + attn_vec = attn_vec.contiguous().view( + attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) + + ##### linear projection + attn_out = self.o_net(attn_vec) + attn_out = self.drop(attn_out) + + if self.pre_lnorm: + ##### residual connection + output = h + attn_out + else: + ##### residual connection + layer normalization + output = self.layer_norm(h + attn_out) + + return output + +class RelMultiHeadAttn(nn.Module): + def __init__(self, n_head, d_model, d_head, dropout, dropatt=0, + tgt_len=None, ext_len=None, mem_len=None, pre_lnorm=False): + super(RelMultiHeadAttn, self).__init__() + + self.n_head = n_head + self.d_model = d_model + self.d_head = d_head + self.dropout = dropout + + self.qkv_net = nn.Linear(d_model, 3 * n_head * d_head, bias=False) + + self.drop = nn.Dropout(dropout) + self.dropatt = nn.Dropout(dropatt) + self.o_net = nn.Linear(n_head * d_head, d_model, bias=False) + + self.layer_norm = nn.LayerNorm(d_model) + + self.scale = 1 / (d_head ** 0.5) + + self.pre_lnorm = pre_lnorm + + def _parallelogram_mask(self, h, w, left=False): + mask = torch.ones((h, w)).byte() + m = min(h, w) + mask[:m,:m] = torch.triu(mask[:m,:m]) + mask[-m:,-m:] = torch.tril(mask[-m:,-m:]) + + if left: + return mask + else: + return mask.flip(0) + + def _shift(self, x, qlen, klen, mask, left=False): + if qlen > 1: + zero_pad = torch.zeros((x.size(0), qlen-1, x.size(2), x.size(3)), + device=x.device, dtype=x.dtype) + else: + zero_pad = torch.zeros(0, device=x.device, dtype=x.dtype) + + if left: + mask = mask.flip(1) + x_padded = torch.cat([zero_pad, x], dim=1).expand(qlen, -1, -1, -1) + else: + x_padded = torch.cat([x, zero_pad], dim=1).expand(qlen, -1, -1, -1) + + x = x_padded.masked_select(mask[:,:,None,None]) \ + .view(qlen, klen, x.size(2), x.size(3)) + + return x + + def _rel_shift(self, x, zero_triu=False): + zero_pad = torch.zeros((x.size(0), 1, *x.size()[2:]), + device=x.device, dtype=x.dtype) + x_padded = torch.cat([zero_pad, x], dim=1) + + x_padded = x_padded.view(x.size(1) + 1, x.size(0), *x.size()[2:]) + + x = x_padded[1:].view_as(x) + + if zero_triu: + ones = torch.ones((x.size(0), x.size(1))) + x = x * torch.tril(ones, x.size(1) - x.size(0))[:,:,None,None] + + return x + + def forward(self, w, r, attn_mask=None, mems=None): + + raise NotImplementedError + +class RelPartialLearnableMultiHeadAttn(RelMultiHeadAttn): + def __init__(self, *args, **kwargs): + super(RelPartialLearnableMultiHeadAttn, self).__init__(*args, **kwargs) + + self.r_net = nn.Linear(self.d_model, self.n_head * self.d_head, bias=False) + + def forward(self, w, r, r_w_bias, r_r_bias, attn_mask=None, mems=None): + qlen, rlen, bsz = w.size(0), r.size(0), w.size(1) + if mems is not None: + cat = torch.cat([mems, w], 0) + if self.pre_lnorm: + w_heads = self.qkv_net(self.layer_norm(cat)) + else: + w_heads = self.qkv_net(cat) + r_head_k = self.r_net(r) + + w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + w_head_q = w_head_q[-qlen:] + else: + if self.pre_lnorm: + w_heads = self.qkv_net(self.layer_norm(w)) + else: + w_heads = self.qkv_net(w) + r_head_k = self.r_net(r) + + w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + + klen = w_head_k.size(0) + + w_head_q = w_head_q.view(qlen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head + w_head_k = w_head_k.view(klen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head + w_head_v = w_head_v.view(klen, bsz, self.n_head, self.d_head) # qlen x bsz x n_head x d_head + + r_head_k = r_head_k.view(rlen, self.n_head, self.d_head) # qlen x n_head x d_head + + #### compute attention score + rw_head_q = w_head_q + r_w_bias # qlen x bsz x n_head x d_head + + AC = torch.einsum('ibnd,jbnd->ijbn', (rw_head_q, w_head_k)) # qlen x klen x bsz x n_head + + rr_head_q = w_head_q + r_r_bias + BD = torch.einsum('ibnd,jnd->ijbn', (rr_head_q, r_head_k)) # qlen x klen x bsz x n_head + BD = self._rel_shift(BD) + + # [qlen x klen x bsz x n_head] + attn_score = AC + BD + attn_score.mul_(self.scale) + + #### compute attention probability + attn_mask_bool = attn_mask.bool() + if attn_mask is not None and attn_mask_bool.any().item(): + if attn_mask.dim() == 2: + attn_score.masked_fill_(attn_mask[None, :, :, None], -float('inf')).bool().type_as(attn_score) + elif attn_mask.dim() == 3: + attn_score.masked_fill_(attn_mask[:, :, :, None], -float('inf')).bool().type_as(attn_score) + + # [qlen x klen x bsz x n_head] + attn_prob = F.softmax(attn_score, dim=1) + attn_prob = self.dropatt(attn_prob) + + #### compute attention vector + attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, w_head_v)) + + # [qlen x bsz x n_head x d_head] + attn_vec = attn_vec.contiguous().view( + attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) + + ##### linear projection + attn_out = self.o_net(attn_vec) + attn_out = self.drop(attn_out) + + if self.pre_lnorm: + ##### residual connection + output = w + attn_out + else: + ##### residual connection + layer normalization + output = self.layer_norm(w + attn_out) + + return output + +class RelLearnableMultiHeadAttn(RelMultiHeadAttn): + def __init__(self, *args, **kwargs): + super(RelLearnableMultiHeadAttn, self).__init__(*args, **kwargs) + + def forward(self, w, r_emb, r_w_bias, r_bias, attn_mask=None, mems=None): + # r_emb: [klen, n_head, d_head], used for term B + # r_w_bias: [n_head, d_head], used for term C + # r_bias: [klen, n_head], used for term D + + qlen, bsz = w.size(0), w.size(1) + + if mems is not None: + cat = torch.cat([mems, w], 0) + if self.pre_lnorm: + w_heads = self.qkv_net(self.layer_norm(cat)) + else: + w_heads = self.qkv_net(cat) + w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + + w_head_q = w_head_q[-qlen:] + else: + if self.pre_lnorm: + w_heads = self.qkv_net(self.layer_norm(w)) + else: + w_heads = self.qkv_net(w) + w_head_q, w_head_k, w_head_v = torch.chunk(w_heads, 3, dim=-1) + + klen = w_head_k.size(0) + + w_head_q = w_head_q.view(qlen, bsz, self.n_head, self.d_head) + w_head_k = w_head_k.view(klen, bsz, self.n_head, self.d_head) + w_head_v = w_head_v.view(klen, bsz, self.n_head, self.d_head) + + if klen > r_emb.size(0): + r_emb_pad = r_emb[0:1].expand(klen-r_emb.size(0), -1, -1) + r_emb = torch.cat([r_emb_pad, r_emb], 0) + r_bias_pad = r_bias[0:1].expand(klen-r_bias.size(0), -1) + r_bias = torch.cat([r_bias_pad, r_bias], 0) + else: + r_emb = r_emb[-klen:] + r_bias = r_bias[-klen:] + + #### compute attention score + rw_head_q = w_head_q + r_w_bias[None] # qlen x bsz x n_head x d_head + + AC = torch.einsum('ibnd,jbnd->ijbn', (rw_head_q, w_head_k)) # qlen x klen x bsz x n_head + B_ = torch.einsum('ibnd,jnd->ijbn', (w_head_q, r_emb)) # qlen x klen x bsz x n_head + D_ = r_bias[None, :, None] # 1 x klen x 1 x n_head + BD = self._rel_shift(B_ + D_) + + # [qlen x klen x bsz x n_head] + attn_score = AC + BD + attn_score.mul_(self.scale) + + #### compute attention probability + if attn_mask is not None and attn_mask.any().item(): + if attn_mask.dim() == 2: + attn_score.masked_fill_(attn_mask[None,:,:,None], -float('inf')) + elif attn_mask.dim() == 3: + attn_score.masked_fill_(attn_mask[:,:,:,None], -float('inf')) + + # [qlen x klen x bsz x n_head] + attn_prob = F.softmax(attn_score, dim=1) + attn_prob = self.dropatt(attn_prob) + + #### compute attention vector + attn_vec = torch.einsum('ijbn,jbnd->ibnd', (attn_prob, w_head_v)) + + # [qlen x bsz x n_head x d_head] + attn_vec = attn_vec.contiguous().view( + attn_vec.size(0), attn_vec.size(1), self.n_head * self.d_head) + + ##### linear projection + attn_out = self.o_net(attn_vec) + attn_out = self.drop(attn_out) + + if self.pre_lnorm: + ##### residual connection + output = w + attn_out + else: + ##### residual connection + layer normalization + output = self.layer_norm(w + attn_out) + + return output + +class DecoderLayer(nn.Module): + def __init__(self, n_head, d_model, d_head, d_inner, dropout, **kwargs): + super(DecoderLayer, self).__init__() + + self.dec_attn = MultiHeadAttn(n_head, d_model, d_head, dropout, **kwargs) + self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, + pre_lnorm=kwargs.get('pre_lnorm')) + + def forward(self, dec_inp, dec_attn_mask=None, mems=None): + output = self.dec_attn(dec_inp, attn_mask=dec_attn_mask, + mems=mems) + output = self.pos_ff(output) + + return output + +class RelLearnableDecoderLayer(nn.Module): + def __init__(self, n_head, d_model, d_head, d_inner, dropout, + **kwargs): + super(RelLearnableDecoderLayer, self).__init__() + + self.dec_attn = RelLearnableMultiHeadAttn(n_head, d_model, d_head, dropout, + **kwargs) + self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, + pre_lnorm=kwargs.get('pre_lnorm')) + + def forward(self, dec_inp, r_emb, r_w_bias, r_bias, dec_attn_mask=None, mems=None): + output = self.dec_attn(dec_inp, r_emb, r_w_bias, r_bias, + attn_mask=dec_attn_mask, + mems=mems) + output = self.pos_ff(output) + + return output + +class RelPartialLearnableDecoderLayer(nn.Module): + def __init__(self, n_head, d_model, d_head, d_inner, dropout, + **kwargs): + super(RelPartialLearnableDecoderLayer, self).__init__() + + self.dec_attn = RelPartialLearnableMultiHeadAttn(n_head, d_model, + d_head, dropout, **kwargs) + self.pos_ff = PositionwiseFF(d_model, d_inner, dropout, + pre_lnorm=kwargs.get('pre_lnorm')) + + def forward(self, dec_inp, r, r_w_bias, r_r_bias, dec_attn_mask=None, mems=None): + output = self.dec_attn(dec_inp, r, r_w_bias, r_r_bias, + attn_mask=dec_attn_mask, + mems=mems) + output = self.pos_ff(output) + + return output + + +class AdaptiveEmbedding(nn.Module): + def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, + sample_softmax=False): + super(AdaptiveEmbedding, self).__init__() + + self.n_token = n_token + self.d_embed = d_embed + + self.cutoffs = cutoffs + [n_token] + self.div_val = div_val + self.d_proj = d_proj + + self.emb_scale = d_proj ** 0.5 + + self.cutoff_ends = [0] + self.cutoffs + + self.emb_layers = nn.ModuleList() + self.emb_projs = nn.ParameterList() + if div_val == 1: + print("n_token:", n_token) + print("d_embed:", d_embed) + # self.emb_layers.append( + # nn.Embedding(n_token, 512, sparse=sample_softmax>0) + # ) + self.emb_layers.append( + nn.Embedding(n_token, d_embed, sparse=sample_softmax>0) + ) + if d_proj != d_embed: + self.emb_projs.append(nn.Parameter(torch.Tensor(d_proj, d_embed))) + else: + for i in range(len(self.cutoffs)): + l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i+1] + d_emb_i = d_embed // (div_val ** i) + self.emb_layers.append(nn.Embedding(r_idx-l_idx, d_emb_i)) + self.emb_projs.append(nn.Parameter(torch.Tensor(d_proj, d_emb_i))) + + def forward(self, inp): + if self.div_val == 1: + embed = self.emb_layers[0](inp) + if self.d_proj != self.d_embed: + embed = F.linear(embed, self.emb_projs[0]) + else: + param = next(self.parameters()) + inp_flat = inp.view(-1) + emb_flat = torch.zeros([inp_flat.size(0), self.d_proj], + dtype=param.dtype, device=param.device) + for i in range(len(self.cutoffs)): + l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i + 1] + + mask_i = (inp_flat >= l_idx) & (inp_flat < r_idx) + indices_i = mask_i.nonzero().squeeze() + + if indices_i.numel() == 0: + continue + + inp_i = inp_flat.index_select(0, indices_i) - l_idx + emb_i = self.emb_layers[i](inp_i) + emb_i = F.linear(emb_i, self.emb_projs[i]) + + emb_flat.index_copy_(0, indices_i, emb_i) + + embed = emb_flat.view(*inp.size(), self.d_proj) + + embed.mul_(self.emb_scale) + + return embed + +class MemTransformerLM(nn.Module): + def __init__(self, n_token, n_layer, n_head, d_model, d_head, d_inner, + dropout, dropatt, tie_weight=True, d_embed=None, + div_val=1, tie_projs=[False], pre_lnorm=False, + tgt_len=None, ext_len=None, mem_len=None, + cutoffs=[], adapt_inp=False, + same_length=False, attn_type=0, clamp_len=-1, + sample_softmax=-1): + super(MemTransformerLM, self).__init__() + self.n_token = n_token + + d_embed = d_model if d_embed is None else d_embed + self.d_embed = d_embed + self.d_model = d_model + self.n_head = n_head + self.d_head = d_head + self.word_emb = AdaptiveEmbedding(n_token, d_embed, d_model, cutoffs, + div_val=div_val) + self.drop = nn.Dropout(dropout) + + self.n_layer = n_layer + + self.tgt_len = tgt_len + self.mem_len = mem_len + self.ext_len = ext_len + self.max_klen = tgt_len + ext_len + mem_len + + self.attn_type = attn_type + + self.layers = nn.ModuleList() + if attn_type == 0: # the default attention + for i in range(n_layer): + self.layers.append( + RelPartialLearnableDecoderLayer( + n_head, d_model, d_head, d_inner, dropout, + tgt_len=tgt_len, ext_len=ext_len, mem_len=mem_len, + dropatt=dropatt, pre_lnorm=pre_lnorm) + ) + elif attn_type == 1: # learnable embeddings + for i in range(n_layer): + self.layers.append( + RelLearnableDecoderLayer( + n_head, d_model, d_head, d_inner, dropout, + tgt_len=tgt_len, ext_len=ext_len, mem_len=mem_len, + dropatt=dropatt, pre_lnorm=pre_lnorm) + ) + elif attn_type in [2, 3]: # absolute embeddings + for i in range(n_layer): + self.layers.append( + DecoderLayer( + n_head, d_model, d_head, d_inner, dropout, + dropatt=dropatt, pre_lnorm=pre_lnorm) + ) + + self.sample_softmax = sample_softmax + # use sampled softmax + if sample_softmax > 0: + self.out_layer = nn.Linear(d_model, n_token) + if tie_weight: + self.out_layer.weight = self.word_emb.weight + self.tie_weight = tie_weight + self.sampler = LogUniformSampler(n_token, sample_softmax) + + # use adaptive softmax (including standard softmax) + else: + # dump_tensor(n_token, 'n_token.pt') + # dump_tensor(d_embed, 'd_embed.pt') + # dump_tensor(d_model, 'd_model.pt') + # dump_tensor(cutoffs, 'cutoffs.pt') + # dump_tensor(div_val, 'div_val.pt') + + self.crit = ProjectedAdaptiveLogSoftmax(n_token, d_embed, d_model, + cutoffs, div_val=div_val) + + if tie_weight: + for i in range(len(self.crit.out_layers)): + self.crit.out_layers[i].weight = self.word_emb.emb_layers[i].weight + + if tie_projs: + for i, tie_proj in enumerate(tie_projs): + if tie_proj and div_val == 1 and d_model != d_embed: + self.crit.out_projs[i] = self.word_emb.emb_projs[0] + elif tie_proj and div_val != 1: + self.crit.out_projs[i] = self.word_emb.emb_projs[i] + + self.same_length = same_length + self.clamp_len = clamp_len + + self._create_params() + + def backward_compatible(self): + self.sample_softmax = -1 + + def _create_params(self): + if self.attn_type == 0: # default attention + self.pos_emb = PositionalEmbedding(self.d_model) + self.r_w_bias = nn.Parameter(torch.Tensor(self.n_head, self.d_head)) + self.r_r_bias = nn.Parameter(torch.Tensor(self.n_head, self.d_head)) + elif self.attn_type == 1: # learnable + self.r_emb = nn.Parameter(torch.Tensor( + self.n_layer, self.max_klen, self.n_head, self.d_head)) + self.r_w_bias = nn.Parameter(torch.Tensor( + self.n_layer, self.n_head, self.d_head)) + self.r_bias = nn.Parameter(torch.Tensor( + self.n_layer, self.max_klen, self.n_head)) + elif self.attn_type == 2: # absolute standard + self.pos_emb = PositionalEmbedding(self.d_model) + elif self.attn_type == 3: # absolute deeper SA + self.r_emb = nn.Parameter(torch.Tensor( + self.n_layer, self.max_klen, self.n_head, self.d_head)) + + def reset_length(self, tgt_len, ext_len, mem_len): + self.tgt_len = tgt_len + self.mem_len = mem_len + self.ext_len = ext_len + + def init_mems(self): + if self.mem_len > 0: + mems = [] + param = next(self.parameters()) + for i in range(self.n_layer+1): + empty = torch.empty(0, dtype=param.dtype, device=param.device) + mems.append(empty) + + return mems + else: + return None + + def _update_mems(self, hids, mems, qlen, mlen): + # does not deal with None + if mems is None: return None + + # mems is not None + assert len(hids) == len(mems), 'len(hids) != len(mems)' + + # There are `mlen + qlen` steps that can be cached into mems + # For the next step, the last `ext_len` of the `qlen` tokens + # will be used as the extended context. Hence, we only cache + # the tokens from `mlen + qlen - self.ext_len - self.mem_len` + # to `mlen + qlen - self.ext_len`. + with torch.no_grad(): + new_mems = [] + end_idx = mlen + max(0, qlen - 0 - self.ext_len) + beg_idx = max(0, end_idx - self.mem_len) + for i in range(len(hids)): + + cat = torch.cat([mems[i], hids[i]], dim=0) + new_mems.append(cat[beg_idx:end_idx].detach()) + + return new_mems + + def _forward(self, dec_inp, mems=None): + qlen, bsz = dec_inp.size() + word_emb = self.word_emb(dec_inp.long()) + + mlen = mems[0].size(0) if mems is not None else 0 + klen = mlen + qlen + if self.same_length: + all_ones = word_emb.new_ones(qlen, klen) + mask_len = klen - self.mem_len + if mask_len > 0: + mask_shift_len = qlen - mask_len + else: + mask_shift_len = qlen + dec_attn_mask = (torch.triu(all_ones, 1+mlen) + + torch.tril(all_ones, -mask_shift_len)).byte()[:, :, None] # -1 + else: + dec_attn_mask = torch.triu( + word_emb.new_ones(qlen, klen), diagonal=1+mlen).byte()[:,:,None] + + hids = [] + if self.attn_type == 0: # default + pos_seq = torch.arange(klen-1, -1, -1.0, device=word_emb.device, + dtype=word_emb.dtype) + if self.clamp_len > 0: + pos_seq.clamp_(max=self.clamp_len) + pos_emb = self.pos_emb(pos_seq) + + core_out = self.drop(word_emb) + pos_emb = self.drop(pos_emb) + + hids.append(core_out) + for i, layer in enumerate(self.layers): + mems_i = None if mems is None else mems[i] + core_out = layer(core_out, pos_emb, self.r_w_bias, + self.r_r_bias, dec_attn_mask=dec_attn_mask, mems=mems_i) + hids.append(core_out) + elif self.attn_type == 1: # learnable + core_out = self.drop(word_emb) + hids.append(core_out) + for i, layer in enumerate(self.layers): + if self.clamp_len > 0: + r_emb = self.r_emb[i][-self.clamp_len :] + r_bias = self.r_bias[i][-self.clamp_len :] + else: + r_emb, r_bias = self.r_emb[i], self.r_bias[i] + + mems_i = None if mems is None else mems[i] + core_out = layer(core_out, r_emb, self.r_w_bias[i], + r_bias, dec_attn_mask=dec_attn_mask, mems=mems_i) + hids.append(core_out) + elif self.attn_type == 2: # absolute + pos_seq = torch.arange(klen - 1, -1, -1.0, device=word_emb.device, + dtype=word_emb.dtype) + if self.clamp_len > 0: + pos_seq.clamp_(max=self.clamp_len) + pos_emb = self.pos_emb(pos_seq) + + core_out = self.drop(word_emb + pos_emb[-qlen:]) + + hids.append(core_out) + for i, layer in enumerate(self.layers): + mems_i = None if mems is None else mems[i] + if mems_i is not None and i == 0: + mems_i += pos_emb[:mlen] + core_out = layer(core_out, dec_attn_mask=dec_attn_mask, + mems=mems_i) + hids.append(core_out) + elif self.attn_type == 3: + core_out = self.drop(word_emb) + + hids.append(core_out) + for i, layer in enumerate(self.layers): + mems_i = None if mems is None else mems[i] + if mems_i is not None and mlen > 0: + cur_emb = self.r_emb[i][:-qlen] + cur_size = cur_emb.size(0) + if cur_size < mlen: + cur_emb_pad = cur_emb[0:1].expand(mlen-cur_size, -1, -1) + cur_emb = torch.cat([cur_emb_pad, cur_emb], 0) + else: + cur_emb = cur_emb[-mlen:] + mems_i += cur_emb.view(mlen, 1, -1) + core_out += self.r_emb[i][-qlen:].view(qlen, 1, -1) + + core_out = layer(core_out, dec_attn_mask=dec_attn_mask, + mems=mems_i) + hids.append(core_out) + + core_out = self.drop(core_out) + + new_mems = self._update_mems(hids, mems, mlen, qlen) + + return core_out, new_mems + + def forward(self, data, target, *mems): + # nn.DataParallel does not allow size(0) tensors to be broadcasted. + # So, have to initialize size(0) mems inside the model forward. + # Moreover, have to return new_mems to allow nn.DataParallel to piece + # them together. + if not mems: mems = self.init_mems() + + tgt_len = target.size(0) + hidden, new_mems = self._forward(data, mems=mems) + + pred_hid = hidden[-tgt_len:] + if self.sample_softmax > 0 and self.training: + assert self.tie_weight + logit = sample_logits(self.word_emb, + self.out_layer.bias, target, pred_hid, self.sampler) + loss = -F.log_softmax(logit, -1)[:, :, 0] + else: + loss = self.crit(pred_hid.view(-1, pred_hid.size(-1)), target.view(-1)) + loss = loss.view(tgt_len, -1) + loss = loss.npu() + + if new_mems is None: + return [loss] + else: + return [loss] + new_mems + +def set_device(obj, device='cpu'): + if isinstance(obj, (tuple, list)): + dump = [] + for item in obj: + dump.append(set_device(item, device)) + return dump + elif isinstance(obj, dict): + dump = {} + for k, v in obj.items(): + dump[k] = set_device(v, device) + return dump + elif isinstance(obj, torch.Tensor): + return obj.to(device) + else: + return obj + + +def dump_tensor(output, name): + dump = set_device(output, 'cpu') + torch.save(dump, name) + print('%s dump success!' % (name)) + + +def load_tensor(name, device): + output = torch.load(name) + dump = set_device(output, device) + print('%s load success!' % (name)) + return dump + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser(description='unit test') + + parser.add_argument('--n_layer', type=int, default=4, help='') + parser.add_argument('--n_rel_layer', type=int, default=4, help='') + parser.add_argument('--n_head', type=int, default=2, help='') + parser.add_argument('--d_head', type=int, default=2, help='') + parser.add_argument('--d_model', type=int, default=200, help='') + parser.add_argument('--d_embed', type=int, default=200, help='') + parser.add_argument('--d_inner', type=int, default=200, help='') + parser.add_argument('--dropout', type=float, default=0.0, help='') + parser.add_argument('--cuda', action='store_true', help='') + parser.add_argument('--seed', type=int, default=1111, help='') + parser.add_argument('--multi_gpu', action='store_true', help='') + + args = parser.parse_args() + + #device = torch.device("cuda" if args.cuda else "cpu") + device = torch.device("npu:0") + + B = 4 + tgt_len, mem_len, ext_len = 36, 36, 0 + data_len = tgt_len * 20 + args.n_token = 10000 + + import data_utils + + data = torch.LongTensor(data_len*B).random_(0, args.n_token).to(device) + diter = data_utils.LMOrderedIterator(data, B, tgt_len, device=device, ext_len=ext_len) + + cutoffs = [args.n_token // 2] + tie_projs = [False] + [True] * len(cutoffs) + + for div_val in [1, 2]: + for d_embed in [200, 100]: + model = MemTransformerLM(args.n_token, args.n_layer, args.n_head, + args.d_model, args.d_head, args.d_inner, args.dropout, + dropatt=args.dropout, tie_weight=True, + d_embed=d_embed, div_val=div_val, + tie_projs=tie_projs, pre_lnorm=True, + tgt_len=tgt_len, ext_len=ext_len, mem_len=mem_len, + cutoffs=cutoffs, attn_type=0).to(device) + + print(sum(p.numel() for p in model.parameters())) + + mems = tuple() + for idx, (inp, tgt, seqlen) in enumerate(diter): + print('batch {}'.format(idx)) + out = model(inp, tgt, *mems) + mems = out[1:] diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt new file mode 100644 index 0000000000..559473df33 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt @@ -0,0 +1,3 @@ +FuncStatus: +PerfStatus: +PrecisionStatus: \ No newline at end of file diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/requirements.txt b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/requirements.txt new file mode 100644 index 0000000000..7f4eaeb22a --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/requirements.txt @@ -0,0 +1,5 @@ +torchvision +tqdm +numpy +itertools +argparse \ No newline at end of file diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_1p_npu.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_1p_npu.py new file mode 100644 index 0000000000..8f213a8237 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_1p_npu.py @@ -0,0 +1,542 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +import time +import math +import os +import itertools + +import torch +import torch.nn as nn +import torch.optim as optim + +from data_utils import get_lm_corpus +from mem_transformer import MemTransformerLM +from utils.exp_utils import create_exp_dir +from utils.data_parallel import BalancedDataParallel +from apex import amp +import apex + +parser = argparse.ArgumentParser(description='PyTorch Transformer Language Model') +parser.add_argument('--data', type=str, default='../data/enwik8', + help='location of the data corpus') +parser.add_argument('--dataset', type=str, default='enwik8', + choices=['wt103', 'lm1b', 'enwik8', 'text8'], + help='dataset name') +parser.add_argument('--n_layer', type=int, default=12, + help='number of total layers') +parser.add_argument('--n_head', type=int, default=8, + help='number of heads') +parser.add_argument('--d_head', type=int, default=64, + help='head dimension') +parser.add_argument('--d_embed', type=int, default=-1, + help='embedding dimension') +parser.add_argument('--d_model', type=int, default=512, + help='model dimension') +parser.add_argument('--d_inner', type=int, default=2048, + help='inner dimension in FF') +parser.add_argument('--dropout', type=float, default=0.1, + help='global dropout rate') +parser.add_argument('--dropatt', type=float, default=0.0, + help='attention probability dropout rate') +parser.add_argument('--init', default='normal', type=str, + help='parameter initializer to use.') +parser.add_argument('--emb_init', default='normal', type=str, + help='parameter initializer to use.') +parser.add_argument('--init_range', type=float, default=0.1, + help='parameters initialized by U(-init_range, init_range)') +parser.add_argument('--emb_init_range', type=float, default=0.01, + help='parameters initialized by U(-init_range, init_range)') +parser.add_argument('--init_std', type=float, default=0.02, + help='parameters initialized by N(0, init_std)') +parser.add_argument('--proj_init_std', type=float, default=0.01, + help='parameters initialized by N(0, init_std)') +parser.add_argument('--optim', default='adam', type=str, + choices=['adam', 'sgd', 'adagrad'], + help='optimizer to use.') +parser.add_argument('--lr', type=float, default=0.00025, + help='initial learning rate (0.00025|5 for adam|sgd)') +parser.add_argument('--epochs', type=int, default=50, + help='train epochs') +parser.add_argument('--mom', type=float, default=0.0, + help='momentum for sgd') +parser.add_argument('--scheduler', default='cosine', type=str, + choices=['cosine', 'inv_sqrt', 'dev_perf', 'constant'], + help='lr scheduler to use.') +parser.add_argument('--warmup_step', type=int, default=0, + help='upper epoch limit') +parser.add_argument('--decay_rate', type=float, default=0.5, + help='decay factor when ReduceLROnPlateau is used') +parser.add_argument('--lr_min', type=float, default=0.0, + help='minimum learning rate during annealing') +parser.add_argument('--clip', type=float, default=0.25, + help='gradient clipping') +parser.add_argument('--clip_nonemb', action='store_true', + help='only clip the gradient of non-embedding params') +parser.add_argument('--max_step', type=int, default=100000, + help='upper epoch limit') +parser.add_argument('--batch_size', type=int, default=22, + help='batch size') +parser.add_argument('--batch_chunk', type=int, default=1, + help='split batch into chunks to save memory') +parser.add_argument('--tgt_len', type=int, default=512, + help='number of tokens to predict') +parser.add_argument('--eval_tgt_len', type=int, default=128, + help='number of tokens to predict for evaluation') +parser.add_argument('--ext_len', type=int, default=0, + help='length of the extended context') +parser.add_argument('--mem_len', type=int, default=512, + help='length of the retained previous heads') +parser.add_argument('--not_tied', action='store_true', + help='do not tie the word embedding and softmax weights') +parser.add_argument('--seed', type=int, default=1111, + help='random seed') +parser.add_argument('--workers', type=int, default=64, + help='workers num') +parser.add_argument('--npu', default=True, help='use NPU') +parser.add_argument('--adaptive', action='store_true', + help='use adaptive softmax') +parser.add_argument('--div_val', type=int, default=1, + help='divident value for adapative input and softmax') +parser.add_argument('--pre_lnorm', action='store_true', + help='apply LayerNorm to the input instead of the output') +parser.add_argument('--varlen', action='store_true', + help='use variable length') +parser.add_argument('--multi_gpu', action='store_true', + help='use multiple GPU') +parser.add_argument('--log-interval', type=int, default=200, + help='report interval') +parser.add_argument('--eval-interval', type=int, default=4000, + help='evaluation interval') +parser.add_argument('--work_dir', default='LM-TFM', type=str, + help='experiment directory.') +parser.add_argument('--restart', action='store_true', + help='restart training from the saved checkpoint') +parser.add_argument('--restart_dir', type=str, default='', + help='restart dir') +parser.add_argument('--debug', action='store_true', + help='run in debug mode (do not create exp dir)') +parser.add_argument('--same_length', action='store_true', + help='use the same attn length for all tokens') +parser.add_argument('--attn_type', type=int, default=0, + help='attention type. 0 for ours, 1 for Shaw et al,' + '2 for Vaswani et al, 3 for Al Rfou et al.') +parser.add_argument('--clamp_len', type=int, default=-1, + help='use the same pos embeddings after clamp_len') +parser.add_argument('--eta_min', type=float, default=0.0, + help='min learning rate for cosine scheduler') +parser.add_argument('--gpu0_bsz', type=int, default=-1, + help='batch size on gpu 0') +parser.add_argument('--max_eval_steps', type=int, default=-1, + help='max eval steps') +parser.add_argument('--sample_softmax', type=int, default=-1, + help='number of samples in sampled softmax') +parser.add_argument('--patience', type=int, default=0, + help='patience') +parser.add_argument('--finetune_v2', action='store_true', + help='finetune v2') +parser.add_argument('--finetune_v3', action='store_true', + help='finetune v3') +parser.add_argument('--static-loss-scale', type=float, default=128.0, + help='Static loss scale, positive power of 2 values can ' + 'improve fp16 convergence.') +parser.add_argument('--dynamic-loss-scale', action='store_true', + help='Use dynamic loss scaling. If supplied, this argument' + ' supersedes --static-loss-scale.') +args = parser.parse_args() +args.tied = not args.not_tied + +if args.d_embed < 0: + args.d_embed = args.d_model + +assert args.ext_len >= 0, 'extended context length must be non-negative' +assert args.batch_size % args.batch_chunk == 0 + +args.work_dir = '{}-{}'.format(args.work_dir, args.dataset) +args.work_dir = os.path.join(args.work_dir, time.strftime('%Y%m%d-%H%M%S')) +logging = create_exp_dir(args.work_dir, + scripts_to_save=['train.py', 'mem_transformer.py'], debug=args.debug) + +device = torch.device('npu:0') + +############################################################################### +# Load data +############################################################################### +corpus = get_lm_corpus(args.data, args.dataset) +ntokens = len(corpus.vocab) +args.n_token = ntokens + +eval_batch_size = 10 +tr_iter = corpus.get_iterator('train', args.batch_size, args.tgt_len, + device=device, ext_len=args.ext_len) +va_iter = corpus.get_iterator('valid', eval_batch_size, args.eval_tgt_len, + device=device, ext_len=args.ext_len) +te_iter = corpus.get_iterator('test.py', eval_batch_size, args.eval_tgt_len, + device=device, ext_len=args.ext_len) + +# adaptive softmax / embedding +cutoffs, tie_projs = [], [False] +if args.adaptive: + assert args.dataset in ['wt103', 'lm1b'] + if args.dataset == 'wt103': + cutoffs = [20000, 40000, 200000] + tie_projs += [True] * len(cutoffs) + elif args.dataset == 'lm1b': + cutoffs = [60000, 100000, 640000] + tie_projs += [False] * len(cutoffs) + +############################################################################### +# Build the model +############################################################################### +def init_weight(weight): + if args.init == 'uniform': + nn.init.uniform_(weight, -args.init_range, args.init_range) + elif args.init == 'normal': + nn.init.normal_(weight, 0.0, args.init_std) + +def init_bias(bias): + nn.init.constant_(bias, 0.0) + +def weights_init(m): + classname = m.__class__.__name__ + if classname.find('Linear') != -1: + if hasattr(m, 'weight') and m.weight is not None: + init_weight(m.weight) + if hasattr(m, 'bias') and m.bias is not None: + init_bias(m.bias) + elif classname.find('AdaptiveEmbedding') != -1: + if hasattr(m, 'emb_projs'): + for i in range(len(m.emb_projs)): + if m.emb_projs[i] is not None: + nn.init.normal_(m.emb_projs[i], 0.0, args.proj_init_std) + elif classname.find('Embedding') != -1: + if hasattr(m, 'weight'): + init_weight(m.weight) + elif classname.find('ProjectedAdaptiveLogSoftmax') != -1: + if hasattr(m, 'cluster_weight') and m.cluster_weight is not None: + init_weight(m.cluster_weight) + if hasattr(m, 'cluster_bias') and m.cluster_bias is not None: + init_bias(m.cluster_bias) + if hasattr(m, 'out_projs'): + for i in range(len(m.out_projs)): + if m.out_projs[i] is not None: + nn.init.normal_(m.out_projs[i], 0.0, args.proj_init_std) + elif classname.find('LayerNorm') != -1: + if hasattr(m, 'weight'): + nn.init.normal_(m.weight, 1.0, args.init_std) + if hasattr(m, 'bias') and m.bias is not None: + init_bias(m.bias) + elif classname.find('TransformerLM') != -1: + if hasattr(m, 'r_emb'): + init_weight(m.r_emb) + if hasattr(m, 'r_w_bias'): + init_weight(m.r_w_bias) + if hasattr(m, 'r_r_bias'): + init_weight(m.r_r_bias) + if hasattr(m, 'r_bias'): + init_bias(m.r_bias) + +def update_dropout(m): + classname = m.__class__.__name__ + if classname.find('Dropout') != -1: + if hasattr(m, 'p'): + m.p = args.dropout + +def update_dropatt(m): + if hasattr(m, 'dropatt'): + m.dropatt.p = args.dropatt + +if args.restart: + with open(os.path.join(args.restart_dir, 'model.pt'), 'rb') as f: + model = torch.load(f) + model.apply(update_dropout) + model.apply(update_dropatt) +else: + model = MemTransformerLM(ntokens, args.n_layer, args.n_head, args.d_model, + args.d_head, args.d_inner, args.dropout, args.dropatt, + tie_weight=args.tied, d_embed=args.d_embed, div_val=args.div_val, + tie_projs=tie_projs, pre_lnorm=args.pre_lnorm, tgt_len=args.tgt_len, + ext_len=args.ext_len, mem_len=args.mem_len, cutoffs=cutoffs, + same_length=args.same_length, attn_type=args.attn_type, + clamp_len=args.clamp_len, sample_softmax=args.sample_softmax) + model.apply(weights_init) + model.word_emb.apply(weights_init) # ensure embedding init is not overridden by out_layer in case of weight sharing +args.n_all_param = sum([p.nelement() for p in model.parameters()]) +args.n_nonemb_param = sum([p.nelement() for p in model.layers.parameters()]) + + +if args.multi_gpu: + model = model.to(device) + if args.gpu0_bsz >= 0: + para_model = BalancedDataParallel(args.gpu0_bsz // args.batch_chunk, + model, dim=1).to(device) + else: + para_model = nn.DataParallel(model, dim=1).to(device) +else: + para_model = model.to(device) + +#### optimizer +if args.optim.lower() == 'sgd': + if args.sample_softmax > 0: + dense_params, sparse_params = [], [] + for param in model.parameters(): + if param.size() == model.word_emb.weight.size(): + sparse_params.append(param) + else: + dense_params.append(param) + optimizer_sparse = optim.SGD(sparse_params, lr=args.lr * 2) + optimizer = optim.SGD(dense_params, lr=args.lr, momentum=args.mom) + else: + optimizer = optim.SGD(model.parameters(), lr=args.lr, + momentum=args.mom) +elif args.optim.lower() == 'adam': + if args.sample_softmax > 0: + dense_params, sparse_params = [], [] + for param in model.parameters(): + if param.size() == model.word_emb.weight.size(): + sparse_params.append(param) + else: + dense_params.append(param) + optimizer_sparse = optim.SparseAdam(sparse_params, lr=args.lr) + optimizer = optim.Adam(dense_params, lr=args.lr) + else: + optimizer = apex.optimizers.NpuFusedAdam(model.parameters(), lr=args.lr) +elif args.optim.lower() == 'adagrad': + optimizer = optim.Adagrad(model.parameters(), lr=args.lr) + + +################################################################################################### +opt_level = "O2" +model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level, loss_scale=128.0, combine_grad=True) +################################################################################################### + + +#### scheduler +if args.scheduler == 'cosine': + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, + args.max_step, eta_min=args.eta_min) + if args.sample_softmax > 0: + scheduler_sparse = optim.lr_scheduler.CosineAnnealingLR(optimizer_sparse, + args.max_step, eta_min=args.eta_min) +elif args.scheduler == 'inv_sqrt': + + def lr_lambda(step): + if step == 0 and args.warmup_step == 0: + return 1. + else: + return 1. / (step ** 0.5) if step > args.warmup_step \ + else step / (args.warmup_step ** 1.5) + scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda) + +elif args.scheduler == 'dev_perf': + scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, + factor=args.decay_rate, patience=args.patience, min_lr=args.lr_min) + if args.sample_softmax > 0: + scheduler_sparse = optim.lr_scheduler.ReduceLROnPlateau(optimizer_sparse, + factor=args.decay_rate, patience=args.patience, min_lr=args.lr_min) +elif args.scheduler == 'constant': + pass + + +if args.restart: + if os.path.exists(os.path.join(args.restart_dir, 'optimizer.pt')): + with open(os.path.join(args.restart_dir, 'optimizer.pt'), 'rb') as f: + opt_state_dict = torch.load(f) + optimizer.load_state_dict(opt_state_dict) + else: + print('Optimizer was not saved. Start from scratch.') + +logging('=' * 100) +for k, v in args.__dict__.items(): + logging(' - {} : {}'.format(k, v)) +logging('=' * 100) +logging('#params = {}'.format(args.n_all_param)) +logging('#non emb params = {}'.format(args.n_nonemb_param)) + +############################################################################### +# Training code +############################################################################### + +def evaluate(eval_iter): + model.eval() + if args.mem_len == 0: + model.reset_length(args.eval_tgt_len, + args.ext_len+args.tgt_len-args.eval_tgt_len, args.mem_len) + else: + model.reset_length(args.eval_tgt_len, + args.ext_len, args.mem_len+args.tgt_len-args.eval_tgt_len) + + # Evaluation + total_len, total_loss = 0, 0. + with torch.no_grad(): + mems = tuple() + for i, (data, target, seq_len) in enumerate(eval_iter): + if args.max_eval_steps > 0 and i >= args.max_eval_steps: + break + ret = model(data, target, *mems) + loss, mems = ret[0], ret[1:] + loss = loss.mean() + total_loss += seq_len * loss.float().item() + total_len += seq_len + + model.reset_length(args.tgt_len, args.ext_len, args.mem_len) + model.train() + + return total_loss / total_len + + +def train(): + global train_step, train_loss, best_val_loss, eval_start_time, log_start_time + model.train() + if args.batch_chunk > 1: + mems = [tuple() for _ in range(args.batch_chunk)] + else: + mems = tuple() + train_iter = tr_iter.get_varlen_iter() if args.varlen else tr_iter + for batch, (data, target, seq_len) in enumerate(train_iter): + model.zero_grad() + if args.batch_chunk > 1: + data_chunks = torch.chunk(data, args.batch_chunk, 1) + target_chunks = torch.chunk(target, args.batch_chunk, 1) + for i in range(args.batch_chunk): + data_i = data_chunks[i].contiguous() + target_i = target_chunks[i].contiguous() + ret = para_model(data_i, target_i, *mems[i]) + loss, mems[i] = ret[0], ret[1:] + loss = loss.float().mean().type_as(loss) / args.batch_chunk + #################################################################### + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + #################################################################### + with torch.no_grad(): + train_loss += loss.float().bool().item() + else: + ret = para_model(data, target, *mems) + loss, mems = ret[0], ret[1:] + loss = loss.float().mean().type_as(loss) + #################################################### + with torch.no_grad(): + train_loss += loss.float().item() + ################################################################### + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + + + optimizer.step() + if args.sample_softmax > 0: + optimizer_sparse.step() + + # step-wise learning rate annealing + train_step += 1 + if args.scheduler in ['cosine', 'constant', 'dev_perf']: + # linear warmup stage + if train_step < args.warmup_step: + curr_lr = args.lr * train_step / args.warmup_step + optimizer.param_groups[0]['lr'] = curr_lr + if args.sample_softmax > 0: + optimizer_sparse.param_groups[0]['lr'] = curr_lr * 2 + else: + if args.scheduler == 'cosine': + scheduler.step(train_step) + if args.sample_softmax > 0: + scheduler_sparse.step(train_step) + elif args.scheduler == 'inv_sqrt': + scheduler.step(train_step) + + if train_step % args.log_interval == 0: + cur_loss = train_loss / args.log_interval + elapsed = time.time() - log_start_time + log_str = '| epoch {:3d} step {:>8d} | {:>6d} batches | lr {:.3g} ' \ + '| ms/batch {:5.2f} | loss {:5.2f} | fps {:.2f}'.format( + epoch, train_step, batch+1, optimizer.param_groups[0]['lr'], + elapsed * 1000 / args.log_interval, cur_loss, args.log_interval*args.batch_size*args.tgt_len/elapsed) + if args.dataset in ['enwik8', 'text8']: + log_str += ' | bpc {:9.5f}'.format(cur_loss / math.log(2)) + else: + log_str += ' | ppl {:9.3f}'.format(math.exp(cur_loss)) + logging(log_str) + train_loss = 0 + log_start_time = time.time() + + if train_step % args.eval_interval == 0: + ts = time.time() + val_loss = evaluate(va_iter) + print('evaluation use time {} s'.format(time.time()-ts)) + logging('-' * 100) + log_str = '| Eval {:3d} at step {:>8d} | time: {:5.2f}s ' \ + '| valid loss {:5.2f}'.format( + train_step // args.eval_interval, train_step, + (time.time() - eval_start_time), val_loss) + if args.dataset in ['enwik8', 'text8']: + log_str += ' | bpc {:9.5f}'.format(val_loss / math.log(2)) + else: + log_str += ' | valid ppl {:9.3f}'.format(math.exp(val_loss)) + logging(log_str) + logging('-' * 100) + # Save the model if the validation loss is the best we've seen so far. + if not best_val_loss or val_loss < best_val_loss: + if not args.debug: + with open('model.pt', 'wb') as f: + torch.save(model.state_dict(), f) + with open('optimizer.pt', 'wb') as f: + torch.save(optimizer.state_dict(), f) + best_val_loss = val_loss + + # dev-performance based learning rate annealing + if args.scheduler == 'dev_perf': + scheduler.step(val_loss) + if args.sample_softmax > 0: + scheduler_sparse.step(val_loss) + + eval_start_time = time.time() + + if train_step == args.max_step: + break + +# Loop over epochs. +train_step = 0 +train_loss = 0 +best_val_loss = None + +log_start_time = time.time() +eval_start_time = time.time() + +# At any point you can hit Ctrl + C to break out of training early. +try: + for epoch in itertools.count(start=1): + train() + if train_step == args.max_step: + logging('-' * 100) + logging('End of training') + break +except KeyboardInterrupt: + logging('-' * 100) + logging('Exiting from training early') + +## Load the best saved model. +#with open('model.pt', 'rb') as f: +# model.load_state_dict(torch.load(f, map_location=device)) +#para_model = model.to(device) + +## Run on test data. +#test_loss = evaluate(te_iter) +#logging('=' * 100) +#if args.dataset in ['enwik8', 'text8']: +# logging('| End of training | test loss {:5.2f} | test bpc {:9.5f}'.format( +# test_loss, test_loss / math.log(2))) +#else: +# logging('| End of training | test loss {:5.2f} | test ppl {:9.3f}'.format( +# test_loss, math.exp(test_loss))) +#logging('=' * 100) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py new file mode 100644 index 0000000000..7ef2b16e75 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py @@ -0,0 +1,648 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import time +import math +import os, sys +import itertools +import numpy as np + +import torch +import torch.nn as nn +import torch.optim as optim +import torch.multiprocessing as mp +from data_utils import get_lm_corpus +from mem_transformer import MemTransformerLM +from utils.exp_utils import create_exp_dir +from utils.data_parallel import BalancedDataParallel +from apex import amp +import torch.distributed as dist +import apex +import warnings + + +parser = argparse.ArgumentParser(description='PyTorch Transformer Language Model') +parser.add_argument('--data', type=str, default='../data/enwik8', + help='location of the data corpus') +parser.add_argument('--dataset', type=str, default='enwik8', + choices=['wt103', 'lm1b', 'enwik8', 'text8'], + help='dataset name') +parser.add_argument('--n_layer', type=int, default=12, + help='number of total layers') +parser.add_argument('--n_head', type=int, default=8, + help='number of heads') +parser.add_argument('--d_head', type=int, default=64, + help='head dimension') +parser.add_argument('--d_embed', type=int, default=-1, + help='embedding dimension') +parser.add_argument('--d_model', type=int, default=512, + help='model dimension') +parser.add_argument('--d_inner', type=int, default=2048, + help='inner dimension in FF') +parser.add_argument('--dropout', type=float, default=0.1, + help='global dropout rate') +parser.add_argument('--dropatt', type=float, default=0.0, + help='attention probability dropout rate') +parser.add_argument('--init', default='normal', type=str, + help='parameter initializer to use.') +parser.add_argument('--emb_init', default='normal', type=str, + help='parameter initializer to use.') +parser.add_argument('--init_range', type=float, default=0.1, + help='parameters initialized by U(-init_range, init_range)') +parser.add_argument('--emb_init_range', type=float, default=0.01, + help='parameters initialized by U(-init_range, init_range)') +parser.add_argument('--init_std', type=float, default=0.02, + help='parameters initialized by N(0, init_std)') +parser.add_argument('--proj_init_std', type=float, default=0.01, + help='parameters initialized by N(0, init_std)') +parser.add_argument('--optim', default='adam', type=str, + choices=['adam', 'sgd', 'adagrad'], + help='optimizer to use.') +parser.add_argument('--lr', type=float, default=0.00025, + help='initial learning rate (0.00025|5 for adam|sgd)') +parser.add_argument('--mom', type=float, default=0.0, + help='momentum for sgd') +parser.add_argument('--scheduler', default='cosine', type=str, + choices=['cosine', 'inv_sqrt', 'dev_perf', 'constant'], + help='lr scheduler to use.') +parser.add_argument('--warmup_step', type=int, default=0, + help='upper epoch limit') +parser.add_argument('--decay_rate', type=float, default=0.5, + help='decay factor when ReduceLROnPlateau is used') +parser.add_argument('--lr_min', type=float, default=0.0, + help='minimum learning rate during annealing') +parser.add_argument('--clip', type=float, default=0.25, # 源码中 clip 的 default=0.25 + help='gradient clipping') +parser.add_argument('--clip_nonemb', action='store_true', + help='only clip the gradient of non-embedding params') +parser.add_argument('--max_step', type=int, default=400000, + help='upper epoch limit') +parser.add_argument('--batch_size', type=int, default=22, + help='batch size') +parser.add_argument('--batch_chunk', type=int, default=1, + help='split batch into chunks to save memory') +parser.add_argument('--tgt_len', type=int, default=512, + help='number of tokens to predict') +parser.add_argument('--eval_tgt_len', type=int, default=128, + help='number of tokens to predict for evaluation') +parser.add_argument('--ext_len', type=int, default=0, + help='length of the extended context') +parser.add_argument('--mem_len', type=int, default=512, + help='length of the retained previous heads') +parser.add_argument('--not_tied', action='store_true', + help='do not tie the word embedding and softmax weights') +parser.add_argument('--seed', type=int, default=1111, + help='random seed') +# parser.add_argument('--npu', default=True, help='use NPU') +parser.add_argument('--adaptive', action='store_true', + help='use adaptive softmax') +parser.add_argument('--div_val', type=int, default=1, + help='divident value for adapative input and softmax') +parser.add_argument('--pre_lnorm', action='store_true', + help='apply LayerNorm to the input instead of the output') +parser.add_argument('--varlen', action='store_true', + help='use variable length') +parser.add_argument('--multi_gpu', action='store_true', + help='use multiple GPU') +parser.add_argument('--log-interval', type=int, default=200, + help='report interval') +parser.add_argument('--eval-interval', type=int, default=4000, + help='evaluation interval') +parser.add_argument('--work_dir', default='LM-TFM', type=str, + help='experiment directory.') +parser.add_argument('--restart', action='store_true', + help='restart training from the saved checkpoint') +parser.add_argument('--restart_dir', type=str, default='', + help='restart dir') +parser.add_argument('--debug', action='store_true', + help='run in debug mode (do not create exp dir)') +parser.add_argument('--same_length', action='store_true', + help='use the same attn length for all tokens') +parser.add_argument('--attn_type', type=int, default=0, + help='attention type. 0 for ours, 1 for Shaw et al,' + '2 for Vaswani et al, 3 for Al Rfou et al.') +parser.add_argument('--clamp_len', type=int, default=-1, + help='use the same pos embeddings after clamp_len') +parser.add_argument('--eta_min', type=float, default=0.0, + help='min learning rate for cosine scheduler') +parser.add_argument('--gpu0_bsz', type=int, default=-1, + help='batch size on gpu 0') +parser.add_argument('--max_eval_steps', type=int, default=-1, + help='max eval steps') +parser.add_argument('--sample_softmax', type=int, default=-1, + help='number of samples in sampled softmax') +parser.add_argument('--patience', type=int, default=0, + help='patience') +parser.add_argument('--finetune_v2', action='store_true', + help='finetune v2') +parser.add_argument('--finetune_v3', action='store_true', + help='finetune v3') +parser.add_argument('--static-loss-scale', type=float, default=128.0, + help='Static loss scale, positive power of 2 values can ' + 'improve fp16 convergence.') +parser.add_argument('--dynamic-loss-scale', action='store_true', + help='Use dynamic loss scaling. If supplied, this argument' + ' supersedes --static-loss-scale.') +#edit this for 8p +parser.add_argument('--dist-backend', type=str, default='hccl') +parser.add_argument('--world-size', type=int, default=-1) +parser.add_argument('--rank', type=int, default=-1) +parser.add_argument('--local_rank', type=int, default=0) +parser.add_argument('--addr', type=str, default='127.0.0.1') +parser.add_argument('--device_num', type=int, default=-1) +parser.add_argument('--workers', type=int, default=32) +parser.add_argument('--device-list', default='', type=str) +parser.add_argument('--dist-url', type=str, default='tcp://127.0.0.1:50000') +parser.add_argument('--device', type=str, default='npu') +parser.add_argument('--gpu', default=None, type=int, help='GPU id to use.') +parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') +warnings.filterwarnings('ignore') +#############end################# + +def main(): + args = parser.parse_args() + args.tied = not args.not_tied + torch.manual_seed(args.seed) + + global train_step, train_loss, best_val_loss, eval_start_time, log_start_time + ############################## + # edit this for 8p + os.environ['MASTER_ADDR'] = args.addr + os.environ['MASTER_PORT'] = '29888' + os.environ['LOCAL_DEVICE_ID'] = str(0) + print("+++++++++++++++++++++++++++LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID']) + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + if args.device_list != '': + ngpus_per_node = len(args.device_list.split(',')) + elif args.device_num != -1: + ngpus_per_node = args.device_num + elif args.device == 'npu': + ngpus_per_node = int(os.environ["RANK_SIZE"]) + else: + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + args.world_size = ngpus_per_node * args.world_size + if args.device == 'npu': + main_worker(args.local_rank, ngpus_per_node,args) + else: + main_worker(args.gpu, ngpus_per_node, args) + ############################## + + +def main_worker(gpu, ngpus_per_node, args): + + global train_step, train_loss, best_val_loss, eval_start_time, log_start_time + if args.d_embed < 0: + args.d_embed = args.d_model + + assert args.ext_len >= 0, 'extended context length must be non-negative' + assert args.batch_size % args.batch_chunk == 0 + + args.work_dir = '{}-{}'.format(args.work_dir, args.dataset) + args.work_dir = os.path.join(args.work_dir, time.strftime('%Y%m%d-%H%M%S')) + logging = create_exp_dir(args.work_dir, + scripts_to_save=['train.py', 'mem_transformer.py'], debug=args.debug) + + if args.device_list != '': + args.gpu = int(args.device_list.split(',')[gpu]) + else: + args.gpu = gpu + + print("[npu id:", args.gpu, "]", "++++++++++++++++ before set LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID']) + os.environ['LOCAL_DEVICE_ID'] = str(args.gpu) + print("[npu id:", args.gpu, "]", "++++++++++++++++ LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID']) + + if args.gpu is not None: + print("[npu id:", args.gpu, "]", "Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + args.rank = args.rank * ngpus_per_node + gpu + + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, + world_size=args.world_size, rank=args.rank) + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(loc) + + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + print("[npu id:", args.gpu, "]", args) + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + + + ############################################################################### + # Load data + ############################################################################### + corpus = get_lm_corpus(args.data, args.dataset) + ntokens = len(corpus.vocab) + args.n_token = ntokens + + eval_batch_size = 10 + tr_iter = corpus.get_iterator('train', args.batch_size, args.tgt_len, + device=loc, ext_len=args.ext_len) + va_iter = corpus.get_iterator('valid', eval_batch_size, args.eval_tgt_len, + device=loc, ext_len=args.ext_len) + te_iter = corpus.get_iterator('test.py', eval_batch_size, args.eval_tgt_len, + device=loc, ext_len=args.ext_len) + + # adaptive softmax / embedding + cutoffs, tie_projs = [], [False] + if args.adaptive: + assert args.dataset in ['wt103', 'lm1b'] + if args.dataset == 'wt103': + cutoffs = [20000, 40000, 200000] + tie_projs += [True] * len(cutoffs) + elif args.dataset == 'lm1b': + cutoffs = [60000, 100000, 640000] + tie_projs += [False] * len(cutoffs) + + ############################################################################### + # Build the model + ############################################################################### + def init_weight(weight): + if args.init == 'uniform': + nn.init.uniform_(weight, -args.init_range, args.init_range) + elif args.init == 'normal': + nn.init.normal_(weight, 0.0, args.init_std) + + def init_bias(bias): + nn.init.constant_(bias, 0.0) + + def weights_init(m): + classname = m.__class__.__name__ + if classname.find('Linear') != -1: + if hasattr(m, 'weight') and m.weight is not None: + init_weight(m.weight) + if hasattr(m, 'bias') and m.bias is not None: + init_bias(m.bias) + elif classname.find('AdaptiveEmbedding') != -1: + if hasattr(m, 'emb_projs'): + for i in range(len(m.emb_projs)): + if m.emb_projs[i] is not None: + nn.init.normal_(m.emb_projs[i], 0.0, args.proj_init_std) + elif classname.find('Embedding') != -1: + if hasattr(m, 'weight'): + init_weight(m.weight) + elif classname.find('ProjectedAdaptiveLogSoftmax') != -1: + if hasattr(m, 'cluster_weight') and m.cluster_weight is not None: + init_weight(m.cluster_weight) + if hasattr(m, 'cluster_bias') and m.cluster_bias is not None: + init_bias(m.cluster_bias) + if hasattr(m, 'out_projs'): + for i in range(len(m.out_projs)): + if m.out_projs[i] is not None: + nn.init.normal_(m.out_projs[i], 0.0, args.proj_init_std) + elif classname.find('LayerNorm') != -1: + if hasattr(m, 'weight'): + nn.init.normal_(m.weight, 1.0, args.init_std) + if hasattr(m, 'bias') and m.bias is not None: + init_bias(m.bias) + elif classname.find('TransformerLM') != -1: + if hasattr(m, 'r_emb'): + init_weight(m.r_emb) + if hasattr(m, 'r_w_bias'): + init_weight(m.r_w_bias) + if hasattr(m, 'r_r_bias'): + init_weight(m.r_r_bias) + if hasattr(m, 'r_bias'): + init_bias(m.r_bias) + + def update_dropout(m): + classname = m.__class__.__name__ + if classname.find('Dropout') != -1: + if hasattr(m, 'p'): + m.p = args.dropout + + def update_dropatt(m): + if hasattr(m, 'dropatt'): + m.dropatt.p = args.dropatt + + if args.restart: + with open(os.path.join(args.restart_dir, 'model.pt'), 'rb') as f: + model = MemTransformerLM(ntokens, args.n_layer, args.n_head, args.d_model, + args.d_head, args.d_inner, args.dropout, args.dropatt, + tie_weight=args.tied, d_embed=args.d_embed, div_val=args.div_val, + tie_projs=tie_projs, pre_lnorm=args.pre_lnorm, tgt_len=args.tgt_len, + ext_len=args.ext_len, mem_len=args.mem_len, cutoffs=cutoffs, + same_length=args.same_length, attn_type=args.attn_type, + clamp_len=args.clamp_len, sample_softmax=args.sample_softmax) + model.apply(weights_init) + model.word_emb.apply(weights_init) + model = model.to(loc) + ckpt = torch.load(f, map_location=loc) + model.load_state_dict(ckpt) + model.apply(update_dropout) + model.apply(update_dropatt) + else: + model = MemTransformerLM(ntokens, args.n_layer, args.n_head, args.d_model, + args.d_head, args.d_inner, args.dropout, args.dropatt, + tie_weight=args.tied, d_embed=args.d_embed, div_val=args.div_val, + tie_projs=tie_projs, pre_lnorm=args.pre_lnorm, tgt_len=args.tgt_len, + ext_len=args.ext_len, mem_len=args.mem_len, cutoffs=cutoffs, + same_length=args.same_length, attn_type=args.attn_type, + clamp_len=args.clamp_len, sample_softmax=args.sample_softmax) + model.apply(weights_init) + model.word_emb.apply(weights_init) # ensure embedding init is not overridden by out_layer in case of weight sharing + + + + args.n_all_param = sum([p.nelement() for p in model.parameters()]) + args.n_nonemb_param = sum([p.nelement() for p in model.layers.parameters()]) + + + + #### optimizer + if args.optim.lower() == 'sgd': + if args.sample_softmax > 0: + dense_params, sparse_params = [], [] + for param in model.parameters(): + if param.size() == model.word_emb.weight.size(): + sparse_params.append(param) + else: + dense_params.append(param) + optimizer_sparse = optim.SGD(sparse_params, lr=args.lr * 2) + optimizer = optim.SGD(dense_params, lr=args.lr, momentum=args.mom) + else: + optimizer = optim.SGD(model.parameters(), lr=args.lr, + momentum=args.mom) + elif args.optim.lower() == 'adam': + if args.sample_softmax > 0: + dense_params, sparse_params = [], [] + for param in model.parameters(): + if param.size() == model.word_emb.weight.size(): + sparse_params.append(param) + else: + dense_params.append(param) + optimizer_sparse = optim.SparseAdam(sparse_params, lr=args.lr) + optimizer = optim.Adam(dense_params, lr=args.lr) + else: + #optimizer = optim.Adam(model.parameters(), lr=args.lr) + optimizer = apex.optimizers.NpuFusedAdam(model.parameters(), lr=args.lr) + elif args.optim.lower() == 'adagrad': + optimizer = optim.Adagrad(model.parameters(), lr=args.lr) + + model = model.to(loc) + ################################################################################################### + opt_level = "O2" + model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level, loss_scale=128.0, combine_grad=True) + ################################################################################################### + + if args.multi_gpu: + + if args.gpu0_bsz >= 0: + para_model = BalancedDataParallel(args.gpu0_bsz // args.batch_chunk, + model, dim=1).to(loc) + else: + para_model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False) + else: + para_model = model.to(loc) + + #### scheduler + if args.scheduler == 'cosine': + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, + args.max_step, eta_min=args.eta_min) + if args.sample_softmax > 0: + scheduler_sparse = optim.lr_scheduler.CosineAnnealingLR(optimizer_sparse, + args.max_step, eta_min=args.eta_min) + elif args.scheduler == 'inv_sqrt': + def lr_lambda(step): + if step == 0 and args.warmup_step == 0: + return 1. + else: + return 1. / (step ** 0.5) if step > args.warmup_step \ + else step / (args.warmup_step ** 1.5) + scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda) + elif args.scheduler == 'dev_perf': + scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, + factor=args.decay_rate, patience=args.patience, min_lr=args.lr_min) + if args.sample_softmax > 0: + scheduler_sparse = optim.lr_scheduler.ReduceLROnPlateau(optimizer_sparse, + factor=args.decay_rate, patience=args.patience, min_lr=args.lr_min) + elif args.scheduler == 'constant': + pass + + + if args.restart: + if os.path.exists(os.path.join(args.restart_dir, 'optimizer.pt')): + with open(os.path.join(args.restart_dir, 'optimizer.pt'), 'rb') as f: + opt_state_dict = torch.load(f, map_location=loc) + optimizer.load_state_dict(opt_state_dict) + else: + print('Optimizer was not saved. Start from scratch.') + + logging('=' * 100) + for k, v in args.__dict__.items(): + logging(' - {} : {}'.format(k, v)) + logging('=' * 100) + logging('#params = {}'.format(args.n_all_param)) + logging('#non emb params = {}'.format(args.n_nonemb_param)) + + ############################################################################### + # Training code + ############################################################################### + + def evaluate(eval_iter): + model.eval() + if args.mem_len == 0: + model.reset_length(args.eval_tgt_len, + args.ext_len+args.tgt_len-args.eval_tgt_len, args.mem_len) + else: + model.reset_length(args.eval_tgt_len, + args.ext_len, args.mem_len+args.tgt_len-args.eval_tgt_len) + + # Evaluation + total_len, total_loss = 0, 0. + with torch.no_grad(): + mems = tuple() + for i, (data, target, seq_len) in enumerate(eval_iter): + if args.max_eval_steps > 0 and i >= args.max_eval_steps: + break + ret = model(data, target, *mems) + loss, mems = ret[0], ret[1:] + loss = loss.mean() + total_loss += seq_len * loss.float().item() + total_len += seq_len + + model.reset_length(args.tgt_len, args.ext_len, args.mem_len) + model.train() + return total_loss / total_len + + + def train(): + # Turn on training mode which enables dropout. + global train_step, train_loss, best_val_loss, eval_start_time, log_start_time + + model.train() + if args.batch_chunk > 1: + mems = [tuple() for _ in range(args.batch_chunk)] + else: + mems = tuple() + train_iter = tr_iter.get_varlen_iter() if args.varlen else tr_iter + for batch, (data, target, seq_len) in enumerate(train_iter): + model.zero_grad() + if args.batch_chunk > 1: + data_chunks = torch.chunk(data, args.batch_chunk, 1) + target_chunks = torch.chunk(target, args.batch_chunk, 1) + for i in range(args.batch_chunk): + data_i = data_chunks[i].contiguous() + target_i = target_chunks[i].contiguous() + ret = para_model(data_i, target_i, *mems[i]) + loss, mems[i] = ret[0], ret[1:] + loss = loss.float().mean().type_as(loss) / args.batch_chunk + #################################################################### + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + #################################################################### + with torch.no_grad(): + train_loss += loss.float().bool().item() + else: + ret = para_model(data, target, *mems) + loss, mems = ret[0], ret[1:] + loss = loss.float().mean().type_as(loss) + #################################################### + with torch.no_grad(): + train_loss += loss.float().item() + ################################################################### + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + + + optimizer.step() + if args.sample_softmax > 0: + optimizer_sparse.step() + + # step-wise learning rate annealing + train_step += 1 + if args.scheduler in ['cosine', 'constant', 'dev_perf']: + # linear warmup stage + if train_step < args.warmup_step: + curr_lr = args.lr * train_step / args.warmup_step + optimizer.param_groups[0]['lr'] = curr_lr + if args.sample_softmax > 0: + optimizer_sparse.param_groups[0]['lr'] = curr_lr * 2 + else: + if args.scheduler == 'cosine': + scheduler.step(train_step) + if args.sample_softmax > 0: + scheduler_sparse.step(train_step) + elif args.scheduler == 'inv_sqrt': + scheduler.step(train_step) + + if train_step % args.log_interval == 0: + cur_loss = train_loss / args.log_interval + elapsed = time.time() - log_start_time + log_str = '| epoch {:3d} step {:>8d} | {:>6d} batches | lr {:.3g} ' \ + '| ms/batch {:5.2f} | loss {:5.2f} | fps {:.2f}'.format( + epoch, train_step, batch+1, optimizer.param_groups[0]['lr'], + elapsed * 1000 / args.log_interval, cur_loss, args.log_interval*args.batch_size*args.tgt_len*8/elapsed) + if args.dataset in ['enwik8', 'text8']: + log_str += ' | bpc {:9.5f}'.format(cur_loss / math.log(2)) + else: + log_str += ' | ppl {:9.3f}'.format(math.exp(cur_loss)) + logging(log_str) + train_loss = 0 + log_start_time = time.time() + + if train_step % args.eval_interval == 0: + print('train_step is :', train_step) + print('ars.eval_interval is :', args.eval_interval) + print(train_step % args.eval_interval) + print('*'*50) + ts = time.time() + val_loss = evaluate(va_iter) + print('evaluation use time {} s'.format(time.time()-ts)) + logging('-' * 100) + log_str = '| Eval {:3d} at step {:>8d} | time: {:5.2f}s ' \ + '| valid loss {:5.2f}'.format( + train_step // args.eval_interval, train_step, + (time.time() - ts), val_loss) + if args.dataset in ['enwik8', 'text8']: + log_str += ' | bpc {:9.5f}'.format(val_loss / math.log(2)) + else: + log_str += ' | valid ppl {:9.3f}'.format(math.exp(val_loss)) + logging(log_str) + logging('-' * 100) + # Save the model if the validation loss is the best we've seen so far. + if not best_val_loss or val_loss < best_val_loss: + if not args.debug: + with open('model.pt', 'wb') as f: + torch.save(model.state_dict(), f) + with open('optimizer.pt', 'wb') as f: + torch.save(optimizer.state_dict(), f) + best_val_loss = val_loss + + # dev-performance based learning rate annealing + if args.scheduler == 'dev_perf': + scheduler.step(val_loss) + if args.sample_softmax > 0: + scheduler_sparse.step(val_loss) + + eval_start_time = time.time() + + if train_step == args.max_step: + sys.exit() + + # At any point you can hit Ctrl + C to break out of training early. + try: + for epoch in itertools.count(start=1): + train() + if train_step == args.max_step: + logging('-' * 100) + logging('End of training') + sys.exit() + except KeyboardInterrupt: + logging('-' * 100) + logging('Exiting from training early') + + # # Load the best saved model. + # with open('model.pt', 'rb') as f: + # model.load_state_dict(torch.load(f, map_location=loc)) + # para_model = model.to(loc) + + # # Run on test data. + # test_loss = evaluate(te_iter) + # logging('=' * 100) + # if args.dataset in ['enwik8', 'text8']: + # logging('| End of training | test loss {:5.2f} | test bpc {:9.5f}'.format( + # test_loss, test_loss / math.log(2))) + # else: + # logging('| End of training | test loss {:5.2f} | test ppl {:9.3f}'.format( + # test_loss, math.exp(test_loss))) + # logging('=' * 100) + + +if __name__ == '__main__': + global train_step, train_loss, best_val_loss, eval_start_time, log_start_time + train_step = 0 + train_loss = 0 + best_val_loss = None + log_start_time = time.time() + eval_start_time = time.time() + main() -- Gitee From 8f0c7dbe8f3ab4515ae1a6effa8191e14c82267b Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 02:56:07 +0000 Subject: [PATCH 03/59] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20utils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/.keep diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/.keep b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/.keep new file mode 100644 index 0000000000..e69de29bb2 -- Gitee From f253465f4a62ca7359d4946294aacccbbc8bac72 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 02:56:30 +0000 Subject: [PATCH 04/59] my first commit --- .../utils/adaptive_softmax.py | 106 +++++++++++ .../utils/data_parallel.py | 109 +++++++++++ .../utils/exp_utils.py | 40 ++++ .../utils/log_uniform_sampler.py | 111 +++++++++++ .../utils/proj_adaptive_softmax.py | 160 ++++++++++++++++ .../utils/vocabulary.py | 178 ++++++++++++++++++ 6 files changed, 704 insertions(+) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/data_parallel.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/exp_utils.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/log_uniform_sampler.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/proj_adaptive_softmax.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/vocabulary.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py new file mode 100644 index 0000000000..7a0db658ab --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py @@ -0,0 +1,106 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn +import torch.nn.functional as F + +class AdaptiveLogSoftmax(nn.Module): + def __init__(self, in_features, n_classes, cutoffs, keep_order=False): + super(AdaptiveLogSoftmax, self).__init__() + + cutoffs = list(cutoffs) + + if (cutoffs != sorted(cutoffs)) \ + or (min(cutoffs) <= 0) \ + or (max(cutoffs) >= (n_classes - 1)) \ + or (len(set(cutoffs)) != len(cutoffs)) \ + or any([int(c) != c for c in cutoffs]): + + raise ValueError("cutoffs should be a sequence of unique, positive " + "integers sorted in an increasing order, where " + "each value is between 1 and n_classes-1") + + self.in_features = in_features + self.n_classes = n_classes + self.cutoffs = cutoffs + [n_classes] + + self.shortlist_size = self.cutoffs[0] + self.n_clusters = len(self.cutoffs) - 1 + self.head_size = self.shortlist_size + self.n_clusters + + self.cluster_weight = nn.Parameter(torch.zeros(self.n_clusters, self.in_features)) + self.cluster_bias = nn.Parameter(torch.zeros(self.n_clusters)) + + self.keep_order = keep_order + + + def forward(self, hidden, target, weight, bias, keep_order=False): + if hidden.size(0) != target.size(0): + raise RuntimeError('Input and target should have the same size ' + 'in the batch dimension.') + + head_weight = torch.cat( + [weight[:self.shortlist_size], self.cluster_weight], dim=0) + head_bias = torch.cat( + [bias[:self.shortlist_size], self.cluster_bias], dim=0) + + head_logit = F.linear(hidden, head_weight, bias=head_bias) + head_logprob = F.log_softmax(head_logit, dim=1) + + nll = torch.zeros_like(target, + dtype=hidden.dtype, device=hidden.device) + + offset = 0 + cutoff_values = [0] + self.cutoffs + for i in range(len(cutoff_values) - 1): + l_idx, h_idx = cutoff_values[i], cutoff_values[i + 1] + + mask_i = (target >= l_idx) & (target < h_idx) + indices_i = mask_i.nonzero().squeeze() + + if indices_i.numel() == 0: + continue + + target_i = target.index_select(0, indices_i) - l_idx + head_logprob_i = head_logprob.index_select(0, indices_i) + + if i == 0: + print(f'target_i[:,None]: {target_i[:,None]}') + print(f'target_i[:,None].shape: {target_i[:, None].shape}') + + + logprob_i = head_logprob_i.gather(1, target_i[:,None]).squeeze(1) + else: + weight_i = weight[l_idx:h_idx] + bias_i = bias[l_idx:h_idx] + + hidden_i = hidden.index_select(0, indices_i) + + tail_logit_i = F.linear(hidden_i, weight_i, bias=bias_i) + tail_logprob_i = F.log_softmax(tail_logit_i, dim=1) + + print(f'target_i[:,None]: {target_i[:, None]}') + print(f'target_i[:,None].shape: {target_i[:, None].shape}') + logprob_i = head_logprob_i[:, -i] \ + + tail_logprob_i.gather(1, target_i[:,None]).squeeze(1) + + if (hasattr(self, 'keep_order') and self.keep_order) or keep_order: + nll.index_copy_(0, indices_i, -logprob_i) + else: + nll[offset:offset+logprob_i.size(0)].copy_(-logprob_i) + + offset += logprob_i.size(0) + + return nll diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/data_parallel.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/data_parallel.py new file mode 100644 index 0000000000..1b48aaaa64 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/data_parallel.py @@ -0,0 +1,109 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from torch.nn.parallel import DataParallel +import torch +from torch.nn.parallel._functions import Scatter +from torch.nn.parallel.parallel_apply import parallel_apply + +def scatter(inputs, target_gpus, chunk_sizes, dim=0): + r""" + Slices tensors into approximately equal chunks and + distributes them across given GPUs. Duplicates + references to objects that are not tensors. + """ + def scatter_map(obj): + if isinstance(obj, torch.Tensor): + try: + return Scatter.apply(target_gpus, chunk_sizes, dim, obj) + except: + print('obj', obj.size()) + print('dim', dim) + print('chunk_sizes', chunk_sizes) + quit() + if isinstance(obj, tuple) and len(obj) > 0: + return list(zip(*map(scatter_map, obj))) + if isinstance(obj, list) and len(obj) > 0: + return list(map(list, zip(*map(scatter_map, obj)))) + if isinstance(obj, dict) and len(obj) > 0: + return list(map(type(obj), zip(*map(scatter_map, obj.items())))) + return [obj for targets in target_gpus] + + try: + return scatter_map(inputs) + finally: + scatter_map = None + +def scatter_kwargs(inputs, kwargs, target_gpus, chunk_sizes, dim=0): + r"""Scatter with support for kwargs dictionary""" + inputs = scatter(inputs, target_gpus, chunk_sizes, dim) if inputs else [] + kwargs = scatter(kwargs, target_gpus, chunk_sizes, dim) if kwargs else [] + if len(inputs) < len(kwargs): + inputs.extend([() for _ in range(len(kwargs) - len(inputs))]) + elif len(kwargs) < len(inputs): + kwargs.extend([{} for _ in range(len(inputs) - len(kwargs))]) + inputs = tuple(inputs) + kwargs = tuple(kwargs) + return inputs, kwargs + +class BalancedDataParallel(DataParallel): + def __init__(self, gpu0_bsz, *args, **kwargs): + self.gpu0_bsz = gpu0_bsz + super().__init__(*args, **kwargs) + + def forward(self, *inputs, **kwargs): + if not self.device_ids: + return self.module(*inputs, **kwargs) + if self.gpu0_bsz == 0: + device_ids = self.device_ids[1:] + else: + device_ids = self.device_ids + inputs, kwargs = self.scatter(inputs, kwargs, device_ids) + if len(self.device_ids) == 1: + return self.module(*inputs[0], **kwargs[0]) + replicas = self.replicate(self.module, self.device_ids) + if self.gpu0_bsz == 0: + replicas = replicas[1:] + outputs = self.parallel_apply(replicas, device_ids, inputs, kwargs) + + ######################################3 + # outputs=outputs.to('cpu') + # self.output_device='cpu' + print(f'outputs: {outputs}') + print(f'type(outputs): {type(outputs)}') + print(f'len(outputs): {len(outputs)}') + print(f'self.output_device: {self.output_device}') + + + return self.gather(outputs, self.output_device) + + def parallel_apply(self, replicas, device_ids, inputs, kwargs): + return parallel_apply(replicas, inputs, kwargs, device_ids) + + def scatter(self, inputs, kwargs, device_ids): + bsz = inputs[0].size(self.dim) + num_dev = len(self.device_ids) + gpu0_bsz = self.gpu0_bsz + bsz_unit = (bsz - gpu0_bsz) // (num_dev - 1) + if gpu0_bsz < bsz_unit: + chunk_sizes = [gpu0_bsz] + [bsz_unit] * (num_dev - 1) + delta = bsz - sum(chunk_sizes) + for i in range(delta): + chunk_sizes[i + 1] += 1 + if gpu0_bsz == 0: + chunk_sizes = chunk_sizes[1:] + else: + return super().scatter(inputs, kwargs, device_ids) + return scatter_kwargs(inputs, kwargs, device_ids, chunk_sizes, dim=self.dim) + diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/exp_utils.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/exp_utils.py new file mode 100644 index 0000000000..f290b8e70e --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/exp_utils.py @@ -0,0 +1,40 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import functools +import os +import shutil +import torch + + +def logging(s, log_path, print_=True, log_=True): + if print_: + print(s) + if log_: + with open(log_path, 'a+') as f_log: + f_log.write(s + '\n') + + +def get_logger(log_path, **kwargs): + return functools.partial(logging, log_path=log_path, **kwargs) + + +def create_exp_dir(dir_path, scripts_to_save=None, debug=False): + print('Experiment dir : {}'.format(dir_path)) + return get_logger(log_path='log.txt') + + +def save_checkpoint(model, optimizer, path, epoch): + torch.save(model, os.path.join(path, 'model_{}.pt'.format(epoch))) + torch.save(optimizer.state_dict(), os.path.join(path, 'optimizer_{}.pt'.format(epoch))) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/log_uniform_sampler.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/log_uniform_sampler.py new file mode 100644 index 0000000000..4ebe129747 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/log_uniform_sampler.py @@ -0,0 +1,111 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +from torch import nn + +class LogUniformSampler(object): + def __init__(self, range_max, n_sample): + """ + Reference : https://github.com/tensorflow/tensorflow/blob/r1.10/tensorflow/python/ops/candidate_sampling_ops.py + `P(class) = (log(class + 2) - log(class + 1)) / log(range_max + 1)` + + expected count can be approximated by 1 - (1 - p)^n + and we use a numerically stable version -expm1(num_tries * log1p(-p)) + + Our implementation fixes num_tries at 2 * n_sample, and the actual #samples will vary from run to run + """ + with torch.no_grad(): + self.range_max = range_max + log_indices = torch.arange(1., range_max+2., 1.).log_() + self.dist = (log_indices[1:] - log_indices[:-1]) / log_indices[-1] + # print('P', self.dist.numpy().tolist()[-30:]) + + self.log_q = (- (-self.dist.double().log1p_() * 2 * n_sample).expm1_()).log_().float() + + self.n_sample = n_sample + + def sample(self, labels): + """ + labels: [b1, b2] + Return + true_log_probs: [b1, b2] + samp_log_probs: [n_sample] + neg_samples: [n_sample] + """ + + n_sample = self.n_sample + n_tries = 2 * n_sample + + with torch.no_grad(): + neg_samples = torch.multinomial(self.dist, n_tries, replacement=True).unique() + device = labels.device + neg_samples = neg_samples.to(device) + true_log_probs = self.log_q[labels].to(device) + samp_log_probs = self.log_q[neg_samples].to(device) + return true_log_probs, samp_log_probs, neg_samples + +def sample_logits(embedding, bias, labels, inputs, sampler): + """ + embedding: an nn.Embedding layer + bias: [n_vocab] + labels: [b1, b2] + inputs: [b1, b2, n_emb] + sampler: you may use a LogUniformSampler + Return + logits: [b1, b2, 1 + n_sample] + """ + true_log_probs, samp_log_probs, neg_samples = sampler.sample(labels) + n_sample = neg_samples.size(0) + b1, b2 = labels.size(0), labels.size(1) + all_ids = torch.cat([labels.view(-1), neg_samples]) + all_w = embedding(all_ids) + true_w = all_w[: -n_sample].view(b1, b2, -1) + sample_w = all_w[- n_sample:].view(n_sample, -1) + + all_b = bias[all_ids] + true_b = all_b[: -n_sample].view(b1, b2) + sample_b = all_b[- n_sample:] + + hit = (labels[:, :, None] == neg_samples).detach() + + true_logits = torch.einsum('ijk,ijk->ij', + [true_w, inputs]) + true_b - true_log_probs + sample_logits = torch.einsum('lk,ijk->ijl', + [sample_w, inputs]) + sample_b - samp_log_probs + sample_logits.masked_fill_(hit, -1e30) + logits = torch.cat([true_logits[:, :, None], sample_logits], -1) + + return logits + + +if __name__ == '__main__': + S, B = 3, 4 + n_vocab = 10000 + n_sample = 5 + H = 32 + + labels = torch.LongTensor(S, B).random_(0, n_vocab) + sampler = LogUniformSampler(n_vocab, unique=True) + + embedding = nn.Embedding(n_vocab, H) + bias = torch.zeros(n_vocab) + inputs = torch.Tensor(S, B, H).normal_() + + logits, out_labels = sample_logits(embedding, bias, labels, inputs, sampler, n_sample) + print('logits', logits.detach().numpy().tolist()) + print('logits shape', logits.size()) + print('out_labels', out_labels.detach().numpy().tolist()) + print('out_labels shape', out_labels.size()) + diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/proj_adaptive_softmax.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/proj_adaptive_softmax.py new file mode 100644 index 0000000000..886757190c --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/proj_adaptive_softmax.py @@ -0,0 +1,160 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn +import torch.nn.functional as F + +#################################################################################### +# edit +# CUDA_MAJOR = int(torch.version.cuda.split('.')[0]) # 主cuda +# CUDA_MINOR = int(torch.version.cuda.split('.')[1]) # 辅cuda +##################################################################################### + +class ProjectedAdaptiveLogSoftmax(nn.Module): + def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, + keep_order=False): + super(ProjectedAdaptiveLogSoftmax, self).__init__() + self.n_token = n_token + self.d_embed = d_embed + self.d_proj = d_proj + + self.cutoffs = cutoffs + [n_token] + self.cutoff_ends = [0] + self.cutoffs + self.div_val = div_val + + self.shortlist_size = self.cutoffs[0] + self.n_clusters = len(self.cutoffs) - 1 + self.head_size = self.shortlist_size + self.n_clusters + + if self.n_clusters > 0: + self.cluster_weight = nn.Parameter(torch.zeros(self.n_clusters, self.d_embed)) + self.cluster_bias = nn.Parameter(torch.zeros(self.n_clusters)) + + self.out_layers = nn.ModuleList() + self.out_projs = nn.ParameterList() + + if div_val == 1: + for i in range(len(self.cutoffs)): + if d_proj != d_embed: + self.out_projs.append( + nn.Parameter(torch.Tensor(d_proj, d_embed)) + ) + else: + self.out_projs.append(None) + + self.out_layers.append(nn.Linear(d_embed, n_token)) + else: + for i in range(len(self.cutoffs)): + l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i+1] + d_emb_i = d_embed // (div_val ** i) + + self.out_projs.append( + nn.Parameter(torch.Tensor(d_proj, d_emb_i)) + ) + + self.out_layers.append(nn.Linear(d_emb_i, r_idx-l_idx)) + + self.keep_order = keep_order + + def _compute_logit(self, hidden, weight, bias, proj): + if proj is None: + logit = F.linear(hidden, weight, bias=bias) + else: + proj_hid = F.linear(hidden, proj.t().contiguous()) + logit = F.linear(proj_hid, weight, bias=bias) + + return logit + + def forward(self, hidden, target, keep_order=False): + ''' + hidden :: [len*bsz x d_proj] + target :: [len*bsz] + ''' + + if hidden.size(0) != target.size(0): + raise RuntimeError('Input and target should have the same size ' + 'in the batch dimension.') + + if self.n_clusters == 0: + logit = self._compute_logit(hidden, self.out_layers[0].weight, + self.out_layers[0].bias, self.out_projs[0]) + + torch.save(logit,"logit.pt") + + nll = -F.log_softmax(logit, dim=-1).gather(1, target.unsqueeze(1).long()).squeeze(1) + + else: + weights, biases = [], [] + for i in range(len(self.cutoffs)): + if self.div_val == 1: + l_idx, r_idx = self.cutoff_ends[i], self.cutoff_ends[i + 1] + weight_i = self.out_layers[0].weight[l_idx:r_idx] + bias_i = self.out_layers[0].bias[l_idx:r_idx] + else: + weight_i = self.out_layers[i].weight + bias_i = self.out_layers[i].bias + + if i == 0: + weight_i = torch.cat( + [weight_i, self.cluster_weight], dim=0) + bias_i = torch.cat( + [bias_i, self.cluster_bias], dim=0) + + weights.append(weight_i) + biases.append(bias_i) + + head_weight, head_bias, head_proj = weights[0], biases[0], self.out_projs[0] + + head_logit = self._compute_logit(hidden, head_weight, head_bias, head_proj) + head_logprob = F.log_softmax(head_logit, dim=1) + + nll = torch.zeros_like(target, + dtype=hidden.dtype, device=hidden.device) + + offset = 0 + cutoff_values = [0] + self.cutoffs + for i in range(len(cutoff_values) - 1): + l_idx, r_idx = cutoff_values[i], cutoff_values[i + 1] + + mask_i = (target >= l_idx) & (target < r_idx) + indices_i = mask_i.nonzero().squeeze() + + if indices_i.numel() == 0: + continue + + target_i = target.index_select(0, indices_i) - l_idx + head_logprob_i = head_logprob.index_select(0, indices_i) + + if i == 0: + logprob_i = head_logprob_i.gather(1, target_i[:,None]).squeeze(1) + else: + weight_i, bias_i, proj_i = weights[i], biases[i], self.out_projs[i] + + hidden_i = hidden.index_select(0, indices_i) + + tail_logit_i = self._compute_logit(hidden_i, weight_i, bias_i, proj_i) + tail_logprob_i = F.log_softmax(tail_logit_i, dim=1) + + logprob_i = head_logprob_i[:, -i] \ + + tail_logprob_i.gather(1, target_i[:,None]).squeeze(1) + + if (hasattr(self, 'keep_order') and self.keep_order) or keep_order: + nll.index_copy_(0, indices_i, -logprob_i) + else: + nll[offset:offset+logprob_i.size(0)].copy_(-logprob_i) + + offset += logprob_i.size(0) + + return nll diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/vocabulary.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/vocabulary.py new file mode 100644 index 0000000000..a13e4183c2 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/vocabulary.py @@ -0,0 +1,178 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from collections import Counter, OrderedDict +import torch + + +class Vocab(object): + def __init__(self, special=[], min_freq=0, max_size=None, lower_case=True, + delimiter=None, vocab_file=None): + self.counter = Counter() + self.special = special + self.min_freq = min_freq + self.max_size = max_size + self.lower_case = lower_case + self.delimiter = delimiter + self.vocab_file = vocab_file + + + def tokenize(self, line, add_eos=False, add_double_eos=False): + line = line.strip() + if self.lower_case: + line = line.lower() + + if self.delimiter == '': + symbols = line + else: + symbols = line.split(self.delimiter) + + if add_double_eos: # lm1b + return [''] + symbols + [''] + elif add_eos: + return symbols + [''] + else: + return symbols + + + + def count_file(self, path, verbose=False, add_eos=False): + if verbose: print('counting file {} ...'.format(path)) + assert os.path.exists(path) + + sents = [] + with open(path, 'r', encoding='utf-8') as f: + for idx, line in enumerate(f): + if verbose and idx > 0 and idx % 500000 == 0: + print(' line {}'.format(idx)) + symbols = self.tokenize(line, add_eos=add_eos) + self.counter.update(symbols) + sents.append(symbols) + + return sents + + + def count_sents(self, sents, verbose=False): + """ + sents : a list of sentences, each a list of tokenized symbols + """ + if verbose: print('counting {} sents ...'.format(len(sents))) + for idx, symbols in enumerate(sents): + if verbose and idx > 0 and idx % 500000 == 0: + print(' line {}'.format(idx)) + self.counter.update(symbols) + + def _build_from_file(self, vocab_file): + self.idx2sym = [] + self.sym2idx = OrderedDict() + + with open(vocab_file, 'r', encoding='utf-8') as f: + for line in f: + symb = line.strip().split()[0] + self.add_symbol(symb) + self.unk_idx = self.sym2idx[''] + + def build_vocab(self): + if self.vocab_file: + print('building vocab from {}'.format(self.vocab_file)) + self._build_from_file(self.vocab_file) + print('final vocab size {}'.format(len(self))) + else: + print('building vocab with min_freq={}, max_size={}'.format( + self.min_freq, self.max_size)) + self.idx2sym = [] + self.sym2idx = OrderedDict() + + for sym in self.special: + self.add_special(sym) + + for sym, cnt in self.counter.most_common(self.max_size): + if cnt < self.min_freq: break + self.add_symbol(sym) + + print('final vocab size {} from {} unique tokens'.format( + len(self), len(self.counter))) + + def encode_file(self, path, ordered=False, verbose=False, add_eos=True, + add_double_eos=False): + if verbose: print('encoding file {} ...'.format(path)) + assert os.path.exists(path) + encoded = [] + with open(path, 'r', encoding='utf-8') as f: + for idx, line in enumerate(f): + if verbose and idx > 0 and idx % 500000 == 0: + print(' line {}'.format(idx)) + symbols = self.tokenize(line, add_eos=add_eos, + add_double_eos=add_double_eos) + encoded.append(self.convert_to_tensor(symbols)) + + if ordered: + encoded = torch.cat(encoded) + + return encoded + + def encode_sents(self, sents, ordered=False, verbose=False): + if verbose: print('encoding {} sents ...'.format(len(sents))) + encoded = [] + for idx, symbols in enumerate(sents): + if verbose and idx > 0 and idx % 500000 == 0: + print(' line {}'.format(idx)) + encoded.append(self.convert_to_tensor(symbols)) + + if ordered: + encoded = torch.cat(encoded) + + return encoded + + def add_special(self, sym): + if sym not in self.sym2idx: + self.idx2sym.append(sym) + self.sym2idx[sym] = len(self.idx2sym) - 1 + setattr(self, '{}_idx'.format(sym.strip('<>')), self.sym2idx[sym]) + + def add_symbol(self, sym): + if sym not in self.sym2idx: + self.idx2sym.append(sym) + self.sym2idx[sym] = len(self.idx2sym) - 1 + + def get_sym(self, idx): + assert 0 <= idx < len(self), 'Index {} out of range'.format(idx) + return self.idx2sym[idx] + + def get_idx(self, sym): + if sym in self.sym2idx: + return self.sym2idx[sym] + else: + assert '' not in sym + assert hasattr(self, 'unk_idx') + return self.sym2idx.get(sym, self.unk_idx) + + def get_symbols(self, indices): + return [self.get_sym(idx) for idx in indices] + + def get_indices(self, symbols): + return [self.get_idx(sym) for sym in symbols] + + def convert_to_tensor(self, symbols): + return torch.LongTensor(self.get_indices(symbols)) + + def convert_to_sent(self, indices, exclude=None): + if exclude is None: + return ' '.join([self.get_sym(idx) for idx in indices]) + else: + return ' '.join([self.get_sym(idx) for idx in indices if idx not in exclude]) + + def __len__(self): + return len(self.idx2sym) -- Gitee From 31e0519a73244717a60ec6b3a989848988b546c6 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 02:56:44 +0000 Subject: [PATCH 05/59] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/.keep diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/.keep b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/.keep new file mode 100644 index 0000000000..e69de29bb2 -- Gitee From 0a50aef3458bd911484a851ccf5f0dff31e9c858 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 02:57:37 +0000 Subject: [PATCH 06/59] my first commit --- .../test/env_npu.sh | 71 +++++++++ .../test/train_eval_1p.sh | 119 ++++++++++++++ .../test/train_full_8p.sh | 144 +++++++++++++++++ .../test/train_performance_1p.sh | 150 ++++++++++++++++++ .../test/train_performance_8p.sh | 149 +++++++++++++++++ 5 files changed, 633 insertions(+) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/env_npu.sh create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/env_npu.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/env_npu.sh new file mode 100644 index 0000000000..280fca96da --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/env_npu.sh @@ -0,0 +1,71 @@ +#!/bin/bash +export install_path=/usr/local/Ascend + +if [ -d ${install_path}/toolkit ]; then + export LD_LIBRARY_PATH=/usr/include/hdf5/lib/:/usr/local/:/usr/local/lib/:/usr/lib/:${install_path}/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons:${path_lib}:${LD_LIBRARY_PATH} + export PATH=${install_path}/fwkacllib/ccec_compiler/bin:${install_path}/fwkacllib/bin:$PATH + export PYTHONPATH=${install_path}/fwkacllib/python/site-packages:${install_path}/tfplugin/python/site-packages:${install_path}/toolkit/python/site-packages:$PYTHONPATH + export PYTHONPATH=/usr/local/python3.7.5/lib/python3.7/site-packages:$PYTHONPATH + export ASCEND_OPP_PATH=${install_path}/opp +else + if [ -d ${install_path}/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:${install_path}/nnae/latest/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:${install_path}/nnae/latest/fwkacllib/ccec_compiler/bin/:${install_path}/nnae/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=${install_path}/nnae/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=${install_path}/nnae/latest/fwkacllib/python/site-packages/:${install_path}/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:${install_path}/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + export ASCEND_AICPU_PATH=${install_path}/nnae/latest + else + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:${install_path}/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:${install_path}/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=${install_path}/ascend-toolkit/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/:${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + export ASCEND_AICPU_PATH=${install_path}/ascend-toolkit/latest + fi +fi + + +#将Host日志输出到串口,0-关闭/1-开启 +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +#设置默认日志级别,0-debug/1-info/2-warning/3-error +export ASCEND_GLOBAL_LOG_LEVEL=3 +#设置Host侧Event日志开启标志,0-关闭/1-开启 +export ASCEND_GLOBAL_EVENT_ENABLE=0 +#设置是否开启taskque,0-关闭/1-开启 +export TASK_QUEUE_ENABLE=1 +#设置是否开启PTCopy,0-关闭/1-开启 +export PTCOPY_ENABLE=1 +#设置是否开启combined标志,0-关闭/1-开启 +export COMBINED_ENABLE=1 +#设置特殊场景是否需要重新编译,不需要修改 +export DYNAMIC_OP="ADD#MUL" +#HCCL白名单开关,1-关闭/0-开启 +export HCCL_WHITELIST_DISABLE=1 +#设置Device侧日志等级为error +${install_path}/driver/tools/msnpureport -g error +#关闭Device侧Event日志 +${install_path}/driver/tools/msnpureport -e disable +export BMMV2_ENABLE=1 + +path_lib=$(python3.7 -c """ +import sys +import re +result='' +for index in range(len(sys.path)): + match_sit = re.search('-packages', sys.path[index]) + if match_sit is not None: + match_lib = re.search('lib', sys.path[index]) + + if match_lib is not None: + end=match_lib.span()[1] + result += sys.path[index][0:end] + ':' + + result+=sys.path[index] + '/torch/lib:' +print(result)""" +) + +echo ${path_lib} + +export LD_LIBRARY_PATH=/usr/local/python3.7.5/lib/:${path_lib}:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/gcc7.3.0/lib64:${LD_LIBRARY_PATH} \ No newline at end of file diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh new file mode 100644 index 0000000000..da30622701 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +################基础配置参数，需要模型审视修改################## +# 必选字段(必须在此处定义的参数): Network batch_size RANK_SIZE +# 网络名称，同目录名称 +Network="Transformer_XL_for_PyTorch" +# 训练batch_size +batch_size=22 +# 训练使用的npu卡数 +export RANK_SIZE=1 +# 数据集路径,保持为空,不需要修改 +data_path="/home/huangwei/data/enwik8" +# checkpoint文件路径，以实际路径为准 +pth_path="/home/huangwei/transformer-xl" +# 训练epoch +train_epochs=50 +# 指定训练所使用的npu device卡id +device_id=0 +# 加载数据进程数 +workers=128 + + +# 参数校验，data_path为必传参数，其他参数的增删由模型自身决定；此处新增参数需在上面有定义并赋值 +for para in $* +do + if [[ $para == --device_id* ]];then + device_id=`echo ${para#*=}` + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi +# 校验是否指定了device_id,分动态分配device_id与手动指定device_id,此处不需要修改 +if [ $ASCEND_DEVICE_ID ];then + echo "device id is ${ASCEND_DEVICE_ID}" +elif [ ${device_id} ];then + export ASCEND_DEVICE_ID=${device_id} + echo "device id is ${ASCEND_DEVICE_ID}" +else + "[Error] device id must be config" + exit 1 +fi + + + +###############指定训练脚本执行路径############### +# cd到与test文件夹同层级目录下执行脚本，提高兼容性；test_path_dir为包含test文件夹的路径 +cur_path=`pwd` +cur_path_last_dirname=${cur_path##*/} +if [ x"${cur_path_last_dirname}" == x"test" ];then + test_path_dir=${cur_path} + cd .. + cur_path=`pwd` +else + test_path_dir=${cur_path}/test +fi + + +#################创建日志输出目录，不需要修改################# +if [ -d ${test_path_dir}/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${test_path_dir}/output/${ASCEND_DEVICE_ID} + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +else + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +fi + + +#################启动训练脚本################# +#训练开始时间，不需要修改 +start_time=$(date +%s) +# 非平台场景时source 环境变量 +check_etp_flag=`env | grep etp_running_flag` +etp_flag=`echo ${check_etp_flag#*=}` +if [ x"${etp_flag}" != x"true" ];then + source ${test_path_dir}/env_npu.sh +fi +python3 -u eval_npu.py --split valid > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/Eval_${ASCEND_DEVICE_ID}.log 2>&1 & + +wait + + +##################获取训练数据################ +#训练结束时间，不需要修改 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" + +#输出训练精度,需要模型审视修改 +train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/Eval_${ASCEND_DEVICE_ID}.log|grep -v test|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'|awk 'END {print}'` +#打印，不需要修改 +echo "Final Train bpc : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'perf' + + +#最后一个迭代loss值，不需要修改 +#ActualLoss=`awk 'END {print}' ${test_path_dir}/output/$ASCEND_DEVICE_ID/Eval_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh new file mode 100644 index 0000000000..90303323f0 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +################基础配置参数，需要模型审视修改################## +# 必选字段(必须在此处定义的参数): Network batch_size RANK_SIZE +# 网络名称，同目录名称 +Network="Transformer_XL_for_PyTorch" +# 训练batch_size +batch_size=22 +# 训练使用的npu卡数 +export RANK_SIZE=8 +# 数据集路径,保持为空,不需要修改 +data_path="/home/huangwei/data/enwik8" + +# 训练epoch +train_epochs=40 +# 学习率 +learning_rate=0.00025 +# 加载数据进程数 +workers=124 + + +# 参数校验，data_path为必传参数，其他参数的增删由模型自身决定；此处新增参数需在上面有定义并赋值 +for para in $* +do + if [[ $para == --workers* ]];then + workers=`echo ${para#*=}` + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi + + +###############指定训练脚本执行路径############### +# cd到与test文件夹同层级目录下执行脚本，提高兼容性；test_path_dir为包含test文件夹的路径 +cur_path=`pwd` +cur_path_last_diename=${cur_path##*/} +if [ x"${cur_path_last_diename}" == x"test" ];then + test_path_dir=${cur_path} + cd .. + cur_path=`pwd` +else + test_path_dir=${cur_path}/test +fi + + +#################创建日志输出目录，不需要修改################# +ASCEND_DEVICE_ID=0 +if [ -d ${test_path_dir}/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${test_path_dir}/output/${ASCEND_DEVICE_ID} + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +else + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +fi + + +#################启动训练脚本################# +#训练开始时间，不需要修改 +start_time=$(date +%s) +# 非平台场景时source 环境变量 +check_etp_flag=`env | grep etp_running_flag` +etp_flag=`echo ${check_etp_flag#*=}` +if [ x"${etp_flag}" != x"true" ];then + source ${test_path_dir}/env_npu.sh +fi + +device_id_list=0,1,2,3,4,5,6,7 +export RANK_SIZE=8 +currentDir=$(cd "$(dirname "$0")";pwd) +KERNEL_NUM=$(($(nproc)/8)) +for i in $(seq 0 7) +do + PID_START=$((KERNEL_NUM * i)) + PID_END=$((PID_START + KERNEL_NUM - 1)) + taskset -c $PID_START-$PID_END python3.7 -u $(dirname $currentDir)/train_8p_npu.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --workers=$(nproc) \ + --multiprocessing-distributed \ + --dist-url='tcp://127.0.0.1:50000' \ + --dist-backend='hccl' \ + --world-size=1 \ + --device_num=8 \ + --max_step=400000 \ + --rank=0 \ + --device-list=${device_id_list} \ + --local_rank=$i > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & +done + + +wait + + + +##################获取训练数据################ +#训练结束时间，不需要修改 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +FPS=`grep -a 'fps' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk -F " " '{print $20}'|awk 'END {print}'` +#打印，不需要修改 +echo "Final Performance images/sec : $FPS" + +#输出训练精度,需要模型审视修改 +train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk 'END {print}'|awk -F "|" '{print $NF}'|awk -F " " '{print $1}'` +#打印，不需要修改 +echo "Final Train Accuracy : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'perf' + +##获取性能数据，不需要修改 +#吞吐量 +ActualFPS=${FPS} +#单迭代训练时长 +TrainingTime=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'*1000/'${FPS}'}'` + +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视 +grep epoch ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|grep -v test|awk -F "|" '{print $6}' | awk -F " " '{print $NF}' >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt + +#最后一个迭代loss值，不需要修改 +ActualLoss=`awk 'END {print}' ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh new file mode 100644 index 0000000000..ecd1b34c35 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh @@ -0,0 +1,150 @@ +#!/bin/bash + +################基础配置参数，需要模型审视修改################## +# 必选字段(必须在此处定义的参数): Network batch_size RANK_SIZE +# 网络名称，同目录名称 +Network="Transformer_XL_for_PyTorch" +# 训练batch_size +batch_size=22 +# 训练使用的npu卡数 +export RANK_SIZE=1 +# 数据集路径,保持为空,不需要修改 +data_path="/home/huangwei/data/enwik8" + +# 训练epoch +train_epochs=1 +# 指定训练所使用的npu device卡id +device_id=0 +# 加载数据进程数 +workers=128 + + +# 参数校验，data_path为必传参数，其他参数的增删由模型自身决定；此处新增参数需在上面有定义并赋值 +for para in $* +do + if [[ $para == --device_id* ]];then + device_id=`echo ${para#*=}` + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi +# 校验是否指定了device_id,分动态分配device_id与手动指定device_id,此处不需要修改 +if [ $ASCEND_DEVICE_ID ];then + echo "device id is ${ASCEND_DEVICE_ID}" +elif [ ${device_id} ];then + export ASCEND_DEVICE_ID=${device_id} + echo "device id is ${ASCEND_DEVICE_ID}" +else + "[Error] device id must be config" + exit 1 +fi + + + +###############指定训练脚本执行路径############### +# cd到与test文件夹同层级目录下执行脚本，提高兼容性；test_path_dir为包含test文件夹的路径 +cur_path=`pwd` +cur_path_last_dirname=${cur_path##*/} +if [ x"${cur_path_last_dirname}" == x"test" ];then + test_path_dir=${cur_path} + cd .. + cur_path=`pwd` +else + test_path_dir=${cur_path}/test +fi + + +#################创建日志输出目录，不需要修改################# +if [ -d ${test_path_dir}/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${test_path_dir}/output/${ASCEND_DEVICE_ID} + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +else + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +fi + + +#################启动训练脚本################# +#训练开始时间，不需要修改 +start_time=$(date +%s) +# 非平台场景时source 环境变量 +check_etp_flag=`env | grep etp_running_flag` +etp_flag=`echo ${check_etp_flag#*=}` +if [ x"${etp_flag}" != x"true" ];then + source ${test_path_dir}/env_npu.sh +fi +python3.7 -u ./train_1p_npu.py \ + --data=${data_path} \ + --seed=1111 \ + --workers=${workers} \ + --gpu=${ASCEND_DEVICE_ID} \ + --eval-interval=4000 \ + --log-interval=1 \ + --max_step=100 \ + --epochs=${train_epochs} \ + --static-loss-scale=128 \ + --batch_size=${batch_size} > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & + +wait + + +##################获取训练数据################ +#训练结束时间，不需要修改 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +#FPS=`grep -a 'FPS' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk -F " " '{print $11}'|awk 'END {print}'` +#FPS=${FPS#* } + +grep "fps" ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log | awk '{print $20}' >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_${ASCEND_DEVICE_ID}_fps.log +FPS=`cat ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}_fps.log | awk '{a+=$1} END {if (NR != 0) printf("%.3f",a/NR)}'` + +#打印，不需要修改 +echo "Final Performance characters/sec : $FPS" + +#输出训练精度,需要模型审视修改 +train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|grep -v test|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'|awk 'END {print}'` +#打印，不需要修改 +echo "Final Train bpc : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'perf' + +##获取性能数据，不需要修改 +#吞吐量 +ActualFPS=${FPS} + +TrainingTime=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'*1000}'` + +#单迭代训练时长 +TrainingTime=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'*1000/'${FPS}'}'` + +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视 +grep "epoch" ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|grep -v test|awk -F "|" '{print $6}' | awk -F " " '{print $NF}' >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt + +#最后一个迭代loss值，不需要修改 +ActualLoss=`awk 'END {print}' ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log + + diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh new file mode 100644 index 0000000000..37c3a430f4 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +################基础配置参数，需要模型审视修改################## +# 必选字段(必须在此处定义的参数): Network batch_size RANK_SIZE +# 网络名称，同目录名称 +Network="Transformer_XL_for_PyTorch" +# 训练batch_size +batch_size=22 +# 训练使用的npu卡数 +export RANK_SIZE=8 +# 数据集路径,保持为空,不需要修改 +data_path="/home/huangwei/data/enwik8" + +# 训练epoch +train_epochs=50 +# 学习率 +learning_rate=0.00025 +# 加载数据进程数 +workers=124 + + +# 参数校验，data_path为必传参数，其他参数的增删由模型自身决定；此处新增参数需在上面有定义并赋值 +for para in $* +do + if [[ $para == --workers* ]];then + workers=`echo ${para#*=}` + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi + + +###############指定训练脚本执行路径############### +# cd到与test文件夹同层级目录下执行脚本，提高兼容性；test_path_dir为包含test文件夹的路径 +cur_path=`pwd` +cur_path_last_diename=${cur_path##*/} +if [ x"${cur_path_last_diename}" == x"test" ];then + test_path_dir=${cur_path} + cd .. + cur_path=`pwd` +else + test_path_dir=${cur_path}/test +fi + + +#################创建日志输出目录，不需要修改################# +ASCEND_DEVICE_ID=0 +if [ -d ${test_path_dir}/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${test_path_dir}/output/${ASCEND_DEVICE_ID} + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +else + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +fi + + +#################启动训练脚本################# +#训练开始时间，不需要修改 +start_time=$(date +%s) +# 非平台场景时source 环境变量 +check_etp_flag=`env | grep etp_running_flag` +etp_flag=`echo ${check_etp_flag#*=}` +if [ x"${etp_flag}" != x"true" ];then + source ${test_path_dir}/env_npu.sh +fi + +device_id_list=0,1,2,3,4,5,6,7 +export RANK_SIZE=8 +currentDir=$(cd "$(dirname "$0")";pwd) +KERNEL_NUM=$(($(nproc)/8)) +for i in $(seq 0 7) +do + PID_START=$((KERNEL_NUM * i)) + PID_END=$((PID_START + KERNEL_NUM - 1)) + taskset -c $PID_START-$PID_END python3.7 -u $(dirname $currentDir)/train_8p_npu.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --workers=$(nproc) \ + --multiprocessing-distributed \ + --dist-url='tcp://127.0.0.1:50000' \ + --dist-backend='hccl' \ + --world-size=1 \ + --device_num=8 \ + --log-interval=1 \ + --eval-interval=4000 \ + --max_step=100 \ + --rank=0 \ + --device-list=${device_id_list} \ + --local_rank=$i > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & +done + + +wait + + + +##################获取训练数据################ +#训练结束时间，不需要修改 + +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +grep "fps" ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log | awk '{print $20}' > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}_fps.log +FPS=`cat ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}_fps.log|awk '{a+=$1} END {if (NR !=0) printf("%.3f", a/NR)}'` +#打印，不需要修改 +echo "Final Performance images/sec : $FPS" + +#输出训练精度,需要模型审视修改 +train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|grep -v test|awk 'END {print}'|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'` + +#打印，不需要修改 +echo "Final Train Accuracy : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'perf' + +##获取性能数据，不需要修改 +#吞吐量 +ActualFPS=${FPS} +#单迭代训练时长 +TrainingTime=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'*1000/'${FPS}'}'` + +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视 +grep "epoch" ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_$ASCEND_DEVICE_ID.log|grep -v test|awk -F "|" '{print $6}' | awk -F " " '{print $NF}' > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${CaseName}_loss.txt + +#最后一个迭代loss值，不需要修改 +ActualLoss=`awk 'END {print}' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log -- Gitee From ad9dc0e98ad998df4b1bdd3d8e61a22e94a444a1 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 02:58:41 +0000 Subject: [PATCH 07/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md. --- PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md index 0412217fa7..4d220f9310 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md @@ -35,7 +35,7 @@ bash test/train_performance_8p.sh bash test/train_full_8p.sh # 1p eval -bash test/eval_1p.sh +bash test/train_eval_1p.sh ``` -- Gitee From 6d49906bfe5d21404a4a29df1c731fe4f6683ca7 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 03:00:41 +0000 Subject: [PATCH 08/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt. --- .../nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt index 559473df33..82f29898a4 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/modelzoo_level.txt @@ -1,3 +1,3 @@ -FuncStatus: -PerfStatus: -PrecisionStatus: \ No newline at end of file +FuncStatus:OK +PerfStatus:POK +PrecisionStatus:POK \ No newline at end of file -- Gitee From 13d2ad05ee1e76af9dcbb85395a896143a6cdc71 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 03:59:31 +0000 Subject: [PATCH 09/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/nlp/Transformer-xl=5Ffor=5FPyTorch/adaptive=5Fsof?= =?UTF-8?q?tmax.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adaptive_softmax.py | 105 ------------------ 1 file changed, 105 deletions(-) delete mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py deleted file mode 100644 index 8360772d99..0000000000 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py +++ /dev/null @@ -1,105 +0,0 @@ -from collections import defaultdict - -import numpy as np - -import torch -import torch.nn as nn -import torch.nn.functional as F - -class AdaptiveLogSoftmax(nn.Module): - def __init__(self, in_features, n_classes, cutoffs, keep_order=False): - super(AdaptiveLogSoftmax, self).__init__() - - cutoffs = list(cutoffs) - - if (cutoffs != sorted(cutoffs)) \ - or (min(cutoffs) <= 0) \ - or (max(cutoffs) >= (n_classes - 1)) \ - or (len(set(cutoffs)) != len(cutoffs)) \ - or any([int(c) != c for c in cutoffs]): - - raise ValueError("cutoffs should be a sequence of unique, positive " - "integers sorted in an increasing order, where " - "each value is between 1 and n_classes-1") - - self.in_features = in_features - self.n_classes = n_classes - self.cutoffs = cutoffs + [n_classes] - - self.shortlist_size = self.cutoffs[0] - self.n_clusters = len(self.cutoffs) - 1 - self.head_size = self.shortlist_size + self.n_clusters - - self.cluster_weight = nn.Parameter(torch.zeros(self.n_clusters, self.in_features)) - self.cluster_bias = nn.Parameter(torch.zeros(self.n_clusters)) - - self.keep_order = keep_order - - - def forward(self, hidden, target, weight, bias, keep_order=False): - if hidden.size(0) != target.size(0): - raise RuntimeError('Input and target should have the same size ' - 'in the batch dimension.') - - head_weight = torch.cat( - [weight[:self.shortlist_size], self.cluster_weight], dim=0) - head_bias = torch.cat( - [bias[:self.shortlist_size], self.cluster_bias], dim=0) - - head_logit = F.linear(hidden, head_weight, bias=head_bias) - head_logprob = F.log_softmax(head_logit, dim=1) - - nll = torch.zeros_like(target, - dtype=hidden.dtype, device=hidden.device) - - offset = 0 - cutoff_values = [0] + self.cutoffs - for i in range(len(cutoff_values) - 1): - l_idx, h_idx = cutoff_values[i], cutoff_values[i + 1] - - mask_i = (target >= l_idx) & (target < h_idx) - indices_i = mask_i.nonzero().squeeze() - - if indices_i.numel() == 0: - continue - - target_i = target.index_select(0, indices_i) - l_idx - head_logprob_i = head_logprob.index_select(0, indices_i) - - if i == 0: - - # aa=target_i[:, None] - # aa=aa.to('cpu') - # logprob_i = head_logprob_i.gather(1, aa).squeeze(1) - print(f'target_i[:,None]: {target_i[:,None]}') - print(f'target_i[:,None].shape: {target_i[:, None].shape}') - - - logprob_i = head_logprob_i.gather(1, target_i[:,None]).squeeze(1) - else: - weight_i = weight[l_idx:h_idx] - bias_i = bias[l_idx:h_idx] - - hidden_i = hidden.index_select(0, indices_i) - - tail_logit_i = F.linear(hidden_i, weight_i, bias=bias_i) - tail_logprob_i = F.log_softmax(tail_logit_i, dim=1) - - # aa = target_i[:, None] - # aa = aa.to('cpu') - # logprob_i = head_logprob_i[:, -i] \ - # + tail_logprob_i.gather(1, aa).squeeze(1) - - print(f'target_i[:,None]: {target_i[:, None]}') - print(f'target_i[:,None].shape: {target_i[:, None].shape}') - logprob_i = head_logprob_i[:, -i] \ - + tail_logprob_i.gather(1, target_i[:,None]).squeeze(1) - - if (hasattr(self, 'keep_order') and self.keep_order) or keep_order: - nll.index_copy_(0, indices_i, -logprob_i) - else: - nll[offset:offset+logprob_i.size(0)].copy_(-logprob_i) - - offset += logprob_i.size(0) - - return nll -- Gitee From b81ca8284234df3e1aefe7ed447226005094cfcc Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 03:59:36 +0000 Subject: [PATCH 10/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/nlp/Transformer-xl=5Ffor=5FPyTorch/data=5Futils.p?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Transformer-xl_for_PyTorch/data_utils.py | 270 ------------------ 1 file changed, 270 deletions(-) delete mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py deleted file mode 100644 index 554739ebea..0000000000 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py +++ /dev/null @@ -1,270 +0,0 @@ -import os, sys -import glob -import numpy as np -import torch - -from utils.vocabulary import Vocab - - -class LMOrderedIterator(object): # 有序数据集 - def __init__(self, data, bsz, bptt, device='npu:0', ext_len=None): - """ - data -- LongTensor -- the LongTensor is strictly ordered - """ - self.bsz = bsz # batch_size - self.bptt = bptt # 步长？ - self.ext_len = ext_len if ext_len is not None else 0 - self.device = device - - # Work out how cleanly we can divide the dataset into bsz parts. - self.n_step = data.size(0) // bsz - - # Trim off any extra elements that wouldn't cleanly fit (remainders). - data = data.narrow(0, 0, self.n_step * bsz) - - # Evenly divide the data across the bsz batches. - self.data = data.view(bsz, -1).t().contiguous().to(device) - - # Number of mini-batches - self.n_batch = (self.n_step + self.bptt - 1) // self.bptt - - def get_batch(self, i, bptt=None): # 一个batch的data和index范围 - if bptt is None: bptt = self.bptt - seq_len = min(bptt, self.data.size(0) - 1 - i) - - end_idx = i + seq_len # 结束index - beg_idx = max(0, i - self.ext_len) # 开始index - - data = self.data[beg_idx:end_idx] - target = self.data[i+1:i+1+seq_len] - - return data, target, seq_len # 按照batch形式返回，data和target是两个列表 - - def get_fixlen_iter(self, start=0): - for i in range(start, self.data.size(0) - 1, self.bptt): - yield self.get_batch(i) # yield的作用是把一个函数变成一个 generator - - def get_varlen_iter(self, start=0, std=5, min_len=5, max_deviation=3): - max_len = self.bptt + max_deviation * std - i = start - while True: - bptt = self.bptt if np.random.random() < 0.95 else self.bptt / 2. - bptt = min(max_len, max(min_len, int(np.random.normal(bptt, std)))) - data, target, seq_len = self.get_batch(i, bptt) - i += seq_len - yield data, target, seq_len - if i >= self.data.size(0) - 2: - break - - def __iter__(self): - return self.get_fixlen_iter() - - -class LMShuffledIterator(object): # 乱序数据集 - def __init__(self, data, bsz, bptt, device='npu:0', ext_len=None, shuffle=False): - """ - data -- list[LongTensor] -- there is no order among the LongTensors - """ - self.data = data - - self.bsz = bsz - self.bptt = bptt - self.ext_len = ext_len if ext_len is not None else 0 - - self.device = device - self.shuffle = shuffle - - def get_sent_stream(self): - # index iterator - epoch_indices = np.random.permutation(len(self.data)) if self.shuffle \ - else np.array(range(len(self.data))) - - # sentence iterator - for idx in epoch_indices: - yield self.data[idx] - - def stream_iterator(self, sent_stream): - # streams for each data in the batch - streams = [None] * self.bsz - - data = torch.LongTensor(self.bptt, self.bsz) - target = torch.LongTensor(self.bptt, self.bsz) - - n_retain = 0 - - while True: - # data : [n_retain+bptt x bsz] - # target : [bptt x bsz] - data[n_retain:].fill_(-1) - target.fill_(-1) - - valid_batch = True - - for i in range(self.bsz): - n_filled = 0 - try: - while n_filled < self.bptt: - if streams[i] is None or len(streams[i]) <= 1: - streams[i] = next(sent_stream) - # number of new tokens to fill in - n_new = min(len(streams[i]) - 1, self.bptt - n_filled) - # first n_retain tokens are retained from last batch - data[n_retain+n_filled:n_retain+n_filled+n_new, i] = \ - streams[i][:n_new] - target[n_filled:n_filled+n_new, i] = \ - streams[i][1:n_new+1] - streams[i] = streams[i][n_new:] - n_filled += n_new - except StopIteration: - valid_batch = False - break - - if not valid_batch: - return - - data = data.to(self.device) - target = target.to(self.device) - - yield data, target, self.bptt - - n_retain = min(data.size(0), self.ext_len) - if n_retain > 0: - data[:n_retain] = data[-n_retain:] - data.resize_(n_retain + self.bptt, data.size(1)) - - def __iter__(self): - # sent_stream is an iterator - sent_stream = self.get_sent_stream() - - for batch in self.stream_iterator(sent_stream): - yield batch - - -class LMMultiFileIterator(LMShuffledIterator): - def __init__(self, paths, vocab, bsz, bptt, device='npu:0', ext_len=None, - shuffle=False): - - self.paths = paths - self.vocab = vocab - - self.bsz = bsz - self.bptt = bptt - self.ext_len = ext_len if ext_len is not None else 0 - - self.device = device - self.shuffle = shuffle - - def get_sent_stream(self, path): - sents = self.vocab.encode_file(path, add_double_eos=True) - if self.shuffle: - np.random.shuffle(sents) - sent_stream = iter(sents) - - return sent_stream - - def __iter__(self): - if self.shuffle: - np.random.shuffle(self.paths) - - for path in self.paths: - # sent_stream is an iterator - sent_stream = self.get_sent_stream(path) - for batch in self.stream_iterator(sent_stream): - yield batch - - -class Corpus(object): # 语料库 - def __init__(self, path, dataset, *args, **kwargs): - self.dataset = dataset - self.vocab = Vocab(*args, **kwargs) - - if self.dataset in ['ptb', 'wt2', 'enwik8', 'text8']: - self.vocab.count_file(os.path.join(path, 'train.txt')) - self.vocab.count_file(os.path.join(path, 'valid.txt')) - self.vocab.count_file(os.path.join(path, 'test.txt')) - elif self.dataset == 'wt103': - self.vocab.count_file(os.path.join(path, 'train.txt')) # path='../data/wikitext-103',对train.txt的每一行看做整体，创建列表 - elif self.dataset == 'lm1b': - train_path_pattern = os.path.join( - path, '1-billion-word-language-modeling-benchmark-r13output', - 'training-monolingual.tokenized.shuffled', 'news.en-*') - train_paths = glob.glob(train_path_pattern) - # the vocab will load from file when build_vocab() is called - - self.vocab.build_vocab() - - if self.dataset in ['ptb', 'wt2', 'wt103']: - self.train = self.vocab.encode_file( - os.path.join(path, 'train.txt'), ordered=True) # 按照行转换为tensor - self.valid = self.vocab.encode_file( - os.path.join(path, 'valid.txt'), ordered=True) - self.test = self.vocab.encode_file( - os.path.join(path, 'test.txt'), ordered=True) - elif self.dataset in ['enwik8', 'text8']: - self.train = self.vocab.encode_file( - os.path.join(path, 'train.txt'), ordered=True, add_eos=False) - self.valid = self.vocab.encode_file( - os.path.join(path, 'valid.txt'), ordered=True, add_eos=False) - self.test = self.vocab.encode_file( - os.path.join(path, 'test.txt'), ordered=True, add_eos=False) - elif self.dataset == 'lm1b': - self.train = train_paths - self.valid = self.vocab.encode_file( - os.path.join(path, 'valid.txt'), ordered=False, add_double_eos=True) - self.test = self.vocab.encode_file( - os.path.join(path, 'test.txt'), ordered=False, add_double_eos=True) - - def get_iterator(self, split, *args, **kwargs): - if split == 'train': - if self.dataset in ['ptb', 'wt2', 'wt103', 'enwik8', 'text8']: - data_iter = LMOrderedIterator(self.train, *args, **kwargs) - elif self.dataset == 'lm1b': - kwargs['shuffle'] = True - data_iter = LMMultiFileIterator(self.train, self.vocab, *args, **kwargs) - elif split in ['valid', 'test.py']: - data = self.valid if split == 'valid' else self.test - if self.dataset in ['ptb', 'wt2', 'wt103', 'enwik8', 'text8']: - data_iter = LMOrderedIterator(data, *args, **kwargs) - elif self.dataset == 'lm1b': - data_iter = LMShuffledIterator(data, *args, **kwargs) - - return data_iter - - -def get_lm_corpus(datadir, dataset): - fn = os.path.join(datadir, 'cache.pt') - if os.path.exists(fn): - print('Loading cached dataset...') - corpus = torch.load(fn) - else: - print('Producing dataset {}...'.format(dataset)) # 产生数据集 - kwargs = {} - if dataset in ['wt103', 'wt2']: - kwargs['special'] = [''] - kwargs['lower_case'] = False # 不转为小写字母 - elif dataset == 'ptb': - kwargs['special'] = [''] - kwargs['lower_case'] = True - elif dataset == 'lm1b': - kwargs['special'] = [] - kwargs['lower_case'] = False - kwargs['vocab_file'] = os.path.join(datadir, '1b_word_vocab.txt') - elif dataset in ['enwik8', 'text8']: - pass - corpus = Corpus(datadir, dataset, **kwargs) # data_iter,字典迭代器 - torch.save(corpus, fn) # fn是路径 - return corpus - - -# if __name__ == '__main__': -# import argparse -# parser = argparse.ArgumentParser(description='unit test') -# parser.add_argument('--datadir', type=str, default='../data/enwik8', -# help='location of the data corpus') -# parser.add_argument('--dataset', type=str, default='enwik8', -# choices=['ptb', 'wt2', 'wt103', 'lm1b', 'enwik8', 'text8'], -# help='dataset name') -# args = parser.parse_args() -# -# corpus = get_lm_corpus(args.datadir, args.dataset) # corpus是字典iter -# print('Vocab size : {}'.format(len(corpus.vocab.idx2sym))) -- Gitee From 006876380251c5e79fff5f1dbed084d9940cfedf Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 03:59:43 +0000 Subject: [PATCH 11/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/nlp/Transformer-xl=5Ffor=5FPyTorch/eval=5Fnpu.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Transformer-xl_for_PyTorch/eval_npu.py | 376 ------------------ 1 file changed, 376 deletions(-) delete mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py deleted file mode 100644 index 29e80f07e2..0000000000 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py +++ /dev/null @@ -1,376 +0,0 @@ -# coding: UTF-8 -# Copyright 2022 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import time -import math -import os -import torch -import torch.nn as nn -import torch.optim as optim - -from data_utils import get_lm_corpus -from mem_transformer import MemTransformerLM -from utils.exp_utils import create_exp_dir -from apex import amp -import apex -from utils.exp_utils import get_logger - -parser = argparse.ArgumentParser(description='PyTorch Transformer Language Model') -parser.add_argument('--data', type=str, default='../data/enwik8', - help='location of the data corpus') -parser.add_argument('--dataset', type=str, default='enwik8', - choices=['wt103', 'lm1b', 'enwik8', 'text8'], - help='dataset name') -parser.add_argument('--n_layer', type=int, default=12, - help='number of total layers') -parser.add_argument('--n_head', type=int, default=8, - help='number of heads') -parser.add_argument('--d_head', type=int, default=64, - help='head dimension') -parser.add_argument('--d_embed', type=int, default=-1, - help='embedding dimension') -parser.add_argument('--d_model', type=int, default=512, - help='model dimension') -parser.add_argument('--d_inner', type=int, default=2048, - help='inner dimension in FF') -parser.add_argument('--dropout', type=float, default=0.1, - help='global dropout rate') -parser.add_argument('--dropatt', type=float, default=0.0, - help='attention probability dropout rate') -parser.add_argument('--init', default='normal', type=str, - help='parameter initializer to use.') -parser.add_argument('--emb_init', default='normal', type=str, - help='parameter initializer to use.') -parser.add_argument('--init_range', type=float, default=0.1, - help='parameters initialized by U(-init_range, init_range)') -parser.add_argument('--emb_init_range', type=float, default=0.01, - help='parameters initialized by U(-init_range, init_range)') -parser.add_argument('--init_std', type=float, default=0.02, - help='parameters initialized by N(0, init_std)') -parser.add_argument('--proj_init_std', type=float, default=0.01, - help='parameters initialized by N(0, init_std)') -parser.add_argument('--optim', default='adam', type=str, - choices=['adam', 'sgd', 'adagrad'], - help='optimizer to use.') -parser.add_argument('--lr', type=float, default=0.00025, - help='initial learning rate (0.00025|5 for adam|sgd)') -parser.add_argument('--mom', type=float, default=0.0, - help='momentum for sgd') -parser.add_argument('--scheduler', default='cosine', type=str, - choices=['cosine', 'inv_sqrt', 'dev_perf', 'constant'], - help='lr scheduler to use.') -parser.add_argument('--warmup_step', type=int, default=0, - help='upper epoch limit') -parser.add_argument('--decay_rate', type=float, default=0.5, - help='decay factor when ReduceLROnPlateau is used') -parser.add_argument('--lr_min', type=float, default=0.0, - help='minimum learning rate during annealing') -parser.add_argument('--clip', type=float, default=0.25, - help='gradient clipping') -parser.add_argument('--clip_nonemb', action='store_true', - help='only clip the gradient of non-embedding params') -parser.add_argument('--max_step', type=int, default=100000, - help='upper epoch limit') -parser.add_argument('--batch_size', type=int, default=10, - help='batch size') -parser.add_argument('--batch_chunk', type=int, default=1, - help='split batch into chunks to save memory') -parser.add_argument('--tgt_len', type=int, default=512, - help='number of tokens to predict') -parser.add_argument('--eval_tgt_len', type=int, default=128, - help='number of tokens to predict for evaluation') -parser.add_argument('--ext_len', type=int, default=0, - help='length of the extended context') -parser.add_argument('--mem_len', type=int, default=512, - help='length of the retained previous heads') -parser.add_argument('--not_tied', action='store_true', - help='do not tie the word embedding and softmax weights') -parser.add_argument('--seed', type=int, default=1111, - help='random seed') -parser.add_argument('--npu', default=True, help='use NPU') -parser.add_argument('--adaptive', action='store_true', - help='use adaptive softmax') -parser.add_argument('--div_val', type=int, default=1, - help='divident value for adapative input and softmax') -parser.add_argument('--pre_lnorm', action='store_true', - help='apply LayerNorm to the input instead of the output') -parser.add_argument('--varlen', action='store_true', - help='use variable length') -parser.add_argument('--multi_gpu', action='store_true', - help='use multiple GPU') -parser.add_argument('--log-interval', type=int, default=200, - help='report interval') -parser.add_argument('--eval-interval', type=int, default=4000, - help='evaluation interval') -parser.add_argument('--work_dir', default='LM-TFM', type=str, - help='experiment directory.') -parser.add_argument('--restart', action='store_true', - help='restart training from the saved checkpoint') -parser.add_argument('--restart_dir', type=str, default='', - help='restart dir') -parser.add_argument('--debug', action='store_true', - help='run in debug mode (do not create exp dir)') -parser.add_argument('--same_length', action='store_true', - help='use the same attn length for all tokens') -parser.add_argument('--attn_type', type=int, default=0, - help='attention type. 0 for ours, 1 for Shaw et al,' - '2 for Vaswani et al, 3 for Al Rfou et al.') -parser.add_argument('--clamp_len', type=int, default=-1, - help='use the same pos embeddings after clamp_len') -parser.add_argument('--eta_min', type=float, default=0.0, - help='min learning rate for cosine scheduler') -parser.add_argument('--gpu0_bsz', type=int, default=-1, - help='batch size on gpu 0') -parser.add_argument('--max_eval_steps', type=int, default=-1, - help='max eval steps') -parser.add_argument('--sample_softmax', type=int, default=-1, - help='number of samples in sampled softmax') -parser.add_argument('--patience', type=int, default=0, - help='patience') -parser.add_argument('--finetune_v2', action='store_true', - help='finetune v2') -parser.add_argument('--finetune_v3', action='store_true', - help='finetune v3') -parser.add_argument('--static-loss-scale', type=float, default=128.0, - help='Static loss scale, positive power of 2 values can ' - 'improve fp16 convergence.') -parser.add_argument('--dynamic-loss-scale', action='store_true', - help='Use dynamic loss scaling. If supplied, this argument' - ' supersedes --static-loss-scale.') -parser.add_argument('--no_log', action='store_true', - help='do not log the eval result') -parser.add_argument('--split', default='valid', - choices=['all','valid','test']) - -args = parser.parse_args() -args.tied = not args.not_tied - -if args.d_embed < 0: - args.d_embed = args.d_model - -assert args.ext_len >= 0, 'extended context length must be non-negative' -assert args.batch_size % args.batch_chunk == 0 - -args.work_dir = '{}-{}'.format(args.work_dir, args.dataset) -args.work_dir = os.path.join(args.work_dir, time.strftime('%Y%m%d-%H%M%S')) - -# Get logger -logging = create_exp_dir(args.work_dir, - scripts_to_save=['train.py', 'mem_transformer.py'], debug=args.debug) -logging = get_logger('log.txt', log_=not args.no_log) - -loc = "npu:0" -torch.npu.set_device(loc) - -############################################################################### -# Load data -############################################################################### -corpus = get_lm_corpus(args.data, args.dataset) -ntokens = len(corpus.vocab) -args.n_token = ntokens - -va_iter = corpus.get_iterator('valid', args.batch_size, args.eval_tgt_len, - device=loc, ext_len=args.ext_len) -te_iter = corpus.get_iterator('test.py', args.batch_size, args.eval_tgt_len, - device=loc, ext_len=args.ext_len) - -# adaptive softmax / embedding -cutoffs, tie_projs = [], [False] -if args.adaptive: - assert args.dataset in ['wt103', 'lm1b'] - if args.dataset == 'wt103': - cutoffs = [20000, 40000, 200000] - tie_projs += [True] * len(cutoffs) - elif args.dataset == 'lm1b': - cutoffs = [60000, 100000, 640000] - tie_projs += [False] * len(cutoffs) - -############################################################################### -# Build the model -############################################################################### -def init_weight(weight): - if args.init == 'uniform': - nn.init.uniform_(weight, -args.init_range, args.init_range) - elif args.init == 'normal': - nn.init.normal_(weight, 0.0, args.init_std) - -def init_bias(bias): - nn.init.constant_(bias, 0.0) - -def weights_init(m): - classname = m.__class__.__name__ - if classname.find('Linear') != -1: - if hasattr(m, 'weight') and m.weight is not None: - init_weight(m.weight) - if hasattr(m, 'bias') and m.bias is not None: - init_bias(m.bias) - elif classname.find('AdaptiveEmbedding') != -1: - if hasattr(m, 'emb_projs'): - for i in range(len(m.emb_projs)): - if m.emb_projs[i] is not None: - nn.init.normal_(m.emb_projs[i], 0.0, args.proj_init_std) - elif classname.find('Embedding') != -1: - if hasattr(m, 'weight'): - init_weight(m.weight) - elif classname.find('ProjectedAdaptiveLogSoftmax') != -1: - if hasattr(m, 'cluster_weight') and m.cluster_weight is not None: - init_weight(m.cluster_weight) - if hasattr(m, 'cluster_bias') and m.cluster_bias is not None: - init_bias(m.cluster_bias) - if hasattr(m, 'out_projs'): - for i in range(len(m.out_projs)): - if m.out_projs[i] is not None: - nn.init.normal_(m.out_projs[i], 0.0, args.proj_init_std) - elif classname.find('LayerNorm') != -1: - if hasattr(m, 'weight'): - nn.init.normal_(m.weight, 1.0, args.init_std) - if hasattr(m, 'bias') and m.bias is not None: - init_bias(m.bias) - elif classname.find('TransformerLM') != -1: - if hasattr(m, 'r_emb'): - init_weight(m.r_emb) - if hasattr(m, 'r_w_bias'): - init_weight(m.r_w_bias) - if hasattr(m, 'r_r_bias'): - init_weight(m.r_r_bias) - if hasattr(m, 'r_bias'): - init_bias(m.r_bias) - -def update_dropout(m): - classname = m.__class__.__name__ - if classname.find('Dropout') != -1: - if hasattr(m, 'p'): - m.p = args.dropout - -def update_dropatt(m): - if hasattr(m, 'dropatt'): - m.dropatt.p = args.dropatt - -model = MemTransformerLM(ntokens, args.n_layer, args.n_head, args.d_model, - args.d_head, args.d_inner, args.dropout, args.dropatt, - tie_weight=args.tied, d_embed=args.d_embed, div_val=args.div_val, - tie_projs=tie_projs, pre_lnorm=args.pre_lnorm, tgt_len=args.tgt_len, - ext_len=args.ext_len, mem_len=args.mem_len, cutoffs=cutoffs, - same_length=args.same_length, attn_type=args.attn_type, - clamp_len=args.clamp_len, sample_softmax=args.sample_softmax) -model.apply(weights_init) -model.word_emb.apply(weights_init) -args.n_all_param = sum([p.nelement() for p in model.parameters()]) -args.n_nonemb_param = sum([p.nelement() for p in model.layers.parameters()]) - -model = model.to(loc) - -#### optimizer -if args.optim.lower() == 'sgd': - if args.sample_softmax > 0: - dense_params, sparse_params = [], [] - for param in model.parameters(): - if param.size() == model.word_emb.weight.size(): - sparse_params.append(param) - else: - dense_params.append(param) - optimizer_sparse = optim.SGD(sparse_params, lr=args.lr * 2) - optimizer = optim.SGD(dense_params, lr=args.lr, momentum=args.mom) - else: - optimizer = optim.SGD(model.parameters(), lr=args.lr, - momentum=args.mom) -elif args.optim.lower() == 'adam': - if args.sample_softmax > 0: - dense_params, sparse_params = [], [] - for param in model.parameters(): - if param.size() == model.word_emb.weight.size(): - sparse_params.append(param) - else: - dense_params.append(param) - optimizer_sparse = optim.SparseAdam(sparse_params, lr=args.lr) - optimizer = optim.Adam(dense_params, lr=args.lr) - else: - optimizer = apex.optimizers.NpuFusedAdam(model.parameters(), lr=args.lr) -elif args.optim.lower() == 'adagrad': - optimizer = optim.Adagrad(model.parameters(), lr=args.lr) - - - -logging('=' * 100) -logging('#params = {}'.format(args.n_all_param)) -logging('#non emb params = {}'.format(args.n_nonemb_param)) - -# Load the best saved model. -with open('model_best_bpc.pt', 'rb') as f: - model.load_state_dict(torch.load(f, map_location=loc)) - -logging('Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'.format( - args.batch_size, args.tgt_len, args.ext_len, args.mem_len, args.clamp_len)) - -model.reset_length(args.tgt_len, args.ext_len, args.mem_len) -if args.clamp_len > 0: - model.clamp_len = args.clamp_len -if args.same_length: - model.same_length = True - -############################################################################### -# Evaluation code -############################################################################### - -def evaluate(eval_iter): - model.eval() - total_len, total_loss = 0, 0. - start_time = time.time() - with torch.no_grad(): - mems = tuple() - for idx, (data, target, seq_len) in enumerate(eval_iter): - ts = time.time() - ret = model(data,target,*mems) - loss, mems = ret[0], ret[1:] - loss = loss.mean() - total_loss += seq_len * loss.item() - total_len += seq_len - #print('eval_batch id: {} use time: {:.2f} ms '.format(idx, (time.time()-ts)*1000)) - total_time = time.time() - start_time - logging('Time : {:.2f}s, FPS: {:.2f} characters/s'.format( - total_time, total_len*args.batch_size*args.eval_tgt_len/total_time)) - return total_loss / total_len - - -# Run on test.py data. -if args.split == 'all': - test_loss = evaluate(te_iter) - valid_loss = evaluate(va_iter) -elif args.split == 'valid': - valid_loss = evaluate(va_iter) - test_loss = None -elif args.split == 'test': - test_loss = evaluate(te_iter) - valid_loss = None - -def format_log(loss, split): - if args.dataset in ['enwik8', 'text8']: - log_str = '| {0} loss {1:5.2f} | {0} bpc {2:9.5f} '.format( - split, loss, loss / math.log(2)) - else: - log_str = '| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '.format( - split, loss, math.exp(loss)) - return log_str - -log_str = '' -if valid_loss is not None: - log_str += format_log(valid_loss, 'valid') -if test_loss is not None: - log_str += format_log(test_loss, 'test.py') - -logging('=' * 100) -logging(log_str) -logging('=' * 100) -- Gitee From 7da887a018832c97b3fb40f92bc6e5ac9be751db Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 04:00:06 +0000 Subject: [PATCH 12/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/nlp/Transformer-xl=5Ffor=5FPyTorch/utils/adaptive?= =?UTF-8?q?=5Fsoftmax.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../utils/adaptive_softmax.py | 106 ------------------ 1 file changed, 106 deletions(-) delete mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py deleted file mode 100644 index 7a0db658ab..0000000000 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py +++ /dev/null @@ -1,106 +0,0 @@ -# coding: UTF-8 -# Copyright 2022 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import torch -import torch.nn as nn -import torch.nn.functional as F - -class AdaptiveLogSoftmax(nn.Module): - def __init__(self, in_features, n_classes, cutoffs, keep_order=False): - super(AdaptiveLogSoftmax, self).__init__() - - cutoffs = list(cutoffs) - - if (cutoffs != sorted(cutoffs)) \ - or (min(cutoffs) <= 0) \ - or (max(cutoffs) >= (n_classes - 1)) \ - or (len(set(cutoffs)) != len(cutoffs)) \ - or any([int(c) != c for c in cutoffs]): - - raise ValueError("cutoffs should be a sequence of unique, positive " - "integers sorted in an increasing order, where " - "each value is between 1 and n_classes-1") - - self.in_features = in_features - self.n_classes = n_classes - self.cutoffs = cutoffs + [n_classes] - - self.shortlist_size = self.cutoffs[0] - self.n_clusters = len(self.cutoffs) - 1 - self.head_size = self.shortlist_size + self.n_clusters - - self.cluster_weight = nn.Parameter(torch.zeros(self.n_clusters, self.in_features)) - self.cluster_bias = nn.Parameter(torch.zeros(self.n_clusters)) - - self.keep_order = keep_order - - - def forward(self, hidden, target, weight, bias, keep_order=False): - if hidden.size(0) != target.size(0): - raise RuntimeError('Input and target should have the same size ' - 'in the batch dimension.') - - head_weight = torch.cat( - [weight[:self.shortlist_size], self.cluster_weight], dim=0) - head_bias = torch.cat( - [bias[:self.shortlist_size], self.cluster_bias], dim=0) - - head_logit = F.linear(hidden, head_weight, bias=head_bias) - head_logprob = F.log_softmax(head_logit, dim=1) - - nll = torch.zeros_like(target, - dtype=hidden.dtype, device=hidden.device) - - offset = 0 - cutoff_values = [0] + self.cutoffs - for i in range(len(cutoff_values) - 1): - l_idx, h_idx = cutoff_values[i], cutoff_values[i + 1] - - mask_i = (target >= l_idx) & (target < h_idx) - indices_i = mask_i.nonzero().squeeze() - - if indices_i.numel() == 0: - continue - - target_i = target.index_select(0, indices_i) - l_idx - head_logprob_i = head_logprob.index_select(0, indices_i) - - if i == 0: - print(f'target_i[:,None]: {target_i[:,None]}') - print(f'target_i[:,None].shape: {target_i[:, None].shape}') - - - logprob_i = head_logprob_i.gather(1, target_i[:,None]).squeeze(1) - else: - weight_i = weight[l_idx:h_idx] - bias_i = bias[l_idx:h_idx] - - hidden_i = hidden.index_select(0, indices_i) - - tail_logit_i = F.linear(hidden_i, weight_i, bias=bias_i) - tail_logprob_i = F.log_softmax(tail_logit_i, dim=1) - - print(f'target_i[:,None]: {target_i[:, None]}') - print(f'target_i[:,None].shape: {target_i[:, None].shape}') - logprob_i = head_logprob_i[:, -i] \ - + tail_logprob_i.gather(1, target_i[:,None]).squeeze(1) - - if (hasattr(self, 'keep_order') and self.keep_order) or keep_order: - nll.index_copy_(0, indices_i, -logprob_i) - else: - nll[offset:offset+logprob_i.size(0)].copy_(-logprob_i) - - offset += logprob_i.size(0) - - return nll -- Gitee From ddc53ee26bb4bddf96cd1a22a9e34be5e80060c4 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 04:00:26 +0000 Subject: [PATCH 13/59] my first commit --- .../utils/adaptive_softmax.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py new file mode 100644 index 0000000000..3c54a69204 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/utils/adaptive_softmax.py @@ -0,0 +1,102 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +import torch.nn as nn +import torch.nn.functional as F + +class AdaptiveLogSoftmax(nn.Module): + def __init__(self, in_features, n_classes, cutoffs, keep_order=False): + super(AdaptiveLogSoftmax, self).__init__() + + cutoffs = list(cutoffs) + + if (cutoffs != sorted(cutoffs)) \ + or (min(cutoffs) <= 0) \ + or (max(cutoffs) >= (n_classes - 1)) \ + or (len(set(cutoffs)) != len(cutoffs)) \ + or any([int(c) != c for c in cutoffs]): + + raise ValueError("cutoffs should be a sequence of unique, positive " + "integers sorted in an increasing order, where " + "each value is between 1 and n_classes-1") + + self.in_features = in_features + self.n_classes = n_classes + self.cutoffs = cutoffs + [n_classes] + + self.shortlist_size = self.cutoffs[0] + self.n_clusters = len(self.cutoffs) - 1 + self.head_size = self.shortlist_size + self.n_clusters + + self.cluster_weight = nn.Parameter(torch.zeros(self.n_clusters, self.in_features)) + self.cluster_bias = nn.Parameter(torch.zeros(self.n_clusters)) + + self.keep_order = keep_order + + + def forward(self, hidden, target, weight, bias, keep_order=False): + if hidden.size(0) != target.size(0): + raise RuntimeError('Input and target should have the same size ' + 'in the batch dimension.') + + head_weight = torch.cat( + [weight[:self.shortlist_size], self.cluster_weight], dim=0) + head_bias = torch.cat( + [bias[:self.shortlist_size], self.cluster_bias], dim=0) + + head_logit = F.linear(hidden, head_weight, bias=head_bias) + head_logprob = F.log_softmax(head_logit, dim=1) + + nll = torch.zeros_like(target, + dtype=hidden.dtype, device=hidden.device) + + offset = 0 + cutoff_values = [0] + self.cutoffs + for i in range(len(cutoff_values) - 1): + l_idx, h_idx = cutoff_values[i], cutoff_values[i + 1] + + mask_i = (target >= l_idx) & (target < h_idx) + indices_i = mask_i.nonzero().squeeze() + + if indices_i.numel() == 0: + continue + + target_i = target.index_select(0, indices_i) - l_idx + head_logprob_i = head_logprob.index_select(0, indices_i) + + if i == 0: + logprob_i = head_logprob_i.gather(1, target_i[:,None]).squeeze(1) + else: + weight_i = weight[l_idx:h_idx] + bias_i = bias[l_idx:h_idx] + + hidden_i = hidden.index_select(0, indices_i) + + tail_logit_i = F.linear(hidden_i, weight_i, bias=bias_i) + tail_logprob_i = F.log_softmax(tail_logit_i, dim=1) + + print(f'target_i[:,None]: {target_i[:, None]}') + print(f'target_i[:,None].shape: {target_i[:, None].shape}') + logprob_i = head_logprob_i[:, -i] \ + + tail_logprob_i.gather(1, target_i[:,None]).squeeze(1) + + if (hasattr(self, 'keep_order') and self.keep_order) or keep_order: + nll.index_copy_(0, indices_i, -logprob_i) + else: + nll[offset:offset+logprob_i.size(0)].copy_(-logprob_i) + + offset += logprob_i.size(0) + + return nll -- Gitee From 7d3cf6ab8a61c24ac4887da6d99699018703859f Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 04:00:57 +0000 Subject: [PATCH 14/59] my first commit --- .../adaptive_softmax.py | 97 +++++ .../Transformer-xl_for_PyTorch/data_utils.py | 258 ++++++++++++ .../Transformer-xl_for_PyTorch/eval_npu.py | 376 ++++++++++++++++++ 3 files changed, 731 insertions(+) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py new file mode 100644 index 0000000000..1a65fb1c72 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py @@ -0,0 +1,97 @@ +from collections import defaultdict + +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F + +class AdaptiveLogSoftmax(nn.Module): + def __init__(self, in_features, n_classes, cutoffs, keep_order=False): + super(AdaptiveLogSoftmax, self).__init__() + + cutoffs = list(cutoffs) + + if (cutoffs != sorted(cutoffs)) \ + or (min(cutoffs) <= 0) \ + or (max(cutoffs) >= (n_classes - 1)) \ + or (len(set(cutoffs)) != len(cutoffs)) \ + or any([int(c) != c for c in cutoffs]): + + raise ValueError("cutoffs should be a sequence of unique, positive " + "integers sorted in an increasing order, where " + "each value is between 1 and n_classes-1") + + self.in_features = in_features + self.n_classes = n_classes + self.cutoffs = cutoffs + [n_classes] + + self.shortlist_size = self.cutoffs[0] + self.n_clusters = len(self.cutoffs) - 1 + self.head_size = self.shortlist_size + self.n_clusters + + self.cluster_weight = nn.Parameter(torch.zeros(self.n_clusters, self.in_features)) + self.cluster_bias = nn.Parameter(torch.zeros(self.n_clusters)) + + self.keep_order = keep_order + + + def forward(self, hidden, target, weight, bias, keep_order=False): + if hidden.size(0) != target.size(0): + raise RuntimeError('Input and target should have the same size ' + 'in the batch dimension.') + + head_weight = torch.cat( + [weight[:self.shortlist_size], self.cluster_weight], dim=0) + head_bias = torch.cat( + [bias[:self.shortlist_size], self.cluster_bias], dim=0) + + head_logit = F.linear(hidden, head_weight, bias=head_bias) + head_logprob = F.log_softmax(head_logit, dim=1) + + nll = torch.zeros_like(target, + dtype=hidden.dtype, device=hidden.device) + + offset = 0 + cutoff_values = [0] + self.cutoffs + for i in range(len(cutoff_values) - 1): + l_idx, h_idx = cutoff_values[i], cutoff_values[i + 1] + + mask_i = (target >= l_idx) & (target < h_idx) + indices_i = mask_i.nonzero().squeeze() + + if indices_i.numel() == 0: + continue + + target_i = target.index_select(0, indices_i) - l_idx + head_logprob_i = head_logprob.index_select(0, indices_i) + + if i == 0: + logprob_i = head_logprob_i.gather(1, target_i[:,None]).squeeze(1) + else: + weight_i = weight[l_idx:h_idx] + bias_i = bias[l_idx:h_idx] + + hidden_i = hidden.index_select(0, indices_i) + + tail_logit_i = F.linear(hidden_i, weight_i, bias=bias_i) + tail_logprob_i = F.log_softmax(tail_logit_i, dim=1) + + # aa = target_i[:, None] + # aa = aa.to('cpu') + # logprob_i = head_logprob_i[:, -i] \ + # + tail_logprob_i.gather(1, aa).squeeze(1) + + print(f'target_i[:,None]: {target_i[:, None]}') + print(f'target_i[:,None].shape: {target_i[:, None].shape}') + logprob_i = head_logprob_i[:, -i] \ + + tail_logprob_i.gather(1, target_i[:,None]).squeeze(1) + + if (hasattr(self, 'keep_order') and self.keep_order) or keep_order: + nll.index_copy_(0, indices_i, -logprob_i) + else: + nll[offset:offset+logprob_i.size(0)].copy_(-logprob_i) + + offset += logprob_i.size(0) + + return nll diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py new file mode 100644 index 0000000000..3642f37cef --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/data_utils.py @@ -0,0 +1,258 @@ +import os, sys +import glob +import numpy as np +import torch + +from utils.vocabulary import Vocab + + +class LMOrderedIterator(object): + def __init__(self, data, bsz, bptt, device='npu:0', ext_len=None): + """ + data -- LongTensor -- the LongTensor is strictly ordered + """ + self.bsz = bsz + self.bptt = bptt + self.ext_len = ext_len if ext_len is not None else 0 + self.device = device + + # Work out how cleanly we can divide the dataset into bsz parts. + self.n_step = data.size(0) // bsz + + # Trim off any extra elements that wouldn't cleanly fit (remainders). + data = data.narrow(0, 0, self.n_step * bsz) + + # Evenly divide the data across the bsz batches. + self.data = data.view(bsz, -1).t().contiguous().to(device) + + # Number of mini-batches + self.n_batch = (self.n_step + self.bptt - 1) // self.bptt + + def get_batch(self, i, bptt=None): + if bptt is None: bptt = self.bptt + seq_len = min(bptt, self.data.size(0) - 1 - i) + + end_idx = i + seq_len + beg_idx = max(0, i - self.ext_len) + + data = self.data[beg_idx:end_idx] + target = self.data[i+1:i+1+seq_len] + + return data, target, seq_len + + def get_fixlen_iter(self, start=0): + for i in range(start, self.data.size(0) - 1, self.bptt): + yield self.get_batch(i) + + def get_varlen_iter(self, start=0, std=5, min_len=5, max_deviation=3): + max_len = self.bptt + max_deviation * std + i = start + while True: + bptt = self.bptt if np.random.random() < 0.95 else self.bptt / 2. + bptt = min(max_len, max(min_len, int(np.random.normal(bptt, std)))) + data, target, seq_len = self.get_batch(i, bptt) + i += seq_len + yield data, target, seq_len + if i >= self.data.size(0) - 2: + break + + def __iter__(self): + return self.get_fixlen_iter() + + +class LMShuffledIterator(object): + def __init__(self, data, bsz, bptt, device='npu:0', ext_len=None, shuffle=False): + """ + data -- list[LongTensor] -- there is no order among the LongTensors + """ + self.data = data + + self.bsz = bsz + self.bptt = bptt + self.ext_len = ext_len if ext_len is not None else 0 + + self.device = device + self.shuffle = shuffle + + def get_sent_stream(self): + # index iterator + epoch_indices = np.random.permutation(len(self.data)) if self.shuffle \ + else np.array(range(len(self.data))) + + # sentence iterator + for idx in epoch_indices: + yield self.data[idx] + + def stream_iterator(self, sent_stream): + # streams for each data in the batch + streams = [None] * self.bsz + + data = torch.LongTensor(self.bptt, self.bsz) + target = torch.LongTensor(self.bptt, self.bsz) + + n_retain = 0 + + while True: + # data : [n_retain+bptt x bsz] + # target : [bptt x bsz] + data[n_retain:].fill_(-1) + target.fill_(-1) + + valid_batch = True + + for i in range(self.bsz): + n_filled = 0 + try: + while n_filled < self.bptt: + if streams[i] is None or len(streams[i]) <= 1: + streams[i] = next(sent_stream) + # number of new tokens to fill in + n_new = min(len(streams[i]) - 1, self.bptt - n_filled) + # first n_retain tokens are retained from last batch + data[n_retain+n_filled:n_retain+n_filled+n_new, i] = \ + streams[i][:n_new] + target[n_filled:n_filled+n_new, i] = \ + streams[i][1:n_new+1] + streams[i] = streams[i][n_new:] + n_filled += n_new + except StopIteration: + valid_batch = False + break + + if not valid_batch: + return + + data = data.to(self.device) + target = target.to(self.device) + + yield data, target, self.bptt + + n_retain = min(data.size(0), self.ext_len) + if n_retain > 0: + data[:n_retain] = data[-n_retain:] + data.resize_(n_retain + self.bptt, data.size(1)) + + def __iter__(self): + # sent_stream is an iterator + sent_stream = self.get_sent_stream() + + for batch in self.stream_iterator(sent_stream): + yield batch + + +class LMMultiFileIterator(LMShuffledIterator): + def __init__(self, paths, vocab, bsz, bptt, device='npu:0', ext_len=None, + shuffle=False): + + self.paths = paths + self.vocab = vocab + + self.bsz = bsz + self.bptt = bptt + self.ext_len = ext_len if ext_len is not None else 0 + + self.device = device + self.shuffle = shuffle + + def get_sent_stream(self, path): + sents = self.vocab.encode_file(path, add_double_eos=True) + if self.shuffle: + np.random.shuffle(sents) + sent_stream = iter(sents) + + return sent_stream + + def __iter__(self): + if self.shuffle: + np.random.shuffle(self.paths) + + for path in self.paths: + # sent_stream is an iterator + sent_stream = self.get_sent_stream(path) + for batch in self.stream_iterator(sent_stream): + yield batch + + +class Corpus(object): + def __init__(self, path, dataset, *args, **kwargs): + self.dataset = dataset + self.vocab = Vocab(*args, **kwargs) + + if self.dataset in ['ptb', 'wt2', 'enwik8', 'text8']: + self.vocab.count_file(os.path.join(path, 'train.txt')) + self.vocab.count_file(os.path.join(path, 'valid.txt')) + self.vocab.count_file(os.path.join(path, 'test.txt')) + elif self.dataset == 'wt103': + self.vocab.count_file(os.path.join(path, 'train.txt')) + elif self.dataset == 'lm1b': + train_path_pattern = os.path.join( + path, '1-billion-word-language-modeling-benchmark-r13output', + 'training-monolingual.tokenized.shuffled', 'news.en-*') + train_paths = glob.glob(train_path_pattern) + # the vocab will load from file when build_vocab() is called + + self.vocab.build_vocab() + + if self.dataset in ['ptb', 'wt2', 'wt103']: + self.train = self.vocab.encode_file( + os.path.join(path, 'train.txt'), ordered=True) + self.valid = self.vocab.encode_file( + os.path.join(path, 'valid.txt'), ordered=True) + self.test = self.vocab.encode_file( + os.path.join(path, 'test.txt'), ordered=True) + elif self.dataset in ['enwik8', 'text8']: + self.train = self.vocab.encode_file( + os.path.join(path, 'train.txt'), ordered=True, add_eos=False) + self.valid = self.vocab.encode_file( + os.path.join(path, 'valid.txt'), ordered=True, add_eos=False) + self.test = self.vocab.encode_file( + os.path.join(path, 'test.txt'), ordered=True, add_eos=False) + elif self.dataset == 'lm1b': + self.train = train_paths + self.valid = self.vocab.encode_file( + os.path.join(path, 'valid.txt'), ordered=False, add_double_eos=True) + self.test = self.vocab.encode_file( + os.path.join(path, 'test.txt'), ordered=False, add_double_eos=True) + + def get_iterator(self, split, *args, **kwargs): + if split == 'train': + if self.dataset in ['ptb', 'wt2', 'wt103', 'enwik8', 'text8']: + data_iter = LMOrderedIterator(self.train, *args, **kwargs) + elif self.dataset == 'lm1b': + kwargs['shuffle'] = True + data_iter = LMMultiFileIterator(self.train, self.vocab, *args, **kwargs) + elif split in ['valid', 'test.py']: + data = self.valid if split == 'valid' else self.test + if self.dataset in ['ptb', 'wt2', 'wt103', 'enwik8', 'text8']: + data_iter = LMOrderedIterator(data, *args, **kwargs) + elif self.dataset == 'lm1b': + data_iter = LMShuffledIterator(data, *args, **kwargs) + + return data_iter + + +def get_lm_corpus(datadir, dataset): + fn = os.path.join(datadir, 'cache.pt') + if os.path.exists(fn): + print('Loading cached dataset...') + corpus = torch.load(fn) + else: + print('Producing dataset {}...'.format(dataset)) + kwargs = {} + if dataset in ['wt103', 'wt2']: + kwargs['special'] = [''] + kwargs['lower_case'] = False + elif dataset == 'ptb': + kwargs['special'] = [''] + kwargs['lower_case'] = True + elif dataset == 'lm1b': + kwargs['special'] = [] + kwargs['lower_case'] = False + kwargs['vocab_file'] = os.path.join(datadir, '1b_word_vocab.txt') + elif dataset in ['enwik8', 'text8']: + pass + corpus = Corpus(datadir, dataset, **kwargs) + torch.save(corpus, fn) + return corpus + + diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py new file mode 100644 index 0000000000..29e80f07e2 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py @@ -0,0 +1,376 @@ +# coding: UTF-8 +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import time +import math +import os +import torch +import torch.nn as nn +import torch.optim as optim + +from data_utils import get_lm_corpus +from mem_transformer import MemTransformerLM +from utils.exp_utils import create_exp_dir +from apex import amp +import apex +from utils.exp_utils import get_logger + +parser = argparse.ArgumentParser(description='PyTorch Transformer Language Model') +parser.add_argument('--data', type=str, default='../data/enwik8', + help='location of the data corpus') +parser.add_argument('--dataset', type=str, default='enwik8', + choices=['wt103', 'lm1b', 'enwik8', 'text8'], + help='dataset name') +parser.add_argument('--n_layer', type=int, default=12, + help='number of total layers') +parser.add_argument('--n_head', type=int, default=8, + help='number of heads') +parser.add_argument('--d_head', type=int, default=64, + help='head dimension') +parser.add_argument('--d_embed', type=int, default=-1, + help='embedding dimension') +parser.add_argument('--d_model', type=int, default=512, + help='model dimension') +parser.add_argument('--d_inner', type=int, default=2048, + help='inner dimension in FF') +parser.add_argument('--dropout', type=float, default=0.1, + help='global dropout rate') +parser.add_argument('--dropatt', type=float, default=0.0, + help='attention probability dropout rate') +parser.add_argument('--init', default='normal', type=str, + help='parameter initializer to use.') +parser.add_argument('--emb_init', default='normal', type=str, + help='parameter initializer to use.') +parser.add_argument('--init_range', type=float, default=0.1, + help='parameters initialized by U(-init_range, init_range)') +parser.add_argument('--emb_init_range', type=float, default=0.01, + help='parameters initialized by U(-init_range, init_range)') +parser.add_argument('--init_std', type=float, default=0.02, + help='parameters initialized by N(0, init_std)') +parser.add_argument('--proj_init_std', type=float, default=0.01, + help='parameters initialized by N(0, init_std)') +parser.add_argument('--optim', default='adam', type=str, + choices=['adam', 'sgd', 'adagrad'], + help='optimizer to use.') +parser.add_argument('--lr', type=float, default=0.00025, + help='initial learning rate (0.00025|5 for adam|sgd)') +parser.add_argument('--mom', type=float, default=0.0, + help='momentum for sgd') +parser.add_argument('--scheduler', default='cosine', type=str, + choices=['cosine', 'inv_sqrt', 'dev_perf', 'constant'], + help='lr scheduler to use.') +parser.add_argument('--warmup_step', type=int, default=0, + help='upper epoch limit') +parser.add_argument('--decay_rate', type=float, default=0.5, + help='decay factor when ReduceLROnPlateau is used') +parser.add_argument('--lr_min', type=float, default=0.0, + help='minimum learning rate during annealing') +parser.add_argument('--clip', type=float, default=0.25, + help='gradient clipping') +parser.add_argument('--clip_nonemb', action='store_true', + help='only clip the gradient of non-embedding params') +parser.add_argument('--max_step', type=int, default=100000, + help='upper epoch limit') +parser.add_argument('--batch_size', type=int, default=10, + help='batch size') +parser.add_argument('--batch_chunk', type=int, default=1, + help='split batch into chunks to save memory') +parser.add_argument('--tgt_len', type=int, default=512, + help='number of tokens to predict') +parser.add_argument('--eval_tgt_len', type=int, default=128, + help='number of tokens to predict for evaluation') +parser.add_argument('--ext_len', type=int, default=0, + help='length of the extended context') +parser.add_argument('--mem_len', type=int, default=512, + help='length of the retained previous heads') +parser.add_argument('--not_tied', action='store_true', + help='do not tie the word embedding and softmax weights') +parser.add_argument('--seed', type=int, default=1111, + help='random seed') +parser.add_argument('--npu', default=True, help='use NPU') +parser.add_argument('--adaptive', action='store_true', + help='use adaptive softmax') +parser.add_argument('--div_val', type=int, default=1, + help='divident value for adapative input and softmax') +parser.add_argument('--pre_lnorm', action='store_true', + help='apply LayerNorm to the input instead of the output') +parser.add_argument('--varlen', action='store_true', + help='use variable length') +parser.add_argument('--multi_gpu', action='store_true', + help='use multiple GPU') +parser.add_argument('--log-interval', type=int, default=200, + help='report interval') +parser.add_argument('--eval-interval', type=int, default=4000, + help='evaluation interval') +parser.add_argument('--work_dir', default='LM-TFM', type=str, + help='experiment directory.') +parser.add_argument('--restart', action='store_true', + help='restart training from the saved checkpoint') +parser.add_argument('--restart_dir', type=str, default='', + help='restart dir') +parser.add_argument('--debug', action='store_true', + help='run in debug mode (do not create exp dir)') +parser.add_argument('--same_length', action='store_true', + help='use the same attn length for all tokens') +parser.add_argument('--attn_type', type=int, default=0, + help='attention type. 0 for ours, 1 for Shaw et al,' + '2 for Vaswani et al, 3 for Al Rfou et al.') +parser.add_argument('--clamp_len', type=int, default=-1, + help='use the same pos embeddings after clamp_len') +parser.add_argument('--eta_min', type=float, default=0.0, + help='min learning rate for cosine scheduler') +parser.add_argument('--gpu0_bsz', type=int, default=-1, + help='batch size on gpu 0') +parser.add_argument('--max_eval_steps', type=int, default=-1, + help='max eval steps') +parser.add_argument('--sample_softmax', type=int, default=-1, + help='number of samples in sampled softmax') +parser.add_argument('--patience', type=int, default=0, + help='patience') +parser.add_argument('--finetune_v2', action='store_true', + help='finetune v2') +parser.add_argument('--finetune_v3', action='store_true', + help='finetune v3') +parser.add_argument('--static-loss-scale', type=float, default=128.0, + help='Static loss scale, positive power of 2 values can ' + 'improve fp16 convergence.') +parser.add_argument('--dynamic-loss-scale', action='store_true', + help='Use dynamic loss scaling. If supplied, this argument' + ' supersedes --static-loss-scale.') +parser.add_argument('--no_log', action='store_true', + help='do not log the eval result') +parser.add_argument('--split', default='valid', + choices=['all','valid','test']) + +args = parser.parse_args() +args.tied = not args.not_tied + +if args.d_embed < 0: + args.d_embed = args.d_model + +assert args.ext_len >= 0, 'extended context length must be non-negative' +assert args.batch_size % args.batch_chunk == 0 + +args.work_dir = '{}-{}'.format(args.work_dir, args.dataset) +args.work_dir = os.path.join(args.work_dir, time.strftime('%Y%m%d-%H%M%S')) + +# Get logger +logging = create_exp_dir(args.work_dir, + scripts_to_save=['train.py', 'mem_transformer.py'], debug=args.debug) +logging = get_logger('log.txt', log_=not args.no_log) + +loc = "npu:0" +torch.npu.set_device(loc) + +############################################################################### +# Load data +############################################################################### +corpus = get_lm_corpus(args.data, args.dataset) +ntokens = len(corpus.vocab) +args.n_token = ntokens + +va_iter = corpus.get_iterator('valid', args.batch_size, args.eval_tgt_len, + device=loc, ext_len=args.ext_len) +te_iter = corpus.get_iterator('test.py', args.batch_size, args.eval_tgt_len, + device=loc, ext_len=args.ext_len) + +# adaptive softmax / embedding +cutoffs, tie_projs = [], [False] +if args.adaptive: + assert args.dataset in ['wt103', 'lm1b'] + if args.dataset == 'wt103': + cutoffs = [20000, 40000, 200000] + tie_projs += [True] * len(cutoffs) + elif args.dataset == 'lm1b': + cutoffs = [60000, 100000, 640000] + tie_projs += [False] * len(cutoffs) + +############################################################################### +# Build the model +############################################################################### +def init_weight(weight): + if args.init == 'uniform': + nn.init.uniform_(weight, -args.init_range, args.init_range) + elif args.init == 'normal': + nn.init.normal_(weight, 0.0, args.init_std) + +def init_bias(bias): + nn.init.constant_(bias, 0.0) + +def weights_init(m): + classname = m.__class__.__name__ + if classname.find('Linear') != -1: + if hasattr(m, 'weight') and m.weight is not None: + init_weight(m.weight) + if hasattr(m, 'bias') and m.bias is not None: + init_bias(m.bias) + elif classname.find('AdaptiveEmbedding') != -1: + if hasattr(m, 'emb_projs'): + for i in range(len(m.emb_projs)): + if m.emb_projs[i] is not None: + nn.init.normal_(m.emb_projs[i], 0.0, args.proj_init_std) + elif classname.find('Embedding') != -1: + if hasattr(m, 'weight'): + init_weight(m.weight) + elif classname.find('ProjectedAdaptiveLogSoftmax') != -1: + if hasattr(m, 'cluster_weight') and m.cluster_weight is not None: + init_weight(m.cluster_weight) + if hasattr(m, 'cluster_bias') and m.cluster_bias is not None: + init_bias(m.cluster_bias) + if hasattr(m, 'out_projs'): + for i in range(len(m.out_projs)): + if m.out_projs[i] is not None: + nn.init.normal_(m.out_projs[i], 0.0, args.proj_init_std) + elif classname.find('LayerNorm') != -1: + if hasattr(m, 'weight'): + nn.init.normal_(m.weight, 1.0, args.init_std) + if hasattr(m, 'bias') and m.bias is not None: + init_bias(m.bias) + elif classname.find('TransformerLM') != -1: + if hasattr(m, 'r_emb'): + init_weight(m.r_emb) + if hasattr(m, 'r_w_bias'): + init_weight(m.r_w_bias) + if hasattr(m, 'r_r_bias'): + init_weight(m.r_r_bias) + if hasattr(m, 'r_bias'): + init_bias(m.r_bias) + +def update_dropout(m): + classname = m.__class__.__name__ + if classname.find('Dropout') != -1: + if hasattr(m, 'p'): + m.p = args.dropout + +def update_dropatt(m): + if hasattr(m, 'dropatt'): + m.dropatt.p = args.dropatt + +model = MemTransformerLM(ntokens, args.n_layer, args.n_head, args.d_model, + args.d_head, args.d_inner, args.dropout, args.dropatt, + tie_weight=args.tied, d_embed=args.d_embed, div_val=args.div_val, + tie_projs=tie_projs, pre_lnorm=args.pre_lnorm, tgt_len=args.tgt_len, + ext_len=args.ext_len, mem_len=args.mem_len, cutoffs=cutoffs, + same_length=args.same_length, attn_type=args.attn_type, + clamp_len=args.clamp_len, sample_softmax=args.sample_softmax) +model.apply(weights_init) +model.word_emb.apply(weights_init) +args.n_all_param = sum([p.nelement() for p in model.parameters()]) +args.n_nonemb_param = sum([p.nelement() for p in model.layers.parameters()]) + +model = model.to(loc) + +#### optimizer +if args.optim.lower() == 'sgd': + if args.sample_softmax > 0: + dense_params, sparse_params = [], [] + for param in model.parameters(): + if param.size() == model.word_emb.weight.size(): + sparse_params.append(param) + else: + dense_params.append(param) + optimizer_sparse = optim.SGD(sparse_params, lr=args.lr * 2) + optimizer = optim.SGD(dense_params, lr=args.lr, momentum=args.mom) + else: + optimizer = optim.SGD(model.parameters(), lr=args.lr, + momentum=args.mom) +elif args.optim.lower() == 'adam': + if args.sample_softmax > 0: + dense_params, sparse_params = [], [] + for param in model.parameters(): + if param.size() == model.word_emb.weight.size(): + sparse_params.append(param) + else: + dense_params.append(param) + optimizer_sparse = optim.SparseAdam(sparse_params, lr=args.lr) + optimizer = optim.Adam(dense_params, lr=args.lr) + else: + optimizer = apex.optimizers.NpuFusedAdam(model.parameters(), lr=args.lr) +elif args.optim.lower() == 'adagrad': + optimizer = optim.Adagrad(model.parameters(), lr=args.lr) + + + +logging('=' * 100) +logging('#params = {}'.format(args.n_all_param)) +logging('#non emb params = {}'.format(args.n_nonemb_param)) + +# Load the best saved model. +with open('model_best_bpc.pt', 'rb') as f: + model.load_state_dict(torch.load(f, map_location=loc)) + +logging('Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'.format( + args.batch_size, args.tgt_len, args.ext_len, args.mem_len, args.clamp_len)) + +model.reset_length(args.tgt_len, args.ext_len, args.mem_len) +if args.clamp_len > 0: + model.clamp_len = args.clamp_len +if args.same_length: + model.same_length = True + +############################################################################### +# Evaluation code +############################################################################### + +def evaluate(eval_iter): + model.eval() + total_len, total_loss = 0, 0. + start_time = time.time() + with torch.no_grad(): + mems = tuple() + for idx, (data, target, seq_len) in enumerate(eval_iter): + ts = time.time() + ret = model(data,target,*mems) + loss, mems = ret[0], ret[1:] + loss = loss.mean() + total_loss += seq_len * loss.item() + total_len += seq_len + #print('eval_batch id: {} use time: {:.2f} ms '.format(idx, (time.time()-ts)*1000)) + total_time = time.time() - start_time + logging('Time : {:.2f}s, FPS: {:.2f} characters/s'.format( + total_time, total_len*args.batch_size*args.eval_tgt_len/total_time)) + return total_loss / total_len + + +# Run on test.py data. +if args.split == 'all': + test_loss = evaluate(te_iter) + valid_loss = evaluate(va_iter) +elif args.split == 'valid': + valid_loss = evaluate(va_iter) + test_loss = None +elif args.split == 'test': + test_loss = evaluate(te_iter) + valid_loss = None + +def format_log(loss, split): + if args.dataset in ['enwik8', 'text8']: + log_str = '| {0} loss {1:5.2f} | {0} bpc {2:9.5f} '.format( + split, loss, loss / math.log(2)) + else: + log_str = '| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '.format( + split, loss, math.exp(loss)) + return log_str + +log_str = '' +if valid_loss is not None: + log_str += format_log(valid_loss, 'valid') +if test_loss is not None: + log_str += format_log(test_loss, 'test.py') + +logging('=' * 100) +logging(log_str) +logging('=' * 100) -- Gitee From 164fd135cbeda09e1331b6c704233c4381153ed0 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 06:34:59 +0000 Subject: [PATCH 15/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh index 90303323f0..bee9514a34 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh @@ -111,7 +111,7 @@ echo "Final Performance images/sec : $FPS" #输出训练精度,需要模型审视修改 train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk 'END {print}'|awk -F "|" '{print $NF}'|awk -F " " '{print $1}'` #打印，不需要修改 -echo "Final Train Accuracy : ${train_accuracy}" +echo "Final Train bpc : ${train_accuracy}" echo "E2E Training Duration sec : $e2e_time" #性能看护结果汇总 -- Gitee From a4ce0bd48907a320689aff9afffc2273603bdc11 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 06:35:31 +0000 Subject: [PATCH 16/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh index 37c3a430f4..6400ae45af 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh @@ -116,7 +116,7 @@ echo "Final Performance images/sec : $FPS" train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|grep -v test|awk 'END {print}'|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'` #打印，不需要修改 -echo "Final Train Accuracy : ${train_accuracy}" +echo "Final Train bpc : ${train_accuracy}" echo "E2E Training Duration sec : $e2e_time" #性能看护结果汇总 -- Gitee From c656f892f16edbe5c289d323343ebe4326d23c61 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 25 Mar 2022 06:55:09 +0000 Subject: [PATCH 17/59] my first commit --- .../nlp/Transformer-xl_for_PyTorch/getdata.sh | 90 +++++++++++++++++++ .../Transformer-xl_for_PyTorch/prep_text8.py | 32 +++++++ 2 files changed, 122 insertions(+) create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh create mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/prep_text8.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh new file mode 100644 index 0000000000..780475780e --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh @@ -0,0 +1,90 @@ +echo "=== Acquiring datasets ===" +echo "---" + +mkdir -p data +cd data + +if [[ ! -d 'wikitext-2' ]]; then + echo "- Downloading WikiText-2 (WT2)" + wget --quiet --continue https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip + unzip -q wikitext-2-v1.zip + cd wikitext-2 + mv wiki.train.tokens train.txt + mv wiki.valid.tokens valid.txt + mv wiki.test.tokens test.txt + cd .. +fi + +echo "- Downloading WikiText-103 (WT2)" +if [[ ! -d 'wikitext-103' ]]; then + wget --continue https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip + unzip -q wikitext-103-v1.zip + cd wikitext-103 + mv wiki.train.tokens train.txt + mv wiki.valid.tokens valid.txt + mv wiki.test.tokens test.txt + cd .. +fi + +echo "- Downloading enwik8 (Character)" +if [[ ! -d 'enwik8' ]]; then + mkdir -p enwik8 + cd enwik8 + wget --continue http://mattmahoney.net/dc/enwik8.zip + wget https://raw.githubusercontent.com/salesforce/awd-lstm-lm/master/data/enwik8/prep_enwik8.py + python3 prep_enwik8.py + cd .. +fi + +echo "- Downloading text8 (Character)" +if [[ ! -d 'text8' ]]; then + mkdir -p text8 + cd text8 + wget --continue http://mattmahoney.net/dc/text8.zip + python ../../prep_text8.py + cd .. +fi + +echo "- Downloading Penn Treebank (PTB)" +if [[ ! -d 'penn' ]]; then + wget --quiet --continue http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz + tar -xzf simple-examples.tgz + + mkdir -p penn + cd penn + mv ../simple-examples/data/ptb.train.txt train.txt + mv ../simple-examples/data/ptb.test.txt test.txt + mv ../simple-examples/data/ptb.valid.txt valid.txt + cd .. + + echo "- Downloading Penn Treebank (Character)" + mkdir -p pennchar + cd pennchar + mv ../simple-examples/data/ptb.char.train.txt train.txt + mv ../simple-examples/data/ptb.char.test.txt test.txt + mv ../simple-examples/data/ptb.char.valid.txt valid.txt + cd .. + + rm -rf simple-examples/ +fi + +echo "- Downloading 1B words" + +if [[ ! -d 'one-billion-words' ]]; then + mkdir -p one-billion-words + cd one-billion-words + + wget --no-proxy http://www.statmt.org/lm-benchmark/1-billion-word-language-modeling-benchmark-r13output.tar.gz + tar xzvf 1-billion-word-language-modeling-benchmark-r13output.tar.gz + + path="1-billion-word-language-modeling-benchmark-r13output/heldout-monolingual.tokenized.shuffled/" + cat ${path}/news.en.heldout-00000-of-00050 > valid.txt + cat ${path}/news.en.heldout-00000-of-00050 > test.txt + + wget https://github.com/rafaljozefowicz/lm/raw/master/1b_word_vocab.txt + + cd .. +fi + +echo "---" +echo "Happy language modeling :)" diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/prep_text8.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/prep_text8.py new file mode 100644 index 0000000000..65b1ce7e15 --- /dev/null +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/prep_text8.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python +# coding=utf-8 + +import os +import sys +import zipfile + +from io import open + +if os.path.exists('train.txt'): + print('Tokenized text8 already exists - skipping processing') + sys.exit() + +data = zipfile.ZipFile('text8.zip').extractall() +data = open('text8', 'r', encoding='utf-8').read() + +print('Length of text8: {}'.format(len(data))) + +num_test_chars = 5000000 + +train_data = data[: -2 * num_test_chars] +valid_data = data[-2 * num_test_chars: -num_test_chars] +test_data = data[-num_test_chars:] + +for fn, part in [('train.txt', train_data), ('valid.txt', valid_data), ('test.txt', test_data)]: + print('{} will have {} bytes'.format(fn, len(part))) + print('- Tokenizing...') + # Change space ' ' to underscore '_' + part_str = ' '.join(['_' if c == ' ' else c for c in part.strip()]) + print('- Writing...') + f = open(fn, 'w').write(part_str) + f = open(fn + '.raw', 'w', encoding='utf-8').write(part) -- Gitee From c84d4b4deb7a8a050ca11c2d0b537be117d2a6a5 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Tue, 29 Mar 2022 06:51:43 +0000 Subject: [PATCH 18/59] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20Transformer-xl=5Ffor?= =?UTF-8?q?=5FPyTorch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/.keep diff --git a/ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/.keep b/ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/.keep new file mode 100644 index 0000000000..e69de29bb2 -- Gitee From 5547e470aec267d2f94e714f6c37f320f86121ba Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Tue, 29 Mar 2022 06:55:32 +0000 Subject: [PATCH 19/59] add ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test.py. --- ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test.py diff --git a/ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test.py b/ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test.py new file mode 100644 index 0000000000..cfcd5066e7 --- /dev/null +++ b/ACL_PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test.py @@ -0,0 +1 @@ +import torch -- Gitee From d7432e9a6ce0183d23a1c53a11d1ffe61cd5c5e9 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:21:53 +0000 Subject: [PATCH 20/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh index ecd1b34c35..18157a65d0 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_1p.sh @@ -9,7 +9,7 @@ batch_size=22 # 训练使用的npu卡数 export RANK_SIZE=1 # 数据集路径,保持为空,不需要修改 -data_path="/home/huangwei/data/enwik8" +data_path="" # 训练epoch train_epochs=1 -- Gitee From 4642241611d1d5efbbd4411cce2c577c94495c7e Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:23:59 +0000 Subject: [PATCH 21/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh. --- .../Transformer-xl_for_PyTorch/test/train_performance_8p.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh index 6400ae45af..b1840ab3cb 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_performance_8p.sh @@ -9,7 +9,7 @@ batch_size=22 # 训练使用的npu卡数 export RANK_SIZE=8 # 数据集路径,保持为空,不需要修改 -data_path="/home/huangwei/data/enwik8" +data_path="" # 训练epoch train_epochs=50 @@ -80,6 +80,7 @@ do taskset -c $PID_START-$PID_END python3.7 -u $(dirname $currentDir)/train_8p_npu.py \ --addr=$(hostname -I |awk '{print $1}') \ --workers=$(nproc) \ + --data=${data_path} \ --multiprocessing-distributed \ --dist-url='tcp://127.0.0.1:50000' \ --dist-backend='hccl' \ @@ -110,7 +111,7 @@ echo "------------------ Final result ------------------" grep "fps" ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log | awk '{print $20}' > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}_fps.log FPS=`cat ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}_fps.log|awk '{a+=$1} END {if (NR !=0) printf("%.3f", a/NR)}'` #打印，不需要修改 -echo "Final Performance images/sec : $FPS" +echo "Final Performance characters/sec : $FPS" #输出训练精度,需要模型审视修改 train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|grep -v test|awk 'END {print}'|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'` -- Gitee From 5822596f5bb8282f570e59c32bbf1e7cceb59dc3 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:25:02 +0000 Subject: [PATCH 22/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh index bee9514a34..1896034f31 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh @@ -9,7 +9,7 @@ batch_size=22 # 训练使用的npu卡数 export RANK_SIZE=8 # 数据集路径,保持为空,不需要修改 -data_path="/home/huangwei/data/enwik8" +data_path="" # 训练epoch train_epochs=40 @@ -80,6 +80,7 @@ do taskset -c $PID_START-$PID_END python3.7 -u $(dirname $currentDir)/train_8p_npu.py \ --addr=$(hostname -I |awk '{print $1}') \ --workers=$(nproc) \ + --data=${data_path} \ --multiprocessing-distributed \ --dist-url='tcp://127.0.0.1:50000' \ --dist-backend='hccl' \ @@ -106,7 +107,7 @@ echo "------------------ Final result ------------------" #输出性能FPS，需要模型审视修改 FPS=`grep -a 'fps' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk -F " " '{print $20}'|awk 'END {print}'` #打印，不需要修改 -echo "Final Performance images/sec : $FPS" +echo "Final Performance characters/sec : $FPS" #输出训练精度,需要模型审视修改 train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk 'END {print}'|awk -F "|" '{print $NF}'|awk -F " " '{print $1}'` -- Gitee From ccb3b8a7c58412ba58ac13dd54174a4bb01dbb24 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:30:34 +0000 Subject: [PATCH 23/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh index da30622701..dd7dc51588 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh @@ -9,9 +9,9 @@ batch_size=22 # 训练使用的npu卡数 export RANK_SIZE=1 # 数据集路径,保持为空,不需要修改 -data_path="/home/huangwei/data/enwik8" +data_path="" # checkpoint文件路径，以实际路径为准 -pth_path="/home/huangwei/transformer-xl" +pth_path="" # 训练epoch train_epochs=50 # 指定训练所使用的npu device卡id @@ -79,7 +79,9 @@ etp_flag=`echo ${check_etp_flag#*=}` if [ x"${etp_flag}" != x"true" ];then source ${test_path_dir}/env_npu.sh fi -python3 -u eval_npu.py --split valid > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/Eval_${ASCEND_DEVICE_ID}.log 2>&1 & +python3 -u eval_npu.py --split valid \ + --data=${data_path} \ + --pth_path=${pth_path} > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/Eval_${ASCEND_DEVICE_ID}.log 2>&1 & wait -- Gitee From 19f942a889f5f07132eccf465aba08ae15ea8333 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:38:24 +0000 Subject: [PATCH 24/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py. --- PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py index 29e80f07e2..2cc3b6a73c 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/eval_npu.py @@ -121,6 +121,8 @@ parser.add_argument('--restart', action='store_true', help='restart training from the saved checkpoint') parser.add_argument('--restart_dir', type=str, default='', help='restart dir') +parser.add_argument('--pth', type=str, default='', + help='eval checkpoint') parser.add_argument('--debug', action='store_true', help='run in debug mode (do not create exp dir)') parser.add_argument('--same_length', action='store_true', @@ -309,7 +311,7 @@ logging('#params = {}'.format(args.n_all_param)) logging('#non emb params = {}'.format(args.n_nonemb_param)) # Load the best saved model. -with open('model_best_bpc.pt', 'rb') as f: +with open(args.pth, 'rb') as f: model.load_state_dict(torch.load(f, map_location=loc)) logging('Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'.format( -- Gitee From 107373d5bc9f5764e3caef14e9a1068f426aa7f5 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:39:07 +0000 Subject: [PATCH 25/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh index dd7dc51588..799985972b 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh @@ -81,7 +81,7 @@ if [ x"${etp_flag}" != x"true" ];then fi python3 -u eval_npu.py --split valid \ --data=${data_path} \ - --pth_path=${pth_path} > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/Eval_${ASCEND_DEVICE_ID}.log 2>&1 & + --pth=${pth_path} > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/Eval_${ASCEND_DEVICE_ID}.log 2>&1 & wait -- Gitee From c414132581cf1bc77230bac2fc9522db7de613c7 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:40:18 +0000 Subject: [PATCH 26/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md. --- PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md index 4d220f9310..9575325dc6 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/Readme.md @@ -26,16 +26,16 @@ cd transformer-xl dos2unix ./test/*.sh # 1p train perf -bash test/train_performance_1p.sh +bash test/train_performance_1p.sh --data_path=xxxx # 8p train perf -bash test/train_performance_8p.sh +bash test/train_performance_8p.sh --data_path=xxxx # 8p train full -bash test/train_full_8p.sh +bash test/train_full_8p.sh --data_path=xxxx # 1p eval -bash test/train_eval_1p.sh +bash test/train_eval_1p.sh --data_path=xxxx --pth_path=xxxx ``` -- Gitee From b2ff58bc0e52edbe4b40206c8a7f057099d2688e Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:52:58 +0000 Subject: [PATCH 27/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh. --- .../nlp/Transformer-xl_for_PyTorch/getdata.sh | 72 ------------------- 1 file changed, 72 deletions(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh index 780475780e..8043d1a1aa 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/getdata.sh @@ -4,28 +4,6 @@ echo "---" mkdir -p data cd data -if [[ ! -d 'wikitext-2' ]]; then - echo "- Downloading WikiText-2 (WT2)" - wget --quiet --continue https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip - unzip -q wikitext-2-v1.zip - cd wikitext-2 - mv wiki.train.tokens train.txt - mv wiki.valid.tokens valid.txt - mv wiki.test.tokens test.txt - cd .. -fi - -echo "- Downloading WikiText-103 (WT2)" -if [[ ! -d 'wikitext-103' ]]; then - wget --continue https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-103-v1.zip - unzip -q wikitext-103-v1.zip - cd wikitext-103 - mv wiki.train.tokens train.txt - mv wiki.valid.tokens valid.txt - mv wiki.test.tokens test.txt - cd .. -fi - echo "- Downloading enwik8 (Character)" if [[ ! -d 'enwik8' ]]; then mkdir -p enwik8 @@ -36,55 +14,5 @@ if [[ ! -d 'enwik8' ]]; then cd .. fi -echo "- Downloading text8 (Character)" -if [[ ! -d 'text8' ]]; then - mkdir -p text8 - cd text8 - wget --continue http://mattmahoney.net/dc/text8.zip - python ../../prep_text8.py - cd .. -fi - -echo "- Downloading Penn Treebank (PTB)" -if [[ ! -d 'penn' ]]; then - wget --quiet --continue http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz - tar -xzf simple-examples.tgz - - mkdir -p penn - cd penn - mv ../simple-examples/data/ptb.train.txt train.txt - mv ../simple-examples/data/ptb.test.txt test.txt - mv ../simple-examples/data/ptb.valid.txt valid.txt - cd .. - - echo "- Downloading Penn Treebank (Character)" - mkdir -p pennchar - cd pennchar - mv ../simple-examples/data/ptb.char.train.txt train.txt - mv ../simple-examples/data/ptb.char.test.txt test.txt - mv ../simple-examples/data/ptb.char.valid.txt valid.txt - cd .. - - rm -rf simple-examples/ -fi - -echo "- Downloading 1B words" - -if [[ ! -d 'one-billion-words' ]]; then - mkdir -p one-billion-words - cd one-billion-words - - wget --no-proxy http://www.statmt.org/lm-benchmark/1-billion-word-language-modeling-benchmark-r13output.tar.gz - tar xzvf 1-billion-word-language-modeling-benchmark-r13output.tar.gz - - path="1-billion-word-language-modeling-benchmark-r13output/heldout-monolingual.tokenized.shuffled/" - cat ${path}/news.en.heldout-00000-of-00050 > valid.txt - cat ${path}/news.en.heldout-00000-of-00050 > test.txt - - wget https://github.com/rafaljozefowicz/lm/raw/master/1b_word_vocab.txt - - cd .. -fi - echo "---" echo "Happy language modeling :)" -- Gitee From 5aa6ca244398f16f47ed7219f142b975900b7160 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 02:53:21 +0000 Subject: [PATCH 28/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/nlp/Transformer-xl=5Ffor=5FPyTorch/prep=5Ftext8.p?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Transformer-xl_for_PyTorch/prep_text8.py | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/prep_text8.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/prep_text8.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/prep_text8.py deleted file mode 100644 index 65b1ce7e15..0000000000 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/prep_text8.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 - -import os -import sys -import zipfile - -from io import open - -if os.path.exists('train.txt'): - print('Tokenized text8 already exists - skipping processing') - sys.exit() - -data = zipfile.ZipFile('text8.zip').extractall() -data = open('text8', 'r', encoding='utf-8').read() - -print('Length of text8: {}'.format(len(data))) - -num_test_chars = 5000000 - -train_data = data[: -2 * num_test_chars] -valid_data = data[-2 * num_test_chars: -num_test_chars] -test_data = data[-num_test_chars:] - -for fn, part in [('train.txt', train_data), ('valid.txt', valid_data), ('test.txt', test_data)]: - print('{} will have {} bytes'.format(fn, len(part))) - print('- Tokenizing...') - # Change space ' ' to underscore '_' - part_str = ' '.join(['_' if c == ' ' else c for c in part.strip()]) - print('- Writing...') - f = open(fn, 'w').write(part_str) - f = open(fn + '.raw', 'w', encoding='utf-8').write(part) -- Gitee From 786d915a1e1a0f655208cfe11535d79edd933987 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 03:29:12 +0000 Subject: [PATCH 29/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh index 1896034f31..08e2ce8c06 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh @@ -12,7 +12,7 @@ export RANK_SIZE=8 data_path="" # 训练epoch -train_epochs=40 +train_epochs=50 # 学习率 learning_rate=0.00025 # 加载数据进程数 -- Gitee From 450ac2ea0b2a013cf7e636f7616498190cdf2bed Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 03:33:21 +0000 Subject: [PATCH 30/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh index 08e2ce8c06..8a240838cb 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh @@ -110,7 +110,7 @@ FPS=`grep -a 'fps' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_D echo "Final Performance characters/sec : $FPS" #输出训练精度,需要模型审视修改 -train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk 'END {print}'|awk -F "|" '{print $NF}'|awk -F " " '{print $1}'` +train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|grep -v test|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'|awk 'END {print}'` #打印，不需要修改 echo "Final Train bpc : ${train_accuracy}" echo "E2E Training Duration sec : $e2e_time" -- Gitee From 1ce048fdd05c1d95e3930623c97d11b8d1591fca Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 03:41:19 +0000 Subject: [PATCH 31/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py. --- .../contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py index 7ef2b16e75..8c0cae7896 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py @@ -84,11 +84,11 @@ parser.add_argument('--decay_rate', type=float, default=0.5, help='decay factor when ReduceLROnPlateau is used') parser.add_argument('--lr_min', type=float, default=0.0, help='minimum learning rate during annealing') -parser.add_argument('--clip', type=float, default=0.25, # 源码中 clip 的 default=0.25 +parser.add_argument('--clip', type=float, default=0.25, help='gradient clipping') parser.add_argument('--clip_nonemb', action='store_true', help='only clip the gradient of non-embedding params') -parser.add_argument('--max_step', type=int, default=400000, +parser.add_argument('--max_step', type=int, default=1000, help='upper epoch limit') parser.add_argument('--batch_size', type=int, default=22, help='batch size') @@ -119,7 +119,7 @@ parser.add_argument('--multi_gpu', action='store_true', help='use multiple GPU') parser.add_argument('--log-interval', type=int, default=200, help='report interval') -parser.add_argument('--eval-interval', type=int, default=4000, +parser.add_argument('--eval-interval', type=int, default=1000, help='evaluation interval') parser.add_argument('--work_dir', default='LM-TFM', type=str, help='experiment directory.') -- Gitee From 728424693cb5eb72f61f7b94f6e5f7dbdb1352fb Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 11:00:48 +0000 Subject: [PATCH 32/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py. --- .../contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py index 8c0cae7896..fb782e84f1 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/train_8p_npu.py @@ -571,10 +571,6 @@ def main_worker(gpu, ngpus_per_node, args): log_start_time = time.time() if train_step % args.eval_interval == 0: - print('train_step is :', train_step) - print('ars.eval_interval is :', args.eval_interval) - print(train_step % args.eval_interval) - print('*'*50) ts = time.time() val_loss = evaluate(va_iter) print('evaluation use time {} s'.format(time.time()-ts)) -- Gitee From 915d5c309ca812db7d3ef5ce4c048057ca7ba16b Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 11:30:33 +0000 Subject: [PATCH 33/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh index 8a240838cb..f77b282e9a 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_full_8p.sh @@ -110,7 +110,7 @@ FPS=`grep -a 'fps' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_D echo "Final Performance characters/sec : $FPS" #输出训练精度,需要模型审视修改 -train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|grep -v test|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'|awk 'END {print}'` +train_accuracy=`grep -a 'Eval' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'|awk 'END {print}'` #打印，不需要修改 echo "Final Train bpc : ${train_accuracy}" echo "E2E Training Duration sec : $e2e_time" -- Gitee From 07cf7b422709f40ea5a50d3db8f01d23259f05b5 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 11:42:22 +0000 Subject: [PATCH 34/59] update PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh. --- .../nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh index 799985972b..9b97a2636d 100644 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh +++ b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/test/train_eval_1p.sh @@ -95,7 +95,7 @@ e2e_time=$(( $end_time - $start_time )) echo "------------------ Final result ------------------" #输出训练精度,需要模型审视修改 -train_accuracy=`grep -a 'bpc' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/Eval_${ASCEND_DEVICE_ID}.log|grep -v test|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'|awk 'END {print}'` +train_accuracy=`grep -a 'Eval' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/Eval_${ASCEND_DEVICE_ID}.log|awk -F "|" '{print $NF}'|awk -F " " '{print $NF}'|awk 'END {print}'` #打印，不需要修改 echo "Final Train bpc : ${train_accuracy}" echo "E2E Training Duration sec : $e2e_time" -- Gitee From e7358df01e2942b2cef915c6bf71d960ef3c56fd Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Fri, 1 Apr 2022 11:42:34 +0000 Subject: [PATCH 35/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/nlp/Transformer-xl=5Ffor=5FPyTorch/adaptive=5Fsof?= =?UTF-8?q?tmax.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../adaptive_softmax.py | 97 ------------------- 1 file changed, 97 deletions(-) delete mode 100644 PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py diff --git a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py b/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py deleted file mode 100644 index 1a65fb1c72..0000000000 --- a/PyTorch/contrib/nlp/Transformer-xl_for_PyTorch/adaptive_softmax.py +++ /dev/null @@ -1,97 +0,0 @@ -from collections import defaultdict - -import numpy as np - -import torch -import torch.nn as nn -import torch.nn.functional as F - -class AdaptiveLogSoftmax(nn.Module): - def __init__(self, in_features, n_classes, cutoffs, keep_order=False): - super(AdaptiveLogSoftmax, self).__init__() - - cutoffs = list(cutoffs) - - if (cutoffs != sorted(cutoffs)) \ - or (min(cutoffs) <= 0) \ - or (max(cutoffs) >= (n_classes - 1)) \ - or (len(set(cutoffs)) != len(cutoffs)) \ - or any([int(c) != c for c in cutoffs]): - - raise ValueError("cutoffs should be a sequence of unique, positive " - "integers sorted in an increasing order, where " - "each value is between 1 and n_classes-1") - - self.in_features = in_features - self.n_classes = n_classes - self.cutoffs = cutoffs + [n_classes] - - self.shortlist_size = self.cutoffs[0] - self.n_clusters = len(self.cutoffs) - 1 - self.head_size = self.shortlist_size + self.n_clusters - - self.cluster_weight = nn.Parameter(torch.zeros(self.n_clusters, self.in_features)) - self.cluster_bias = nn.Parameter(torch.zeros(self.n_clusters)) - - self.keep_order = keep_order - - - def forward(self, hidden, target, weight, bias, keep_order=False): - if hidden.size(0) != target.size(0): - raise RuntimeError('Input and target should have the same size ' - 'in the batch dimension.') - - head_weight = torch.cat( - [weight[:self.shortlist_size], self.cluster_weight], dim=0) - head_bias = torch.cat( - [bias[:self.shortlist_size], self.cluster_bias], dim=0) - - head_logit = F.linear(hidden, head_weight, bias=head_bias) - head_logprob = F.log_softmax(head_logit, dim=1) - - nll = torch.zeros_like(target, - dtype=hidden.dtype, device=hidden.device) - - offset = 0 - cutoff_values = [0] + self.cutoffs - for i in range(len(cutoff_values) - 1): - l_idx, h_idx = cutoff_values[i], cutoff_values[i + 1] - - mask_i = (target >= l_idx) & (target < h_idx) - indices_i = mask_i.nonzero().squeeze() - - if indices_i.numel() == 0: - continue - - target_i = target.index_select(0, indices_i) - l_idx - head_logprob_i = head_logprob.index_select(0, indices_i) - - if i == 0: - logprob_i = head_logprob_i.gather(1, target_i[:,None]).squeeze(1) - else: - weight_i = weight[l_idx:h_idx] - bias_i = bias[l_idx:h_idx] - - hidden_i = hidden.index_select(0, indices_i) - - tail_logit_i = F.linear(hidden_i, weight_i, bias=bias_i) - tail_logprob_i = F.log_softmax(tail_logit_i, dim=1) - - # aa = target_i[:, None] - # aa = aa.to('cpu') - # logprob_i = head_logprob_i[:, -i] \ - # + tail_logprob_i.gather(1, aa).squeeze(1) - - print(f'target_i[:,None]: {target_i[:, None]}') - print(f'target_i[:,None].shape: {target_i[:, None].shape}') - logprob_i = head_logprob_i[:, -i] \ - + tail_logprob_i.gather(1, target_i[:,None]).squeeze(1) - - if (hasattr(self, 'keep_order') and self.keep_order) or keep_order: - nll.index_copy_(0, indices_i, -logprob_i) - else: - nll[offset:offset+logprob_i.size(0)].copy_(-logprob_i) - - offset += logprob_i.size(0) - - return nll -- Gitee From 0a6c6c7ccb885c368cd7bcb17dd40d7091105dac Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 06:04:26 +0000 Subject: [PATCH 36/59] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20DETR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PyTorch/contrib/cv/detection/DETR/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 PyTorch/contrib/cv/detection/DETR/.keep diff --git a/PyTorch/contrib/cv/detection/DETR/.keep b/PyTorch/contrib/cv/detection/DETR/.keep new file mode 100644 index 0000000000..e69de29bb2 -- Gitee From fac6e62b71454614444a91b82c0ea1ab73830232 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 06:05:11 +0000 Subject: [PATCH 37/59] my first commit --- .../contrib/cv/detection/DETR/test/env_npu.sh | 71 +++++++++ .../cv/detection/DETR/test/train_full_8p.sh | 147 ++++++++++++++++++ .../DETR/test/train_performance_1p.sh | 142 +++++++++++++++++ .../DETR/test/train_performance_8p.sh | 147 ++++++++++++++++++ 4 files changed, 507 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/test/env_npu.sh create mode 100644 PyTorch/contrib/cv/detection/DETR/test/train_full_8p.sh create mode 100644 PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh create mode 100644 PyTorch/contrib/cv/detection/DETR/test/train_performance_8p.sh diff --git a/PyTorch/contrib/cv/detection/DETR/test/env_npu.sh b/PyTorch/contrib/cv/detection/DETR/test/env_npu.sh new file mode 100644 index 0000000000..280fca96da --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/test/env_npu.sh @@ -0,0 +1,71 @@ +#!/bin/bash +export install_path=/usr/local/Ascend + +if [ -d ${install_path}/toolkit ]; then + export LD_LIBRARY_PATH=/usr/include/hdf5/lib/:/usr/local/:/usr/local/lib/:/usr/lib/:${install_path}/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons:${path_lib}:${LD_LIBRARY_PATH} + export PATH=${install_path}/fwkacllib/ccec_compiler/bin:${install_path}/fwkacllib/bin:$PATH + export PYTHONPATH=${install_path}/fwkacllib/python/site-packages:${install_path}/tfplugin/python/site-packages:${install_path}/toolkit/python/site-packages:$PYTHONPATH + export PYTHONPATH=/usr/local/python3.7.5/lib/python3.7/site-packages:$PYTHONPATH + export ASCEND_OPP_PATH=${install_path}/opp +else + if [ -d ${install_path}/nnae/latest ];then + export LD_LIBRARY_PATH=/usr/local/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:/usr/local/lib/:/usr/lib64/:/usr/lib/:${install_path}/nnae/latest/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons/:/usr/lib/aarch64_64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:${install_path}/nnae/latest/fwkacllib/ccec_compiler/bin/:${install_path}/nnae/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=${install_path}/nnae/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:${install_path}/nnae/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=${install_path}/nnae/latest/fwkacllib/python/site-packages/:${install_path}/nnae/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:${install_path}/nnae/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + export ASCEND_AICPU_PATH=${install_path}/nnae/latest + else + export LD_LIBRARY_PATH=/usr/local/:/usr/local/lib/:/usr/lib64/:/usr/lib/:/usr/local/python3.7.5/lib/:/usr/local/openblas/lib:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/:${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:${install_path}/add-ons/:/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH + export PATH=$PATH:${install_path}/ascend-toolkit/latest/fwkacllib/ccec_compiler/bin/:${install_path}/ascend-toolkit/latest/toolkit/tools/ide_daemon/bin/ + export ASCEND_OPP_PATH=${install_path}/ascend-toolkit/latest/opp/ + export OPTION_EXEC_EXTERN_PLUGIN_PATH=${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libfe.so:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libaicpu_engine.so:${install_path}/ascend-toolkit/latest/fwkacllib/lib64/plugin/opskernel/libge_local_engine.so + export PYTHONPATH=${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/:${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/auto_tune.egg/auto_tune:${install_path}/ascend-toolkit/latest/fwkacllib/python/site-packages/schedule_search.egg:$PYTHONPATH + export ASCEND_AICPU_PATH=${install_path}/ascend-toolkit/latest + fi +fi + + +#将Host日志输出到串口,0-关闭/1-开启 +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +#设置默认日志级别,0-debug/1-info/2-warning/3-error +export ASCEND_GLOBAL_LOG_LEVEL=3 +#设置Host侧Event日志开启标志,0-关闭/1-开启 +export ASCEND_GLOBAL_EVENT_ENABLE=0 +#设置是否开启taskque,0-关闭/1-开启 +export TASK_QUEUE_ENABLE=1 +#设置是否开启PTCopy,0-关闭/1-开启 +export PTCOPY_ENABLE=1 +#设置是否开启combined标志,0-关闭/1-开启 +export COMBINED_ENABLE=1 +#设置特殊场景是否需要重新编译,不需要修改 +export DYNAMIC_OP="ADD#MUL" +#HCCL白名单开关,1-关闭/0-开启 +export HCCL_WHITELIST_DISABLE=1 +#设置Device侧日志等级为error +${install_path}/driver/tools/msnpureport -g error +#关闭Device侧Event日志 +${install_path}/driver/tools/msnpureport -e disable +export BMMV2_ENABLE=1 + +path_lib=$(python3.7 -c """ +import sys +import re +result='' +for index in range(len(sys.path)): + match_sit = re.search('-packages', sys.path[index]) + if match_sit is not None: + match_lib = re.search('lib', sys.path[index]) + + if match_lib is not None: + end=match_lib.span()[1] + result += sys.path[index][0:end] + ':' + + result+=sys.path[index] + '/torch/lib:' +print(result)""" +) + +echo ${path_lib} + +export LD_LIBRARY_PATH=/usr/local/python3.7.5/lib/:${path_lib}:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/gcc7.3.0/lib64:${LD_LIBRARY_PATH} \ No newline at end of file diff --git a/PyTorch/contrib/cv/detection/DETR/test/train_full_8p.sh b/PyTorch/contrib/cv/detection/DETR/test/train_full_8p.sh new file mode 100644 index 0000000000..9522f2fb28 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/test/train_full_8p.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +################基础配置参数，需要模型审视修改################## +# 必选字段(必须在此处定义的参数): Network batch_size RANK_SIZE +# 网络名称，同目录名称 +Network="DETR_for_PyTorch" +# 训练batch_size +batch_size=8 +# 训练使用的npu卡数 +export RANK_SIZE=8 +# 数据集路径,保持为空,不需要修改 +data_path="" + +# 训练epoch +train_epochs=300 +# 学习率 +learning_rate=0.0001 +# 加载数据进程数 +workers=128 + + +# 参数校验，data_path为必传参数，其他参数的增删由模型自身决定；此处新增参数需在上面有定义并赋值 +for para in $* +do + if [[ $para == --workers* ]];then + workers=`echo ${para#*=}` + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi + + +###############指定训练脚本执行路径############### +# cd到与test文件夹同层级目录下执行脚本，提高兼容性；test_path_dir为包含test文件夹的路径 +cur_path=`pwd` +cur_path_last_diename=${cur_path##*/} +if [ x"${cur_path_last_diename}" == x"test" ];then + test_path_dir=${cur_path} + cd .. + cur_path=`pwd` +else + test_path_dir=${cur_path}/test +fi + + +#################创建日志输出目录，不需要修改################# +ASCEND_DEVICE_ID=0 +if [ -d ${test_path_dir}/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${test_path_dir}/output/${ASCEND_DEVICE_ID} + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +else + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +fi + + +#################启动训练脚本################# +#训练开始时间，不需要修改 +start_time=$(date +%s) +# 非平台场景时source 环境变量 +check_etp_flag=`env | grep etp_running_flag` +etp_flag=`echo ${check_etp_flag#*=}` +if [ x"${etp_flag}" != x"true" ];then + source ${test_path_dir}/env_npu.sh +fi + + + +device_id_list=0,1,2,3,4,5,6,7 +KERNEL_NUM=$(($(nproc)/8)) +for i in $(seq 0 7) +do + PID_START=$((KERNEL_NUM * i)) + PID_END=$((PID_START + KERNEL_NUM - 1)) + taskset -c $PID_START-$PID_END python3.7 -u train_npu_hw.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --workers=$(nproc) \ + --multiprocessing_distributed \ + --dist_url='tcp://127.0.0.1:50000' \ + --dist_backend='hccl' \ + --epochs=${train_epochs} \ + --lr=${learning_rate} \ + --world_size=1 \ + --batch_size=${batch_size} \ + --device_num=8 \ + --rank=0 \ + --device_list=${device_id_list} \ + --local_rank=$i > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & +done + + +wait + + + +##################获取训练数据################ +#训练结束时间，不需要修改 + +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +FPS=`grep -a 'FPS' test/output/0/train_0.log|awk -F " " '{print $NF}'|awk -F ":" '{print $2}'|awk 'END {print}'` +#打印，不需要修改 +echo "Final Performance images/sec : $FPS" + +# #输出训练精度,需要模型审视修改 +train_accuracy=`grep -a "Average Precision" test/output/0/train_0.log|awk -F "=" '{print $NF}'|awk 'END {print}'` +# #打印，不需要修改 +echo "Final Train Accuracy : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'perf' + +##获取性能数据，不需要修改 +#吞吐量 +ActualFPS=${FPS} +#单迭代训练时长 +TrainingTime=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'*1000/'${FPS}'}'` + +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视 +grep "epoch" ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_$ASCEND_DEVICE_ID.log|grep -v test|awk -F "|" '{print $6}' | awk -F " " '{print $NF}' > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${CaseName}_loss.txt + +#最后一个迭代loss值，不需要修改 +ActualLoss=`awk 'END {print}' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log diff --git a/PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh b/PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh new file mode 100644 index 0000000000..0fb18080be --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +################基础配置参数，需要模型审视修改################## +# 必选字段(必须在此处定义的参数): Network batch_size RANK_SIZE +# 网络名称，同目录名称 +Network="DETR_for_PyTorch" +# 训练batch_size +batch_size=8 +# 训练使用的npu卡数 +export RANK_SIZE=1 +# 数据集路径,保持为空,不需要修改 +data_path="" + +# 训练epoch +train_epochs=1 +# 指定训练所使用的npu device卡id +device_id=0 +# 加载数据进程数 +workers=128 + + +# 参数校验，data_path为必传参数，其他参数的增删由模型自身决定；此处新增参数需在上面有定义并赋值 +for para in $* +do + if [[ $para == --device_id* ]];then + device_id=`echo ${para#*=}` + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi +# 校验是否指定了device_id,分动态分配device_id与手动指定device_id,此处不需要修改 +if [ $ASCEND_DEVICE_ID ];then + echo "device id is ${ASCEND_DEVICE_ID}" +elif [ ${device_id} ];then + export ASCEND_DEVICE_ID=${device_id} + echo "device id is ${ASCEND_DEVICE_ID}" +else + "[Error] device id must be config" + exit 1 +fi + + + +###############指定训练脚本执行路径############### +# cd到与test文件夹同层级目录下执行脚本，提高兼容性；test_path_dir为包含test文件夹的路径 +cur_path=`pwd` +cur_path_last_dirname=${cur_path##*/} +if [ x"${cur_path_last_dirname}" == x"test" ];then + test_path_dir=${cur_path} + cd .. + cur_path=`pwd` +else + test_path_dir=${cur_path}/test +fi + + +#################创建日志输出目录，不需要修改################# +if [ -d ${test_path_dir}/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${test_path_dir}/output/${ASCEND_DEVICE_ID} + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +else + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +fi + + +#################启动训练脚本################# +#训练开始时间，不需要修改 +start_time=$(date +%s) +# 非平台场景时source 环境变量 +check_etp_flag=`env | grep etp_running_flag` +etp_flag=`echo ${check_etp_flag#*=}` +if [ x"${etp_flag}" != x"true" ];then + source ${test_path_dir}/env_npu.sh +fi + + +nohup taskset -c 0-23 python3.7 train_npu_hw.py \ + --coco_path=${data_path} \ + --workers=${workers} \ + --gpu=${ASCEND_DEVICE_ID} \ + --epochs=${train_epochs} \ + --opt_level='O0' \ + --batch_size=${batch_size} > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & + +wait + + +##################获取训练数据################ +#训练结束时间，不需要修改 +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +FPS=`grep -a 'FPS' test/output/0/train_0.log|awk -F " " '{print $NF}'|awk -F ":" '{print $2}'|awk 'END {print}'` +#打印，不需要修改 +echo "Final Performance images/sec : $FPS" + +# #输出训练精度,需要模型审视修改 +train_accuracy=`grep -a "Average Precision" test/output/0/train_0.log|awk -F "=" '{print $NF}'|awk 'END {print}'` +# #打印，不需要修改 +echo "Final Train Accuracy : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'acc' + +##获取性能数据，不需要修改 +#吞吐量 +ActualFPS=${FPS} +#单迭代训练时长 +TrainingTime=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'*1000/'${FPS}'}'` + +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视 +grep -a loss ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log|awk -F "\t" '{print $2}'|awk -F ":" '{print $2}'|awk 'END {print}' >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt + +#最后一个迭代loss值，不需要修改 +ActualLoss=`awk 'END {print}' ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainAccuracy = ${train_accuracy}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log + + diff --git a/PyTorch/contrib/cv/detection/DETR/test/train_performance_8p.sh b/PyTorch/contrib/cv/detection/DETR/test/train_performance_8p.sh new file mode 100644 index 0000000000..3d36493e1b --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/test/train_performance_8p.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +################基础配置参数，需要模型审视修改################## +# 必选字段(必须在此处定义的参数): Network batch_size RANK_SIZE +# 网络名称，同目录名称 +Network="DETR_for_PyTorch" +# 训练batch_size +batch_size=8 +# 训练使用的npu卡数 +export RANK_SIZE=8 +# 数据集路径,保持为空,不需要修改 +data_path="" + +# 训练epoch +train_epochs=2 +# 学习率 +learning_rate=0.0001 +# 加载数据进程数 +workers=128 + + +# 参数校验，data_path为必传参数，其他参数的增删由模型自身决定；此处新增参数需在上面有定义并赋值 +for para in $* +do + if [[ $para == --workers* ]];then + workers=`echo ${para#*=}` + elif [[ $para == --data_path* ]];then + data_path=`echo ${para#*=}` + fi +done + +# 校验是否传入data_path,不需要修改 +if [[ $data_path == "" ]];then + echo "[Error] para \"data_path\" must be confing" + exit 1 +fi + + +###############指定训练脚本执行路径############### +# cd到与test文件夹同层级目录下执行脚本，提高兼容性；test_path_dir为包含test文件夹的路径 +cur_path=`pwd` +cur_path_last_diename=${cur_path##*/} +if [ x"${cur_path_last_diename}" == x"test" ];then + test_path_dir=${cur_path} + cd .. + cur_path=`pwd` +else + test_path_dir=${cur_path}/test +fi + + +#################创建日志输出目录，不需要修改################# +ASCEND_DEVICE_ID=0 +if [ -d ${test_path_dir}/output/${ASCEND_DEVICE_ID} ];then + rm -rf ${test_path_dir}/output/${ASCEND_DEVICE_ID} + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +else + mkdir -p ${test_path_dir}/output/$ASCEND_DEVICE_ID +fi + + +#################启动训练脚本################# +#训练开始时间，不需要修改 +start_time=$(date +%s) +# 非平台场景时source 环境变量 +check_etp_flag=`env | grep etp_running_flag` +etp_flag=`echo ${check_etp_flag#*=}` +if [ x"${etp_flag}" != x"true" ];then + source ${test_path_dir}/env_npu.sh +fi + + + +device_id_list=0,1,2,3,4,5,6,7 +KERNEL_NUM=$(($(nproc)/8)) +for i in $(seq 0 7) +do + PID_START=$((KERNEL_NUM * i)) + PID_END=$((PID_START + KERNEL_NUM - 1)) + taskset -c $PID_START-$PID_END python3.7 -u train_npu_hw.py \ + --addr=$(hostname -I |awk '{print $1}') \ + --workers=$(nproc) \ + --multiprocessing_distributed \ + --dist_url='tcp://127.0.0.1:50000' \ + --dist_backend='hccl' \ + --epochs=${train_epochs} \ + --lr=${learning_rate} \ + --world_size=1 \ + --batch_size=${batch_size} \ + --device_num=8 \ + --rank=0 \ + --device_list=${device_id_list} \ + --local_rank=$i > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log 2>&1 & +done + + +wait + + + +##################获取训练数据################ +#训练结束时间，不需要修改 + +end_time=$(date +%s) +e2e_time=$(( $end_time - $start_time )) + +#结果打印，不需要修改 +echo "------------------ Final result ------------------" +#输出性能FPS，需要模型审视修改 +FPS=`grep -a 'FPS' test/output/0/train_0.log|awk -F " " '{print $NF}'|awk -F ":" '{print $2}'|awk 'END {print}'` +#打印，不需要修改 +echo "Final Performance images/sec : $FPS" + +# #输出训练精度,需要模型审视修改 +train_accuracy=`grep -a "Average Precision" test/output/0/train_0.log|awk -F "=" '{print $NF}'|awk 'END {print}'` +# #打印，不需要修改 +echo "Final Train Accuracy : ${train_accuracy}" +echo "E2E Training Duration sec : $e2e_time" + +#性能看护结果汇总 +#训练用例信息，不需要修改 +BatchSize=${batch_size} +DeviceType=`uname -m` +CaseName=${Network}_bs${BatchSize}_${RANK_SIZE}'p'_'perf' + +##获取性能数据，不需要修改 +#吞吐量 +ActualFPS=${FPS} +#单迭代训练时长 +TrainingTime=`awk 'BEGIN{printf "%.2f\n", '${batch_size}'*1000/'${FPS}'}'` + +#从train_$ASCEND_DEVICE_ID.log提取Loss到train_${CaseName}_loss.txt中，需要根据模型审视 +grep "epoch" ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_$ASCEND_DEVICE_ID.log|grep -v test|awk -F "|" '{print $6}' | awk -F " " '{print $NF}' > ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${CaseName}_loss.txt + +#最后一个迭代loss值，不需要修改 +ActualLoss=`awk 'END {print}' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${CaseName}_loss.txt` + +#关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" > ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${RANK_SIZE}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualFPS = ${ActualFPS}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "ActualLoss = ${ActualLoss}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "E2ETrainingTime = ${e2e_time}" >> ${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log -- Gitee From 219e14682f6171a376090a2b93fa4a81443d394e Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 06:06:14 +0000 Subject: [PATCH 38/59] my first commit --- .../contrib/cv/detection/DETR/train_npu_hw.py | 375 ++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/train_npu_hw.py diff --git a/PyTorch/contrib/cv/detection/DETR/train_npu_hw.py b/PyTorch/contrib/cv/detection/DETR/train_npu_hw.py new file mode 100644 index 0000000000..6f73582634 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/train_npu_hw.py @@ -0,0 +1,375 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import argparse +import datetime +import json +import random +import time +from pathlib import Path + +import numpy as np +import torch +from torch.utils.data import DataLoader, DistributedSampler + +import datasets +import util.misc as utils +from datasets import build_dataset, get_coco_api_from_dataset +from engine import evaluate, train_one_epoch +from models import build_model + +import apex +from apex import amp +from apex.parallel import convert_syncbn_model +from apex.parallel import DistributedDataParallel +import torch.distributed as dist +import os +import warnings + +def get_args_parser(): + parser = argparse.ArgumentParser('Set transformer detector', add_help=False) + parser.add_argument('--lr', default=1e-4, type=float) + parser.add_argument('--lr_backbone', default=1e-5, type=float) + parser.add_argument('--batch_size', default=8, type=int) + parser.add_argument('--weight_decay', default=1e-4, type=float) + parser.add_argument('--epochs', default=400, type=int) + parser.add_argument('--lr_drop', default=200, type=int) + parser.add_argument('--clip_max_norm', default=0.1, type=float, + help='gradient clipping max norm') + + # Model parameters + parser.add_argument('--frozen_weights', type=str, default=None, + help="Path to the pretrained model. If set, only the mask head will be trained") + # * Backbone + parser.add_argument('--backbone', default='resnet50', type=str, + help="Name of the convolutional backbone to use") + parser.add_argument('--dilation', action='store_true', + help="If true, we replace stride with dilation in the last convolutional block (DC5)") + parser.add_argument('--position_embedding', default='sine', type=str, choices=('sine', 'learned'), + help="Type of positional embedding to use on top of the image features") + + # * Transformer + parser.add_argument('--enc_layers', default=6, type=int, + help="Number of encoding layers in the transformer") + parser.add_argument('--dec_layers', default=6, type=int, + help="Number of decoding layers in the transformer") + parser.add_argument('--dim_feedforward', default=2048, type=int, + help="Intermediate size of the feedforward layers in the transformer blocks") + parser.add_argument('--hidden_dim', default=256, type=int, + help="Size of the embeddings (dimension of the transformer)") + parser.add_argument('--dropout', default=0.1, type=float, + help="Dropout applied in the transformer") + parser.add_argument('--nheads', default=8, type=int, + help="Number of attention heads inside the transformer's attentions") + parser.add_argument('--num_queries', default=100, type=int, + help="Number of query slots") + parser.add_argument('--pre_norm', action='store_true') + + # * Segmentation + parser.add_argument('--masks', action='store_true', + help="Train segmentation head if the flag is provided") + + # Loss + parser.add_argument('--no_aux_loss', dest='aux_loss', action='store_false', + help="Disables auxiliary decoding losses (loss at each layer)") + # * Matcher + parser.add_argument('--set_cost_class', default=1, type=float, + help="Class coefficient in the matching cost") + parser.add_argument('--set_cost_bbox', default=5, type=float, + help="L1 box coefficient in the matching cost") + parser.add_argument('--set_cost_giou', default=2, type=float, + help="giou box coefficient in the matching cost") + # * Loss coefficients + parser.add_argument('--mask_loss_coef', default=1, type=float) + parser.add_argument('--dice_loss_coef', default=1, type=float) + parser.add_argument('--bbox_loss_coef', default=5, type=float) + parser.add_argument('--giou_loss_coef', default=2, type=float) + parser.add_argument('--eos_coef', default=0.1, type=float, + help="Relative classification weight of the no-object class") + + # dataset parameters + parser.add_argument('--dataset_file', default='coco') + parser.add_argument('--coco_path', type=str, default='/opt/npu/dataset/coco') + parser.add_argument('--coco_panoptic_path', type=str) + parser.add_argument('--remove_difficult', action='store_true') + + parser.add_argument('--output_dir', default='', + help='path where to save, empty for no saving') + parser.add_argument('--device', default='npu', + help='device to use for training / testing') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--resume', default='', help='resume from checkpoint') + parser.add_argument('--start_epoch', default=0, type=int, metavar='N', + help='start epoch') + parser.add_argument('--eval', action='store_true') + parser.add_argument('--num_workers', default=8, type=int) + + # edit this for 8p + parser.add_argument('--dist-backend', type=str, default='hccl') + parser.add_argument('--distributed', type=bool, default=True) + parser.add_argument('--world-size', type=int, default=-1) + parser.add_argument('--rank', type=int, default=-1) + parser.add_argument('--local_rank', type=int, default=0) + parser.add_argument('--addr', type=str, default='127.0.0.1') + parser.add_argument('--device_num', type=int, default=-1) + parser.add_argument('--workers', type=int, default=32) + parser.add_argument('--device-list', default='', type=str) + parser.add_argument('--dist-url', type=str, default='tcp://127.0.0.1:50000') + parser.add_argument('--gpu', default=None, type=int, help='GPU id to use.') + parser.add_argument('--multiprocessing-distributed', action='store_true', + help='Use multi-processing distributed training to launch ' + 'N processes per node, which has N GPUs. This is the ' + 'fastest way to use PyTorch for either single node or ' + 'multi node data parallel training') + warnings.filterwarnings('ignore') + #############end################# + return parser + +def main(args): + torch.manual_seed(args.seed) + ############################## + # edit this for 8p + os.environ['MASTER_ADDR'] = args.addr + os.environ['MASTER_PORT'] = '29888' + os.environ['LOCAL_DEVICE_ID'] = str(0) + print("+++++++++++++++++++++++++++LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID']) + if args.dist_url == "env://" and args.world_size == -1: + args.world_size = int(os.environ["WORLD_SIZE"]) + print('==========>args.world_size: ', args.world_size) + args.distributed = args.world_size > 1 or args.multiprocessing_distributed + + if args.device_list != '': + ngpus_per_node = len(args.device_list.split(',')) + elif args.device_num != -1: + ngpus_per_node = args.device_num + elif args.device == 'npu': + ngpus_per_node = int(os.environ["RANK_SIZE"]) + else: + ngpus_per_node = torch.cuda.device_count() + if args.multiprocessing_distributed: + # Since we have ngpus_per_node processes per node, the total world_size + # needs to be adjusted accordingly + args.world_size = ngpus_per_node * args.world_size + # Use torch.multiprocessing.spawn to launch distributed processes: the + # main_worker process function + # The child process uses the environment variables of the parent process, + # we have to set LOCAL_DEVICE_ID for every proc + if args.device == 'npu': + main_worker(args.local_rank, ngpus_per_node, args) + else: + mp.spawn(main_worker, nprocs=ngpus_per_node, args=(ngpus_per_node, args)) + else: + # Simply call main_worker function + main_worker(args.gpu, ngpus_per_node, args) + ############################## + +def main_worker(gpu, ngpus_per_node, args): + + if args.frozen_weights is not None: + assert args.masks, "Frozen training is meant for segmentation only" + #####################begin############################## + if args.device_list != '': + print(args.device_list) + args.gpu = int(args.device_list.split(',')[gpu]) + else: + args.gpu = gpu + + print("[npu id:", args.gpu, "]", "++++++++++++++++ before set LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID']) + os.environ['LOCAL_DEVICE_ID'] = str(args.gpu) + print("[npu id:", args.gpu, "]", "++++++++++++++++ LOCAL_DEVICE_ID:", os.environ['LOCAL_DEVICE_ID']) + + if args.gpu is not None: + print("[npu id:", args.gpu, "]", "Use GPU: {} for training".format(args.gpu)) + + if args.distributed: + if args.dist_url == "env://" and args.rank == -1: + args.rank = int(os.environ["RANK"]) + if args.multiprocessing_distributed: + # For multiprocessing distributed training, rank needs to be the + # global rank among all the processes + args.rank = args.rank * ngpus_per_node + gpu + if args.device == 'npu': + dist.init_process_group(backend=args.dist_backend, # init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + else: + dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + + loc = 'npu:{}'.format(args.gpu) + torch.npu.set_device(loc) + + args.workers = int((args.workers + ngpus_per_node - 1) / ngpus_per_node) + + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + print("[npu id:", args.gpu, "]", args) + print("[npu id:", args.gpu, "]", "===============main_worker()=================") + ##################end################ + + # device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + np.random.seed(seed) + random.seed(seed) + + model, criterion, postprocessors = build_model(args) + # model.to(device) + # model = convert_syncbn_model(model) + model = model.to(loc) + model_without_ddp = model + param_dicts = [ + {"params": [p for n, p in model_without_ddp.named_parameters() if "backbone" not in n and p.requires_grad]}, + { + "params": [p for n, p in model_without_ddp.named_parameters() if "backbone" in n and p.requires_grad], + "lr": args.lr_backbone, + }, + ] + + # utils.init_distributed_mode(args) + print("git:\n {}\n".format(utils.get_sha())) + + optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, + weight_decay=args.weight_decay) + + # optimizer = apex.optimizers.NpuFusedAdamW(param_dicts, lr=args.lr, + # weight_decay=args.weight_decay) + + lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.lr_drop) + + # model.to(device) + opt_level = 'O0' + model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level) + for ls in amp._amp_state.loss_scalers: + ls._scale_seq_len = 50 + ls._loss_scale = 2. ** 24 + + + if args.distributed: + # model = DistributedDataParallel(model) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu], broadcast_buffers=False) + model_without_ddp = model.module + # n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + n_parameters = sum(p.numel() for p in model.parameters()) + print('number of params:', n_parameters) + + dataset_train = build_dataset(image_set='train', args=args) + dataset_val = build_dataset(image_set='val', args=args) + + if args.distributed: + sampler_train = DistributedSampler(dataset_train) + sampler_val = DistributedSampler(dataset_val, shuffle=False) + else: + sampler_train = torch.utils.data.RandomSampler(dataset_train) + sampler_val = torch.utils.data.SequentialSampler(dataset_val) + + batch_sampler_train = torch.utils.data.BatchSampler( + sampler_train, args.batch_size, drop_last=True) + + data_loader_train = DataLoader(dataset_train, batch_sampler=batch_sampler_train, + collate_fn=utils.collate_fn, num_workers=args.num_workers) + data_loader_val = DataLoader(dataset_val, args.batch_size, sampler=sampler_val, + drop_last=False, collate_fn=utils.collate_fn, num_workers=args.num_workers) + + if args.dataset_file == "coco_panoptic": + # We also evaluate AP during panoptic training, on original coco DS + coco_val = datasets.coco.build("val", args) + base_ds = get_coco_api_from_dataset(coco_val) + else: + base_ds = get_coco_api_from_dataset(dataset_val) + + if args.frozen_weights is not None: + checkpoint = torch.load(args.frozen_weights, map_location='cpu') + model_without_ddp.detr.load_state_dict(checkpoint['model']) + + output_dir = Path(args.output_dir) + if args.resume: + if args.resume.startswith('https'): + checkpoint = torch.hub.load_state_dict_from_url( + args.resume, map_location='cpu', check_hash=True) + else: + checkpoint = torch.load(args.resume, map_location='cpu') + model_without_ddp.load_state_dict(checkpoint['model']) + if not args.eval and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: + optimizer.load_state_dict(checkpoint['optimizer']) + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + args.start_epoch = checkpoint['epoch'] + 1 + + if args.eval: + test_stats, coco_evaluator = evaluate(model, criterion, postprocessors, + data_loader_val, base_ds, loc, args.output_dir) + if args.output_dir: + utils.save_on_master(coco_evaluator.coco_eval["bbox"].eval, output_dir / "eval.pth") + return + + print("Start training") + start_time = time.time() + best = 0 + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + sampler_train.set_epoch(epoch) + + train_stats = train_one_epoch( + model, criterion, data_loader_train, optimizer, loc, epoch, + args.clip_max_norm) + lr_scheduler.step() + + if args.output_dir: + checkpoint_paths = [output_dir / 'checkpoint.pth'] + # extra checkpoint before LR drop and every 100 epochs + if (epoch + 1) % args.lr_drop == 0 or (epoch + 1) % 100 == 0: + checkpoint_paths.append(output_dir / f'checkpoint{epoch:04}.pth') + for checkpoint_path in checkpoint_paths: + utils.save_on_master({ + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'epoch': epoch, + 'args': args, + }, checkpoint_path) + + test_stats, coco_evaluator = evaluate( + model, criterion, postprocessors, data_loader_val, base_ds, loc, args.output_dir + ) + + map = coco_evaluator.coco_eval['bbox'].stats[0] + if map >= best: + print(map) + best = map + utils.save_on_master({ + 'model': model_without_ddp.state_dict(), + 'optimizer': optimizer.state_dict(), + 'lr_scheduler': lr_scheduler.state_dict(), + 'epoch': epoch, + 'args': args, + }, 'output/checkpoint_{}.pth'.format(map)) + + log_stats = {**{f'train_{k}': v for k, v in train_stats.items()}, + **{f'test_{k}': v for k, v in test_stats.items()}, + 'epoch': epoch, + 'n_parameters': n_parameters} + + if args.output_dir and utils.is_main_process(): + with (output_dir / "log.txt").open("a") as f: + f.write(json.dumps(log_stats) + "\n") + + # for evaluation logs + if coco_evaluator is not None: + (output_dir / 'eval').mkdir(exist_ok=True) + if "bbox" in coco_evaluator.coco_eval: + filenames = ['latest.pth'] + if epoch % 50 == 0: + filenames.append(f'{epoch:03}.pth') + for name in filenames: + torch.save(coco_evaluator.coco_eval["bbox"].eval, + output_dir / "eval" / name) + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser('DETR training and evaluation script', parents=[get_args_parser()]) + args = parser.parse_args() + if args.output_dir: + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + main(args) -- Gitee From e2550bf8c236ced1c5e305c385151367683d8c29 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:03:28 +0000 Subject: [PATCH 39/59] my first commit --- .../DETR/__pycache__/OXInterface.cpython-37.pyc | Bin 0 -> 32921 bytes .../DETR/__pycache__/coco_eval.cpython-37.pyc | Bin 0 -> 3442 bytes .../DETR/__pycache__/engine.cpython-36.pyc | Bin 0 -> 5897 bytes .../DETR/__pycache__/engine.cpython-37.pyc | Bin 0 -> 5877 bytes .../DETR/__pycache__/hubconf.cpython-36.pyc | Bin 0 -> 5554 bytes .../DETR/__pycache__/hubconf.cpython-37.pyc | Bin 0 -> 5558 bytes .../__pycache__/npu_fused_adamw.cpython-37.pyc | Bin 0 -> 7377 bytes .../__pycache__/train_npu_hw.cpython-37.pyc | Bin 0 -> 10725 bytes 8 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 PyTorch/contrib/cv/detection/DETR/__pycache__/OXInterface.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/__pycache__/coco_eval.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/__pycache__/engine.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/__pycache__/engine.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/__pycache__/hubconf.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/__pycache__/hubconf.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/__pycache__/npu_fused_adamw.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/__pycache__/train_npu_hw.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/OXInterface.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/__pycache__/OXInterface.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc4bc085fa14ef6c0f4434a0e703d94d241af8f7 GIT binary patch literal 32921 zcmdUY3z%Hhec!y_GyBx;YV}|wWTchFA_F@VH zckbMoofVj1Cs}FE+;i@^=bZaL|MP#JGgrpr5f}dM{rO<#y_a0BpD|PY^WkC(KE(mI z%cZ$eF3qiZa_*E{zC9_Ae0x(~`SzuJ^6gLg3()m6Y;mZpG-z)XLHGQ|FJa zO09CcbkEGEwAifM?YhRL#kIs?mzL0@hrFo^kZ;jik#9ZZNv%e%O>0N49l15gb!eT) zbt2b?+#;YZBmF}K#EEk5i^ecHRUQK0?0;A-+!X=)S#@~Tz>fm+b7<@=dUFl*xy|5?d}%xG;W9i>(Ddt#LaBgENXwLTw#yqiV>~^W864L8 ze5UW(Y_Vkewq*xPrgv-pfEgMn?9@x+M!wHuMuv>SXgXsU8I+d_>Z;FcMtAA?V!@Ec z&BRwSxp93*eyCvOx0XsqcF%ap>fC20Ml<{Lbf#Kth9#fJ@XUx@?lm%F!`AgEigV~= zsgBps4GcRiMQPSg7sg9tqFN*s)1s-k7E2|xc&bH9 zq*}F>RGZeCYS-E@^X+Ejx*NA;N|{{;#`Lr7=d6c`mkF2LSwE8ilOU52lQ0u5)w5A1 zF(z>)2_`K}T9FL&b(o=aI-eQU(`hr3=If2;cpXcp_m5|C)}7!`u8=8RywUWJXY-}Y zOrN9yW;R@E`fFdTg50H-n1LcDEx*_F?I{#;rfL9n16pBxPfj=GS=5$qR1XgpM#pmc z1TG`i<%W%yr9yk8=M9&>;4(gk0Za11r!N{VjOrInj9)a7%@iiGh5X)&wr$_F^CC?z z85do5wJxJr)Vx$SJ7imj&O{zWXN_F7VsVyNrGCkLP#pniHj2$!D2d#+$v@4Gi3v-k2Wv@)xiF zYI@t&U0Zj3_4@7UZ9A^oWwry&2X^e*vGv*=sqH&eS@gOa2d>+uD%6&Xy4u86KaYeh z7k42Ec?0fG@?XT~b4w3(>&J_pwy?_?B(?RE^6FSW;CWun2QKH={953!D-}R#5Tzk4 zjMAVET8Ugliy{}+f&0kCv^a86JtjHeIC614fxE58Ji>8(sTRF;q)qOV=lW9Z%%jF4 z)acT>QPQD@kncf$vDS-x6|dtYAK6kilgmzmTU?1%FFbR{Q1U=wJUIrUR!U})jylO( zGDbE7qL$3&lcixj3F0%F&1Z7!OXVi1^rqzMOEzBoxz$o& zWNuBf7777%*XoFrZqc%AWN)FcHwV&4PQGV+2sBckDD|gD^MWlquqc1||avl%_i}8BlXI8FftdWKdJHJ{c?yXE^t}GceJ>-5wyW{8ttXiWz;NE&~UjUft-7C zQ6qGv7G^DAh`?uqnc(I}z{~~!*yzuxy+Na2Fn}PVKV#&Kqhr@0y|! zyB={9zs5gEe;GxdlcX;y4KRyX~R+ z8?mrj#a;z79+nTb8nGsoXM z{qFILPQ8D8?!mXt`AP$Q{@ObU@iw-gCDS`r7^~t&D6iit?_`DwIZdt_Cn{95Ui&B> zE)vWxf5;tjx6WIF)ruc!f;OU^+DkQ=pt*-$nY-`!=~v&bJaLa*u{uq^fu|Rkra0%o zUJ{R?qCzYG!=?MnVK=*nW(tefk7 zCNL|MCLL7em}hqyim%Ki1zr1X&M}uw-+rJk>>vbzOpzo*&FrtCUDFG?Z3aR4Nu144 zm0BADEkzBCucD$cz=T6li4fU1r_YAREN&`#;O}+E+>NlSt6MkUbtJzErjNEH)jQkJ z!Cc!^Z-Kd%CCM?r$l(fOXu2d>1sTFEuzE-S7Jo;yn*sr67tr%%2Ay~@kqttV;? z&>86S8Jp1{qy^~h4g|4YP2>ADV0Ktke7@V;d^ZbCy6dTdU8jlc4<@M<2GeOH#@4T9 zay^rsOk^Iu#49T1#tlqvVsaxBzP#~uCLFbbe41B-Ooo`$Mk`lbjK-Bn_Tf|H^P#xU zo$!Vt32!2ph$Moo;nrY#MCz~=`_G4~E%+3F8_CBI$0>@t4!I$WheQzPbwrOkMDds& zmt3pX2C+J!w@9vC>p-s6y4R^KLat3jaTkR1sNMmiSI;r8C5$g}lq}JfqHY&TI<;jG z*OzPOA>XYfv4B>X0jNB%<79pZGbU7_K~A@AR-e3-o~)P9#b!h){BxHmA-q~JG}wQu zp4~fKnpeChn<=V7Wro7jgSkx6DpSJX>TC4T(e#Fm8?1_BmgFzxs?0WoC2z9`qxEPe zH#VG6b&|;;s1KI?FPT(wsOGSeKbqjLIvp% zKO$5#Q;_^(#;g+UEUG$ag^&r7qMzYY6`WhtikYtrQ9`fl%gLDs(1lR35!7nVX@%a+ zIeJRiJz0u@VgT%ifBfc0-YtL&a7bG#ShyQg$=HC>Sv=GDRw7&zf=zn~S-n^a0F5&; zr!@uO%s1~ieaBmn{m;DjoywihRHh#}GyA+<2SDJ!Xyb%|cmXivP~*?>Q9}z%#w;+% zGoQ$~j+|UzTmgmd3k+-&gZgX7z+mbidF)pYeWa1moM1yksZ2J+=4(rY)j6~!gB;d^FtEl-)(U53_s}0?$AuaRM{89Q#_I-Hno)ww8ZP{K^M>oV{gJSE75b>5g6NeB2P&^VF?aAe`&MEVtD+c(m{Dm2qcLw2(r8EIpmr0 zPOKGm!n4hF^YSTQ*+;v;K9A91Sg)t)F}H=&3hf{MEwz%u_#5qi70tA%!-E8{-&!9oQ#Okcuh1S72_0f2+D# zn*v5+6iKUHQDQYzM0db539VMB{Jh$IJq;c$V=o>xh~o~8mO7^yg8Kk=rP2X|;*LSl zVqq<}z`J8i3?`Jq46Zig0Fy~3w=t=)(3+Mzj2l0}r^p2sS}t0yH{p$iVj-pJCW4`0 z`{MQvgEa^7vH#f57JQ0-4KP%7T^->ka3Xm$FSJOX=F|LeJNdOQ=w3ab2cfHa#rfn< z!EvjHM;a(yKI_+o^6pq|on;efTu zx=_}w^`I@dW395qC{w{6aL!s~OHsB=TaJ2g)jG<~(~>CbM9HGXE-QrtT3-asdd0DT zwo*GE_u$o34*&2+yPg{G~pX$*UX`cpU)@c`_#$qei zuU&*(ua#S`Z9r~`)$by0BSv|#b_vRs+GAO!eFkk_s(luv%TZbzgI8asU3Sc;eNNki zd*^At(Qr3lwXsawjFDchU4c<0@eD`xYuXk(v%<=4)viQtCEnqDeU<(x?JBgoP1}yL z3$&}XYcQj$wa6)QK z%3Jb*eY$6P9RR%vj0ABN!c}MlHokO3u3+Iy$Fg9z>A0kciRCDdfM#`0c}p#27`009 zLYBQF?VDZucirHcSX=5qjo2a5wWVM4O!=n#rOq-ONmGGx0Cg9Y{Ze;=Onjybb>sLX zXc|Mln>NUFkD#(^Dp(Gdd?Sm~y(KmBP&rig^teWrOoj1IzL}-v@T|OlD%LKFq{BH?6t=47P;q<#UZW8G5pHKT|G8r-KsWe$-7 zQLA*hCqHG)g{^w_Z_X9qWh`FN?|5QzgEg=n)gfg{i10|Ns1P_NS0t}06idn5rjkW{ zu#ne^{e88ntNZ!=%Q)A1pWe%1SI2- z`wpp5hFbs>$u~~(ye*W>&AbH%mSYT*48-fGlr|XhEXt#Z=tBFv_67s=4 zEnWTyrqbt0xF=UsA-dpG0CB71<&^JmfF0KX)sGSzsJv|fHDL4rkZuPc;q-(XO-X(J znSdo;yBIUI*aa#ZB$cH?#7Uv6wCn@Wz2m=!Ms4s_j(%(I`B}IOPrdoF5`#~@c@V;& zW27}5Wxf>ds3tG8sYX0?!=698H!3<+FMUHz%Wq*dA3@GO$?>4e5rC7#6B-E6)Bc{t!A~V~z1BXNSmQ zaJH3ii5h^=vZ|W(I16MRufmlXuUkfqNYz4t-vP8^M5T5-=ZZ$$9(T+W5o!T?Afl;S zqz%tNO^`kXZG@fR@w&CR76o96KplEmOKxbqvi`^_N65a#=PDX7U2pF3@8ba*@~7Uu z-$n@~qR-uXq;mW*Os8Yyp8!_aJ=$|^YXr^bx-V!vhLJjEh~_6XFH{l4!Bkz?0;BJ- zv|(YlTe806WnSYnP(*e`I03Dm$=>REo?jUYIcl*z;*BC6b!)h8GCmezT+g9!weU|7 zMoUAkBZyR8>}nrM3AYj$HcnUJG{e$n1h8F^GU4wQ<@-q!%4b7resy4Ko~_qmDv6s- z;4VX^-zK`<%nT9t3q zfvGQukRZgy$}%TN%z#*BiUw6P(~H$+`sfv|Dx7=;(`)EO<3U_CV$-7f*S0Kke^>)l z0FfUtD4Uar(oyTV=@X0ySF)8LRUOF46Zj#4DPE1l0@=&w^)}5Wp$KmDGA@1o!z4;N3c0 zDC(9#3gMTrd1*!x{t1DfKvOQi0zSnn2w0zPG64&^8g&c5)f%G$FwP{sXOj#Vr|-^7*a1)LD#FRG0~|A10LW>}BtZw3VKi}H{e zv3Lw=RRaeT9cawZZ^;(Mu`@$du%A02OpPN3(G*k%L0mPe)9{R~`v1KG5ti<^pF1ZI z1xxQHm|kM?G7}*n^B@>hXj4>1xD%|{06U4J%p=mrf|~~tluaxgaD!p61~*6jzJS@j zM;}rPY z-+cbe`%j#C?=6(fz4rXv;Rh<;zIQjJa!xo&ZslHhp?^>liO)WIv@-oV_NL7}GBfwu zLvyeH`1IosE5=$mc5?2~6J)ISJahW(@60`M?9`k0oqp-r%F*ewPaU2+_9Snec;obg z&)0gW9C;V&ks8n3-S=QTN+MUIs@(b7>DS)=*}*$%{m)m_=A$e|S9#!$Gtb?=bNkk9 zU)QDSFPw>W}o8B?Y0bO)DGxoow7%8O~Y@_|4R^sjH|wWA;4SD#V;j zBIcw)BKhR>L^Ey!lM8#-fZ3AE=J!>NUoak}yy?+&%9CCX5mE%;A7g17$CAzdmt2|f0;J47v8#5cPUYiNj!RD^$`MxTsX zjzv^i(pIxec(`o;tFx>5_UQ+T5}F_m4Kq|ldefWJ^NP9M%cg`llg%Lxb2|%OWAY;= zH2}PhY$_oU)QV8TlIQ>?38xaVzr%_xc11gUeQNkG%U=JT9~!vSdXnLNPrA272(QLtE@qKGTuYOySSWs zaN$ZQn^lixvr1GKifzfRl8ar8@(}4rM4@nI92=fV!PFX|y@Y>tVkn1a8`FLRNyC*PS=S$bRvR9qmWaFx9 zZuq}i_To-nQe~(+B{~M40ZmVGYhfXuJCGbkG(4_!Y+_Vul-%zZ=GvT@DxUcYu+sP; zllPfqn236^hgUSGsI@5ONJa-VE<;OY`3Z0RQzl~bh*yUpJUSH?g@Yq*Sb%2KvPbe> zi3@Qh+Hl3et>8q8!F-H=9;klcTkUW*S#Bl=?{Yj3XOXDVz*+gD8f~l6+mMl62(A$@ zrYU!pHoAUn_wb5^ZY3NHJ}eYCXiIbfV+%EedG|2{q;UB_NbiLJPV2H~{0%f^qr9>w zDFd@2+sc;+iS5+;vvaqfuxqL;Y#ejMRGZ{F4^DTuOh&St1vly5u=xVBYhE=gy1{14v{ch zcjU!=FsOeC+eP7)H4dSw8KEqtSd{56jl<)hnh<-V+K(dJkVxSz6C?eT(n6|DNp#l8 zDcDr4f{2eEZA1pKJ*Ks;5ec3gFU zdU!xRg1GJk^_WCR2W5K3Y}XGv>W8KNi>SXDZFMme1U+C4mm@Phvrzv_i#2bl z*UBxSF$02t>MJs{l;ZOKKLxE)A~Hj2ZCc`_RrHUFWqcu2b>6j!)_gh#l2t3LBEDUS zUd}%9Bh`gHjDHA(a54<~GuBV~=cI|U`L1u7aTrJiuL)vGWK|d)$SZlYlXpCYqUmF> zk?9*LWbsC+G0My=JBhHw+(U-9`rWYWaHXxxVzKjBhcg9}$!XfUX7 z#N7kdO!ka`0inq%ZN>Ek-g-N2Lx*ribSjd~O?eIrW1S_Oih#Yq6To9(*_ViYa2?n+ zq*-j)N66n|Spaek9TJPdcORBJ?m!mCVnEDjiB@vxn3SPG8vGy)0m{!Z#>mqwT;;GL1khX~xNa#U zdKE+@iV@>vn5p>00K&KtABO&}oJwF}w<9_%G17TRk*Bp&E#;PSvsZ`#_dj1b@s3)Hm7|9d^Kkn3BbAqr zLJ*-`a^$4Q2x{%s8P{MYJA3d!3a^mBPd=_%fUHk6syzN4358hwP9M7mt*A!lwFp5S zBDboj$c&>q+&p{mu`}-RXYQFZp zS}j4mfQw0rmtStv_UptOL?o;Y*?4PcwKVi#!DpWJJ&;1IdC;(L+gB}>dw5{ zsCYHiyyuVeDjWa4Uu7N)Z*HP|-L~2?(|%kLTFFDSaVja=EnjA9;}k<1p>cFF+JBMF zYHZ;J3@r}5?<=KzOCJ~O&h=^Jq)6>6R#Vytw20w^wrf;7C zlvgYRIgohFfNOGfRf>Yljp#R7jJ1%=+A~T)^rj+~{KinhnugFLyOvGg(0DFq?8aDy zF&h7dSBIF?^F^b9{rwTE2oEHKL(IW=-Z;XtEVF)W7~QKYy>pbMcE9SKTa|(5KeBF% zV_iyYrn7)*h4Js%0-G_tcopM6@G2pkPX4x5qTxwDPx z32d-7EdP=Wu^*LPOK5vdz|y)D`YEIE6EGj43|7~OyWJfVBT~ZM>0Zt_9EZ)-ZVQns z@{%1I&goM}FxX?RHz+Z(%4Uhzi_8m?4s_I6x6vg`4p%hvFeXZ_0$^4(nc1pBhl~G)GfZc`A2M&A!n=C`u z?i^W!*2lmmM@n6#ZZNo`1K%I2{DZnR|Rx*6hmo6u~EA42v2 z9Y-&ue>1Nd)F07pe##p*={Nq2Sur>Y;U|kz4mrQUSZC8G!RtDTVY?Ln8wsRwnu$&5 zjsLKV>zjR)QD|8Z#CQJLwPMHV?CO2 zbx9b!gXUuw7HEA0#;F?RC*AJ{N5El8c3k27Msc)imtt@hE8^f(97cD&U8Q!TgH8=4 zbJWH!`V&oF{3D~F2jVJxic};U_P_XebP*T@_Pl`bGY-QC)}eNoAeyM`!w!=mS%K!6 z2~j_yS3#p+z&HLNIiyBODQ#}<{{bKkk|)Vs$} zk1a|-8qN8~A3OEd-L^u-G2*Ros2CJ1zS7bA?XgsjJ+IEOvG~R#ub+A2HR?X!KREZ~ zGtSmey>*Y$gU-$(+~NiG6h;co=+v9{5GWrNB4mo(ly} zl02XnhY}{Q!^x98kWkO{*Xr-W;fA%U*pfzM(=- z9E->^ev8Sq`UEB{cVMlf<$*#8$Jt$7FtE)hUor~03!#zRif9QQ*o%kODY}fqU3g&P zwS~giI_V_6hqNGL9AK~Z(9$)9<<{Zs;Ba#OP!}5Ax@2*1kyo7E*Z}twS z*`Gau=xQ&wm7y?$%~M4VJCEZU0gF;WjcgR5nMtXXPBKKly4Sc0AcW)FujhaSgajN@ zoG&%(wpL4@5^8b{wy$89R^O&1=R=CnXTiws<)M>xGe@RX{j-R~Ihr7ikg;293NPbC zgK9(kR>M*Ay@tw#z~4EpCb&>V0D&2Xyhq1|1zEkircSx6c43Y6Be&mxVdw5(a;3Gw z>`EOF<9u18kat%&Ke^tjV=WQ26wb8=jpwf24gxA$(2P5HB?dyjkSQ~6^`XeIQjCtW z(zlQxdQuqJe`mp8G5H@%zRlzzCJ!^IXFSHwQMQMiM%*45r?#5Edl5u!HIR#en|PNrb5%)O9lO8#D~DQ;0(e* zaA+(HYSa*@HSAvrZ6Yv@gN+&H2i+tB16>3<3@!6v=r6G8Lz9S(L=lD;8;RZRK^mWm zAMp&iwSX2p9GXhNoEO>`LIB<*`fe#D5TD;#Zk=h-!dm38iyJqqI*I5h(W$nH-=1dR z`46Ys%ZYM3bd?S~**S3s%yu0#+u>dbL5qu~y5MAw!FB&ZxvSiYP{R-Q|8)OPp(S-| z@zSDln}iy6YYCX}+O!sE8*S*V3vbh1?t+J)b+HS9e%)H@W|v*oCM9inViE3OJ88Ld zrl;I3us{$X+U_ZLlzX%e-oZD#3*{En=#d(~bAu}n^W{_v#=Cf8qTC|7RF4gT89s?X zo8CN3i@3&4W4vAO#246-1R!pn>c*TbM&AiFHZ>mTOx~#_<;Cb}345~c_l|UOl+^nW zsQCH~Os6qpfbrdd>A+0Ow2!srN<1f}-b1bMv%f%ZmomyI6I^L-j1Ua z@Yv~hzjgZUV-U4Y|M0!a$!BZNt8h^Y8cE0B#}xn(zd=dr^xe~Q$6mTZ{k8!FFNBLi z_QKe9*AY2O0Hp2O(RZ?GrH?C7QuO*TL9(uQ$gsN=eH7+mjS=0exRc6Vj{=H#r^+{< zJagz>KpiRU?5E#3dG^saM98$H;z1f(%*XQp4moKT{B{704de}Fn_63@JMsPgo4 zbC0~PrVax0+=&M(uYB*!iHDpZ885;tbkhx6cMj|rxaQ{MuX<{J<+Je*WdbJ8Zgb%`-_v#3&88fbUoU#Jj z^SLH3s7eS9v~<`8s*hM!4SPh?8u{tKr??G?h9H@V~AAaWho&Z4aP_q0~w7b@8BD_de=hxnZ07>b!_1C89zb4sbIDUGgV$wX4pi% z^udD(D8v*MFyu^v4%TjvQl2G52v`?wvG~XB>4=5J?&it$=Y&v`#s+LCDHyx3+Z)Z( z6)bl98142vH@4n#&q{DEi*cu!W0l68->=S`(`Mg%=4?B^*z9+yutp`U8_%OD^^*iEXL^rTgvgj!<#0Yp{h5cjy1ljNY02n2=%6mTnkTTG$VM&O+>;nf#w|@9k zL&}AFLnLFzGO>XTanyR zt4t(uc^a*@3g$V6qZEh#g+9$rN3rVD-bN+oGV3f0st0VxFZAL#6#~Zz7xe{D;FSgxFo4W10|J^jk$8fLbC$c+Dr+{$J&luG?O09r z|71Tf!&!(ebL&U;Z=25TXFk>|PPT5ZYUau$berj(0-d^wh!&B(!xEBF(={FRXiXnN zpCehEX$NQ_MiA!%jPV4uEY3VN@LNj={KU@;4O$0o*yp|MqHaoFs4P+EI`8GvjjHx6 zVIrPA7^-1s&z(VS0aG4nYmxB{~ zi>xF8q%0>96rv79{)QK%Vg}V+Z6O~FEMmHkA*P9D{D}JT9B*tT1&4sUSA+=uz*Hwp z`94IUa}I`MBK~qLp=EQ_C$IjF*t*{cfJ+v$E3U+ zwU|ena3E7e@!)Cxa8C$raK|esxa~4_jNabqS`|nLEs=MsQIKHDAWzo$oojK|R$G*v z*?BCZr6pFissZpkt|^=(#NS4fB`eDfL_%Qu$GF+RdCi2>wVe_hs+8Cz2l6YMRecrv zti=$TvmF%^$S7e7HUCpT{tXM ztS7c|q6TeEJ=j!)b3$;GR(%bGeC=Oqi6!`5nlR9w2kATZrU=OpY9Ul80(u9tSY|v| zQWWNC3>oLF5HzV?wH_=#PThG@sU;TkZ+25{6sv7vvAb)4vxMJ&j=&Y+7O{5N)Fs#| z@p$gZRaKsFu2ZKVZ9rPF$eS#9^}7&X#49MP9(s^J&J7(&f?w;f#H#o;B7t|Xy>Sb# zFU{Y;fRztnZiuH%GiVM1A5zJWL`JafwFiNButT}AW#zXfv=U-!#g4WH0eE(;nYgrX z=Na5LtL%W+!?M*c&tu#4#Hv!P1iKyuP0Vd)T^lXLnf(u9<|k20Mk2LF5XsOAS62|f zw$gUUW1LdgwcdFr-M;^%DyL!tn_)u3#x=eb-$KZ2%2W^b`8)r>RKwBrT-3YqJ3-tznHk-k9My45dr~_9(K9;; zhX{UasAiwPi2^AC1G1y|wWwOa7~}ix^XJ*1g~9X6lSwiq>lAzSgI280Kk2Ei5td?L z^U9=D4HrYX&^jESwbfs3H-pMA+=g{J?Zx&4e9Q(Aa`($_!ue9Q>)O@L_5eit8<_N5 zxMA{G&CYWb{CYeqOhS~b`B5;3*9$*TmW8vckX(7;hLy=-oDsDyshnq{nFH8E1m%K$ zL7XmEFjge34ZFoLeULv{wi|cP=?2&hKNV0*2j;>R1{a=z6CMuq<0tfZ{ItQyT_qC< zO%q@LO`s^Uk-&^PP{|D2*rS32NRCV|=2ry_h;Zi3>?z8=BuJq}Y}+hl@{l$u#^Hys zs%S1Eq{<5pSk&3*u#8VoM&Qj31hYgr}O}=%iHTX?W#eTw7Fgc4Ux(UW&J~3z+$grw7xF!RbQF% z!`9E>WN>lNS^7V47(y989*JLx!q!*@I>R8dP4nz^qtw6ImG2Ud2Y<;7cd3S}a%}VS zS^>@pP{psyu?1h53ZZ=*n9$HZe!jG>{imh0ACZx$4?HoX5xDO zzT!ryh+jJ@Y^vF&=eK^{8_=VAFGfN#mS7vUFwBg1HcL$@3Vwb|D~&RZDb%R@`Wm zQfS(GgzJU!DNI`z+fqsM4gAg2ZsRRHYn(*#D;@SzwP8a{@)yT)*^(khqBW8TRgc?z zjRX7o_%fBePdBd5p515E~<%hhYtZDoX6K>5m{uz@$VDb)=` zzgBnawq&w%ZK4DJ)^%>`x(s?-(&fJ38?OFDD3R#A*kC(YAHQiehcD#}=UNjY&-vl+ zuD~|xp6CGySHf!y)^!(8BnJ7!ad4K(_UouD%X#k2J5RrGa)Dp&oSS~QdT0L6&NzNZ zgeslH+x;1ut^UZxGq_HLXqQam^b6CSc7R79lRc`?dA+5m zGtniyD6)M5rR2wZe{nBiDSkECGN*&9|jg6^R zxyd9wnoE!E*{FVdOwc*Oo@DE8zx7-T8*ODGcBgi>u(bM1@K&W_zqDpO(ZR}MN23YN zY@<0@9~>8(UM@Qdk|%5E^QVf&A6KSV6;%{A)xH$3E#5Y_?d}{x$O+@}b)>XTH literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/coco_eval.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/__pycache__/coco_eval.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fdcebb56a7f46785cded01bb7d2a3ff13e7cff89 GIT binary patch literal 3442 zcmb7G&2J+~74NEU+wHcU#EIj4OeX7{-JO+@NDy!u!tgbFKw_5_6Ben(s@3VLIPLU@ zr>dMuO#74(6OaOfIIt1|*+>2X{+3mpkT~TZ*u$K7Rc&{YO@yGWuC96?^?vWwd$0WC zM#C}SakfJLk4uK}Z?%{|CQN<^FW-lO7_LDO!4Yzi{$dyFuNhlz2`Qc#mE($A(Q_-Z zWk6_t>L=+ygh4?JDoPlhb$HcW`NIE&4I?uq z#?(MYhVrjKeeK638dtK)#3UwAEMh67mw1Z7gUZ+zrPt^X5t~$B*~EDU9-O0bRg^O; ztDfVrLyU2a)GpAZGBu9N?6<g1{l;V=E0g*Zlg6bvuDwC0e+O%enMoEh zdul#2-k{Ha|7$$1j~iKivUFig(YXHF0Go?s>6M)|l+AHNG_!htxxaFbzyl+z18pHg z*+SL;6pI<^w|<8OnyhAvpuhAQgMO1NgZ|Q_JvH6 z1xq2yn%VMS%s(hQz|EG3xnYpiOIVA|{sYnmR7Q@9tdaFgJYKm#WMjhsEGvMeC7NO@ zYmqh3+T4I0Z_FX4;787Tpm;adTk~}uOU-)-XYgiXY#K4lJ|AYEB!~sIjj|0Uy_Z&JIwx>IsNxk zB>V4AKA0NW8u?)FrvI3IsU^U}LgMir$YsudV-wC8Tt%7eSwJqD3B^t>zJN#*d6bDOQK^SEDTaE1}qII=PBd=Fy3}| zkJ6a#o(^|ULq9!*!td=q`)L0cyMzk18$h9Zsx3hMzpd`|XhKg1?5Qeho+BDPP;6)f zbZ0M013%(VcM3T$ZE@O*rFC$Sp1Ss6)+XG>e5N-CU`{#|gQ4(9$gWgFyybTZ2rJ6u zBN%$h^UX2RwFzAhYD;$#l~aGhFD&pK7uu@p6xv&`2$gmV>Ku)6h9`!q^9=o2>#+t1 z02VCLuE`V%R)ayJ6CjoeJqZJaNPMjzf2k0|81;Lf!U z>{)``h~a#+Er~=Iv9_|RaLjEKsyA(H{Wlx;JFnEW^V>7$A#osStPEd3E8Z|#Fy075 z9*Y}MV5_hJV6MHSVed#CR3NT_vNhm-58gZqXmlUKB3yS6K3AG|k0(Qag=_C`ILFmR zZq*wm9CD13`|;&~ituSv-@isBttjOjqO4M)05PvD4Wb1conduVTg2EfW;q4roU*W| z3p6*L9~;{Mjj!9?1llurXF0)C5ku@u62`0vb6Ms=$|#p5C=2+MBmaO#oWUP}{~wp7 zddB|*>Emlih4=Fc(cY`RqLZXOuaGZYC_g0#cf?@=!#;E|;=cvdX3EPU3ZLYDJd z{(4YUiv0Wt z2Y%%FC%s#BT#705;`?A7z^=2iaui|jfU*aQS7Ve61DaQ= zikmDC2Vf5F7e0(-S+#AIg76nEI08-K2glNkXd=--nxk+aAM&##IAUo63FaM8@siTS zo82Cu>G(k~jE9lhvI<+qK2x_>S-a0=wFsQvTtJ$ORJF72x-7*hT#g{Z)b$9L4yyw| zx2B@uMX9fRuN**NJi_yyxUBdM*cxID5<1xBBdR3bYM2f^aW()<*Qcldo4U)lE*edf z74FAy6db~)6BZ_U#&Q?p3)i%2RBp4FdU+s;yMBk{<=x2Lyvvz)Dz~ZoMs7;C53mQd zKa2!->$k`zkl9bc(Y%_?+;bHfeWLnGkoLUyF9MyP!eC$<*|?6{$ii*pm=>xdjLPs^ zHn)(2mQlTKV+WT_2Uc~U*HH^3Y_x>e(E{2=OYo~B3+6Uj2l*C!ZM0z7An&L-Xex@0 zTd-aLKD?Oy5^Q%Hvf_Cp4Lpy17uM_tFvzN^jh#5;fwa`fz6n%!d6w*ar$`N54{p0) z+&V5@B5o~Bl2bNJL>N<9Nm8%Je4=gNDCMH(OM4G;YZ%d|s%LO@Q(G!N;Gg>0)_(wC C<6N@< literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/engine.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/__pycache__/engine.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ac3e653e5de063ae2cb252f772cbf1310ca05563 GIT binary patch literal 5897 zcmchbTaVku6~{TGE);cLt@i3&N3q=`o22Vp+MsY3M$*`6gC9hcmvd}+Z1SuBtU^a^*i)K^mz*OrB8XyW1jjyLtWOflO{l02^W$i3W z%jE1FWSMThXWHgORtPM+2x+!k>Xq%XlIFUVUe&H5U-EK6Em)YOS={TvBGQJp=A^VQfB?rzsZ-f(*Z+{!P9t?(l158XHv5G=gu_QOH!x8}mU z%=SX=b!DL?e6)a0WRbhE>vThxdqNhdLeD=$Hpjgie#?_tZxFWjq}g*1oPH>Jtvo4# zUjtVemw_wNbWKaNKpSgJOIUi3S)lBEmyPmCJ}|feNrq>5mgny3V;v<)nx9C`sgxIj zY?2+bk;$!5;T}8q70TrjJt-vS*tnqGW7oRB)JN8+m>5A`^=Jj=5o|iL?r3esi@bE( zOo|ssQj80U6<9%W%$TO~r8J*tiG}y2ge9e%Xmx7L&zboyPkPeEPEXmi99($??Vg z%snQ4AD;*|0v2py7EX>KSN1i6N0fc?L9?y#MpE}#QcJ3TG=2kXz^cup!t-kyU%ESe zHiAd_A|wW`Ws+XsdxuJ%qIRI|N|L9R`KlU!J~%zK0DO7v(xKjL!u1O9F}M*tOzK_n z8H0x$7T+V^^v-R+&>n5+vUt<)^YEqbgXDJ2 z#}z$>1SUE;f?1B(eop8|Sng}u5uVquBfJeKxvp~!W@eX^Stja<71YGE7rG(lm<#|9 z@chnRtR7U+ROiA@$Lo6s19AQd>M`y9@YGkUJS{%cn5LDS;tJJ&a9R`gF_zD#dOEG- zt`@LoXY1;PJE6X zX)tOgnV7W!a+_?#rg4@Rw+uZ5is#VnU|N3|o%S9)O> z?|J-f+G=F3jdg<3~i16R0eY-lKoy@AXP2ME}nEU7mr z8x9AN+>CDZTYDnxV>fknz4)f*^__tTTV51-k<6t{Y&ApWp{6o#ZqHL_U@1#EZQ)&)`3TI>MZ@{n6&)`|CLG3tAu>4{9o{U^U0KRWt;Xg0 z1ESf&ldBKDOP8d*lr}hR#hHjLRY95GE~YYNSmjIbF|{wBq(Q00LEE9MerXd~fH1m* zL^DigvJzXy-(nek15XyKLsBwK-C{YmVNhCv6f)h^ORPd=^a`uveS?|$GOMw5o$3i{ za)wEy16f^R(i2F(Jtpz|$c(dh7yyG%(&SH0fTR%Tl1x$<1BSRZ0`>q`07TeA-j7Yd z5x`ZY#kdrdg9>1wJl1*Tt}!YCtNx6zUQG;Monh6lv4TsZa#8~35VDk17_IRIgwe9X zsWPysL;wVAsw8M_R7HSY05-9po>cLsra-997g481z@zFd@CGE>F-aZ0HvlC|@e;sj znGgwZvXIngaMDO>a}0@Bf>pi(O|(=K0La>1eYAK4+*<>PC5zCd!L6wUfJA>#b=DuK zvxGVp>NKcUvNX=1-U-q!Ce&(-b*1rg0-NpwKgHhwkd0)Sa4ad3OnihEN~+f-^lOY( z4ql1ZVGVXcyLJvXTpg_?YkXr(3r+y|){}L<`EiDy{8S&ENY;~;WEF6=v7;p?CizEp z0JYcu3D7E{rxohy`bP@8HWYSk@@9%%6PM25cr-$x%-%f$;u#%;>l); z7;!AnHc|L3TvJLpLFYG|zr17cHpLU{-(CkXVs|$@u+72bY-X3IIRq%TOQAB|NwIYb zL69&Dx-^{s6YHj%C{rQwc)V6P1S8F$cOI|botgnC`}jl5@Nk9|6h)`zCRBT{@l8nI z#WjI}!;PVH(s3^uL}I0-*C zxZnHM-Dc=G_>Fpu=RMAPnNGSgEiHJ3rOQ(+O@~O|CT8ew`aSTi<8#>Q`=~H+c9~1L zM&{DR_~jwf3E;+D0r58ScIKI9pW}y_?LogI3vIvaIennpe-EO?>A8Zb6P$tIVXA#N zSIw@p@L5RhEsolGs`F=J+(;TPb-Q-%&C8BU_p5t25kz(t&+(p}y%G=GZS@qzSf+1M zhnGo>GvHe6SAv$7c$^YCt%)rp0NNqoP>`Fc?Nrc0lzIBj6nU{U-G1L8``DJsgrfzifR+0r~LLjCEuguhe$x}F=$#EhyFkoy#BC)+9Gh4`CW{~ z;ZeF3QVb9mD7lElF8JZliEj;n!E^Jg&vJJCzU^Mg#E+@sPbhhn5(TY4rQGY3yoSVH zP|GoOO0_CM+`@~7-8iyK_lXUNZ4-$@JxS(^cc>NeQ1NqW3I~Vuvml5%610D%>urL} zYKU`WHmAESsTRGDL<8dz|1N`*LCZ!4M6832bF6M`5H%|X&ViSSm5G@fcyF?rz6xfp zuvKt%1Me)9tEG&MGIhNI2B$idny5`go>9+Kx`H}%DgH)z5keCzS~|OKt;@XQ@UZ1L z;srblhms#4!6e?Iv#y|RFLjj+j&}#@I3^2pn%wUBQ41$Gq{^A?`ovG8?U_p`SGJdD zFJ=~0OMtvv?5N&e5Bac*Pp0ZCsw})Q{g|q}L>hF`%(n5VHC9IineR`IqBNLe3wgG; fN^=L_YtiI+T1?S0l8iD84l6L<ePfhbmx05RYg;NFrykW>CbfSmh~OOS(e^wFn$)x*cGWeW}vlbD+B>h9|5 z>Z-1<=cPiy*6{o5zkjsz_f<{%7gZ)dYIzY)_-7EVIT~kN?=Xj{x9;fbZ8!$rM#t=? zoD@s?q@6T)W+&6N9Ba(Wc5Nr8%BfDiTW|`hobD96C8sp*Q+CQc@1@&w?aG*@@Sbm1 zK^MG5{Fiv!0(Y6W)PuW+6>-TDIkJr2#Zl~59 z^cs;L^upR8^mq+TU9@laZ`q$>W!mK}Cau>ZPq;fB4`svc_VFsb95jMgFn-`hfdFIf zEw>l+Bfl|YS~ArQxYv={hVUT)lgJ!*Be&iOTOWqM{EBoJLLo2pM1~t`7&-UBsmyY!~729$4$h@T8XV<4;e-(`38rq1)c zaK}1iquh~3Bha@Pu~GhzL0*X$dFhT8mqvx8by&t2#>?PZ$GCHm9nX!5M>Jz4u8c}Y z+HgK9kLKd}BX*yK7orNE!@5wr&6pNd;*xsLkE+}}BF@5SF)qf&k^W(ESdFWrC0f_J z@angt6iH|03N)2`9QDL z;BZ-Z7MuruCH0Q@6kebmjL>W$vcm(#UOM_Iow%FB;aQN5PgM_3dXR+mr0Z78B&=q6+f)=Gon# z>z&>A-C!S))H=((NSyV0Ex+d>T0ghXbH5Qaf^Ppp4gEvXJ*^#VM-=Y$py$<9IBlQr z1P!+nUO>SQlm6SYp->_Pn|8x3T`WR^WYKXg#0q$rYC5!r`edpzUt_ANaY#KA&kH2r zaS)hha|m-Bvc0s>4-v~N+9B#IT2*@oesNvr8qCWsD|3w16H6G0oz8XwY%3W6o9_9o z-AGkb*F@*+`Igu7_WR<(V>Dwj{lV$4R(aBWs_|qyuh94hw=-rRBVs<$lkFr2tq*!3 zOzrVRV?!BKX)Kl@BT3er{tZNw-*`P}wJ15coY$2cS-8>-f@s&{@6hQY z(@i8JHzKEN)jh1`mdqlD1@)G2xuE2vEGLa{(3R$%cPo@R;qgJkOPq2IJ!D#)RG~B} z+N242$aH?#^m|~)RA2bL2of;X^!EKokoP&Z@+YLvFqGDg+t|D53XW9-0(nyJ^<=5h z@%u^-l`-qm?z_TOtAL;|^7=A8=pzSvGOt?DHyHFoxe?y#HFibNL*lLPc+pMI>(%=r zXn0}hg)*Ip*jf;2T3&_-Wn+;gq7JDXmGVRiAhLiFFeY?W)=ckuD&^a1VfCi)uFFga zEYX4VOyr7|7oiXgqW%DdG%XP=bM#hLt0xxZa`ge$G|uGmgKbF!Itz)wiIn=7ZL0xl zE1X=yQ`S|k1D8@O+oVOQSfJBSmcDRi%z+vH1Vl3}X0be5#NTG7zKNR6D$LStmS*b) z(L6YLme;9Yk=peltDwD#@**pMR}DeOT3IsSx}&;08o@CpaW1;j0>2# z1W+^wP{c-nCICJO%$egAj46-i$KYp<&x4~K<5c2Iy9&rrjTQia76||;=usI%546VR z8OVs1+RJUf#a4lYqm*QnWr;YPkyf!Y^&I6iW|0iHn4l^y$ zOxHg`Et7z=XoH`MGX$!}I_csvbV`7m(&)Y}evPpSC(&CM2k5h@z@43dGx*4QvI6wK zkN$vc!*zaI?MwzhZL(V%(|JDQD^rW=;vccyK1KF!pB|k7gxug~&TGePu8Yr94_J5F z<58@0U0Ko)HIyEM-8T5+iu;5rKdH)Fs{9nnn7v8You|iOQ4|q6DhG)&N+*yXkOW1r zs8B=dsuxDHgmeO=P#6NFEGm$qE*u0@Vs-+YAjfT^-wAlaWU(KF(fBkCM0hNx*3kJ) zJQF~&Mt3sYsk~!2ZGuFAbIj0dAzSS11pAKFAK$o~!lZ<3O*@MD2 zz&$Q}{b9nV#-j6fYOHS%Q*q2Q0exSl*;k{zuL(uPDa`Cx# zYL`h=sM%=;l@sMDF1WU$jqgDKz=7~v>hhJ$o!lq{ZBY^dv)!i+>f#wgGld=D#|HPi z;xhGr1LS{Jz@*OzUu*9EZ0_kSlkHc_q~KR*ba8@4lf}~ahAHB={4QW^-RH2_a~Lpo zdznt~jZ7zD`NhQ(@E0=$#8s3X^V#Q~=LhC?zt@u4rr+`EJ>ak?oVBD{_&$gR^lRyalT95+*)V~HbztK(s~GDEVg^kEFB4WK%v?pg#mf3J zFmsVD15;PgW})I4ZDBN_aZ}Z)cM)UgQTQAE1)VntRJx{at;tNi z&Vxq1E}lnSP_hu;0l_BTqHC?7Loaa^)9riv>OLm3baCA7`e6e%I8f!xPG#(;;r7%e zlq)-n(}t-9RR}2WHt{Lc6W>>Jz7g<22e&}=eN<-OoO~u#z9J2}8m5}~h#IMjg3R>B ocTZZ;2NT?B8dqt^O+a^3-Fbg+_rBYIn3eY8gzvG~qe)Vgvjr&n+>fL0!X_oej}9I|cboxxC-fl9 z=0-;Bxl!EVz6T>q`^=4!R?I@mGM&fFYuhF9_ryd z<42sb&aAJmnU@9bbS#)i?QAC4;6|o*A#`2I<4OYhd6ecr} zWLoGZN4C-S389fg{Ln58ca-nMi9;jLUKmt4fAo3V8B~`*0s{rw8ljy1Rbq26>&wq; z?Kq^hr`_69-;JNb_77?ZhZ=UOwYyC(j#^gdS!Ot)dxSgXtO5g*c}OH#lc(Wl$OX9! z{~G+W1*rFMfB5MAS7`no6gcP}w2nGgPt;UmYg|554ppKNo#aR!YPD27L`@fNNvEQjJ<=O=&hgwPJ|0^^b%BbR>AE!192=vRWc2Ul$!#j=j5`2 z|Ax#yg1T!e9}W9l`T18Wzi|A|q4HPm?KEB!0Q+rff^~AU36eiD$S~wYxOEfv6~>E{R?RN&1<2F zc#V}-z17^Jtp*~BEy9FVl|g)y*ey&@IYOVK@L}&_?R%KKk4X<5>;ufBcVQ+bhywN@ zCg>YR)IW1U{SGYROX$#pKw%65A4AJEAuzf9AB{m^wZ3{SC$MfcP-nNV(pgsQ;U2zm z(GPq*x#nQ(%9^*;+}xt)Y6C~`hV?czqBmH=_E2f#cKhk|l{UHE{_AHc?Uja$y8n5W zyJ3A*+uX3Wu0jF(1VnVva<^Yk|7g1I8oj+qC`8n^WxBH$A%M2~UXA3hz2A+8cAOTZPm^{W z5YtbfK+VvP{IDCENOyc7SRMsW|G4WjN=zWW<`JbGV9X}cWE$}VVuKh5;gvujhHxNO zEByN|8upO5u~`t72UJu%NCA=LJ%TsEVBiA{{=?LX43tPZg7in_F>YTzv!PMAU?7!> zRnKI9%qzb%?%=0^adfr=oDS#CNylG|XuMH(I(;5*-9f4mB-_^cF!nhNIsSHX90Q*j z%(qgnd$h5-wf<9Bmb2ne+(3enA7(o6(E}=&&3I@kKs$6h4^1mjaUP$tvY$A-4Y~kA z+PO4Z-Mm}?O_)cBTq;h>vx2uMvLY7*cbS%19oi>=yCRM&a8x0>fP3x|xWl0Y4l2bF zxEq(iy)**%va16`hKT>aA~qY==3BR+U&4e~6_JBf1_;*|m`Bz!W<~7RQ1>Tz2j*j4 zL=X7O2~24BwM+;j!O)C|Tff~}ZHn{3RcuaXXb61+VixWM4+gJkKCFggF5`I1uip1F z7}K+y@}-uu3hfhE4jl5{NTB$~nAsp9xXy{N0o?s9hYKVXzeu>49P58UW%fhGmm@j{ zsZ9X%oyma?6P}}dqh4QIC5@jB7H=5ki?7)6k3)zXmqL6E+NmqSeRO&b3;(K>Qc!0B)`qglj7Z-bY1r+1Op15SP=lack z&LXZE#N`sh7;De=n@8@3#3htnCbUB%BtdX(m8l7Q6N9!m+%r7hF+46XB#RkPwN;fn?)T3FJ_*l~A=+8Ov&*g}SYWIXfrHT96M5 zc0uIzpcopq5ti(dDCdH5Sg|XR&y&KgVo#8F=@nA?n@q~2@?5tk$$MmiTzM|rSBXg` z$yLZrk@v|InTFgn`GCxjS;)@n}wp!{W$tMY*Iq}X!jw! zjw9gP3Ed5|>5(4WZWMR8??KPf4s)ZV6|<1COy@E4+IC4ip7;q2xOMbr_!xMx((HeW z6S*Pv(8zcp3^}+r;ZB;65cxpkQVQly)l??x3++JXa;gz+0qThULOIBBCDqg1p>mK< z^+#%&cT}#y8w$xmYk})&J}smQni_f+Q8rX9uI#c4i zoH~ShM+AE*St;s+W~diYv~ z=yKa@`}7G-%+=-E+OoB3-rla=-i9IK$lScYdEb2E28}&SyqgsmO(M!0^{uLMjKuyI zZO%K%m$e!RtX9+YJKbjBdo4epiRHy%&va<0Mb@Y<(zTUk;w~*&9p27VuhRueW@?uO zSvhn&P7r&7b(!I{ski5}-K3rAf@{^BFeXFy$yC51n-5}_I0+Y=^F8jgSR6Vy)|FE& z3@u;PGou51Mfy|HqtSr;Kn>pMtqn0_Z{uI|zMun(2p8n&C9~L6!Ra^yb1XnrG6{*4 zn*h*ra#_KrA+rym?vmi6V;_T`eZ{eKU^U%b27xF7*`+SCN^);ZbHTX#a-U%auaM22VGnzdL%g})QYuITU)V~&Z88Y0|V`x zVAJm}#3tf37F+dJbDg#t$S4MZO{%KwIy@NS!rsCJ@f@WOdk1UZ#pFFqdhB2(=5cjl z?_+|!*aw)PcaOk-^#uG5e8gw)LXQH4Qxv!XEtjOg`2N3liULdZrAsXWD^>$ByLldF zS+U1^_>Yt8z?b7A2YnZpz4hkWIz5IB7J}D|+r$WN@Ch43&`$g9$Cn3fe82tAuLA9r zj*Hm;I?r9TzNoFOTI=VrfPDleI_bF^7sDT2*IlMJ*9e7(`leiW>Ldlwckd5raN2i2 zftI6}ko_)3JxV*Em`#+)G~x-w2GI|~ zD}g`^;Xr(?Sl_qMv4_HqO@pyKkfP#335X)^A>0W{B_Cw)UnUM@kVMiRq~9rzar*L+ z4UJ+622rV4^-T6pt;#QrTlm+YI64~v4u@msr2S7v5O36-PG84cw@_*X$%b{@jr|U~ z?B5xm$Dn5h<1NpE_Cq<=x6MlT}j0iVy_I$q@_VVIj508Li zJlGS5O!joYxvyEoF@rc8uKsS(d3GOiq3Z5F_iBVZ8sFM}rsxb}sj4@{{ F{10d)GOhps literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/npu_fused_adamw.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/__pycache__/npu_fused_adamw.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f72da710f540d1da76f6d5cbc5b7f9e47a67b83 GIT binary patch literal 7377 zcmcIpO>7)TcJ9BK>G>r&qG)0pdwMvO zEc>!O==!f$uU@_Py;nUyuGcFHp1=8%tJ^y(it?{inLGvxzlAsaZ6r)#s;6vGu6gP} z+tSpuP2bYdrhCSquvL(4hF2Vvwo0nu^6w(E1-AC9^-Ve%BRLfAopD_bg4wlt=KiKS1K9hH@s z`MA7g$a0xg9-CVQR%Nxv%2tupSp#{A&9QmpCR<=lb&_Hd>C?3xpf>2klD5!t>5Z2y}ag z*5-#lTMc~gux(j)_}<7B9CL>Jz={HEn_I3QMvmukX6?Iva2kmAL}do)>I8-f9401^!!?R#&^jq0JAXOV8?q^z0oka3UKEMWgVN)n4m#GB!Wz zyP>tuMF>JxutUsRJG9koTkdq9>(qf^VK&vWe$%>PWoNV+aX$=%A|z8Sn_)Nx6_%T_58D zpIfw+wZk7;54iBTmo3RB%x!c%Ck!__4g}pMIZ34o+cPKGu?Ef|oq^o2170j9g5j#` zTY?Wgrz=m`+QC|#P85i4KPAmjt#x-l3+|8k{VvD_W&Fc?`l3Y=3w z60jz;XJsT=nb25)>~X%G4*0S?z7N6QUbh^dS+}orI#$>5Y1wW+2%tj0Ch5c)xTgCbfT zQiBcHzz00)2Tbb8-6-hxoiK8{*6md59J0}logOKXPw`^k5w7D$PZPb#yTJ(H!mNAT z?e!xI4baj%yf^YN=P0Kq`Mn`}BMCzrxynfW3aw=qhU08|ytQFXie%wa^`mo#6s2j9 zJrM?n762In^?Nxz((Zhc)f3=uYL8IgWeF})-q@+*tq=*q^bKpBul^9rd)yI3(h{)v zobEP{AkgKHXtTeU(U954&!n|{xqWrTLUQ$L3pBd{-`R1yu7qH+-<&S-OC}3s8jd2l z?8ukPI{-ST2h4#Odct8eSt_rKeTRD^M@+fnL-6+8oxKSzAmV}LFqYyMK^)_LFo5;| zE=az>PZ8`-$r5uuPE5F6C>Q{F+x6V&FlYXuJae1LGk^a2SFnCM9E*~QoDfM)&l!ZY zQ)_uU2yhJkGevvL@j_CQhkYnAmwXc*&~|QoaQ9uBKT+3UCnrr?iO7BAuFu*L!9Y$M z0tf+`w|!ZAC-5Qp9{0O2uAR)RfEhpLhk?&Jb_#9nwq=uT-2ZkaHC6b26b-|TwKYc^ zxcluu^wym1aBclZ>sQ*ZUw!kkWSEG5E6e&uvTv-fw=ci8etA8o`>l5QG~HmxeZlu# z{;=)yXbmbm8eDhTTjKDiqc?U|?`+8hzWTo}|LY$<*#7YPms;=deo4>2UGM$i)1`m< z`~Ufm>tAjB>E(as@^M%Jn`<2<6h2TFJ|PRoS{1@Uu?Dq@su$w!v5kv`g~Oe6mg zBTYFHE!oI|TVSg}BHnpB2U>R*e-~kBq1gE046}VywocCooWvtmub?kXwo;2U6*UWG$A=-%Q*wBLZW#hDM1( zA`nRt8ZvOA#2}nbmfVoyVp|4l_5{X3HypwA1ck_nY1`x*ZM$VAM##OLq?kGfQAM|? zQBtR*LCGAF#LTceq+r~(WfW!GUn>9m>RLY-@U;Wi2@U}By*0)ou{Pzvf#p7I4-dsz zkog_F;fqKV%~TsIz30@XeA9YGEviRLGl52X!faQgiFinJX&+L9FQX7Ckt%gUi`6Hp zsvKO1m0kU&^6~r_T67FeW*TYA-no0Ymq1(eO}VREt%6v@{8TD6+YWg|u`svoq$VX| zr*18&peK_+OBYL+1A)@Wq##GMRB@gNU7+M5CA7=?c*7zRMV(h`>d}SM8=J1zrOnVo zrb(BpckqS@5^P0@)$__&$0qehDvU)F9Y)5cT~i`$SC3Vs2Gc)-!7$?)ZG~$va`BH( zjV)eN{QcS1u@M``g^3}n3JmZWG{=RwAV(F*h%t(0Jad(CF)qrH#cX5|BeAYLhSH|A zI_MxGStwS0zz2aiv@?Y+@q?l5?DvvNRsuQx&V3;%}m2l6ddUJMNYD-T$ zKk{TDwW?Hcq;@!Th?kHkYDKH5rnaaps)n|t&6DXL6^-6IKD>P#B*o+%Y&rZ24 zYj71(J1Vl4>xy_kuF957C-&ZlS(e6gOc~GPUBKIn%5fRGUV31Bt_m$S$$^aLq6)Q+ z=aE;CFCecXZ${OqhWt!C7oTA|w0#~qEFoS%PI}oy4y}VOn(-OpX|H_`HAt!+tC=>M zDJUG>ojC}L1qclNAhH0U!$2U%p&$yH1d8E`g+SFoOBZ)wmm$8nD@RbwB5&*5VdA?u zF!39dyh6$MC?V(3YKZS3ODeWA7}7;N>?hxmr%nnSMj>m?8*4MH5N#{p^0u>=P>X^kS{W)K01r3ss8!c8QwcKET1W@-j3 z+em*}1+5y|i>j&DP;SVdp*Ar}{xtH4@(1rY`2iVPGYZ*$4hLOQDSGI%b1(Ays1Qbs z)mYtC1#IjtBAr;ZwHZgL?!r~}q{Z&NbWg!|C(YMShRuSW zjc;slsM;o?5<$K=xiq6txH?VvovN^S9RsD_i`QsMnaisW+^M*UddrYXC*Gt!)lO9R z^eMbFp*bV}+(iBY6GKvSMT*gW zvXIi-5_=&Lpyw-li$E8Z{OPVH-i+X_Q}o=0MKxg9XulK%)>&F#Fz5$qu)I6^&D(6z9o?~US&9TaPMFSd-7viG$Z`7+;uZety zW6&s3+j->-=8u86v3a7t7|$P95TdKQRjiG%=M+}^jJ(DI+D)|A;)d)o(Nq6SM-RM| z_~WR)+W=xO#7)+at`L<(-rHZPy8bGx%KJrB8M1 zPqkc03i7K?=Fmd=dqPHo(~z*5xe0Pje+6-bw~&x~Xpl1iF2NbVF)RY`Yp9WnfZKqZ zXd)=o^b1-Ou7LtY{Pc6`1+}QnYcFXpYBlZX{3#?$g+?oF=D0=Iq%vNKTl*+P@B~bK zq=n50m!d45)1D|4%N^@kZcvUW3{fO7qbxp*3dhAK%AeOTN_~WOvYVeOca+~HFHr(Q znJ2u20ZQe}OWY?+OD|mH9FAUp?uYPaf1&>1Lwo~IO89!}M3Im>pDf$>@1Xm@c07UT z>Tu!?a>d7&=S)v7P5M$-5!!iI?k>c8U@g_`D);HrsGKY4tWt|(DB-Rs?|r4N7lb3h z69aiD=>ISAE+w-^FC)HxM|*e?2-F^J`feUKIw(-S#1ZHtrwQYQ*^s=6kZL^?j+w#gSFQREsNWL}QEA1!U zaLEtS%~F7x)U%s?mwO)O=R2BQZ{^LFbc%XHx`>!-hNh^E`C7reV6K~W^R=QXuhXrv z_!*JJpLO^XOIlEv#-9WsTW3%HZIXYb5IXg}D3t`!bR^vEc4p^x`bScj2C~G%l)bb> W%q8ZvR48vRpcWGBl4a3b)Bhj0)FAQz literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/train_npu_hw.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/__pycache__/train_npu_hw.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aeffc9d3c26caf6a223af3a9cb046ba86c35bddd GIT binary patch literal 10725 zcmbVSO>i4WcAgph5CB2&M-&B;`WgKTl0ZtbCCjuVOBN+twoGX$+Oo%%XNTw^FyH_K z^$a9xJlIWETd7nkN|dB3Rmpl&U=FED%73;p*>({T}d)@sS?~IM*Bz*qj-COJbd{dHsMwR|2iNbY!-Mg|RF^S2x zq(yI8lSN(86!A`I3Gq&9NxT(1)kte;Il>*$GNLW3W${kfxyGnA+Q@79Ui+9fCfW;) zacx4tlJ;0*Qk!f{X;Y$}viCHmwduxQZLg@O?U}|tZC{VRs1-p!Viy~;+N^+O?77Cg zHZRIql=o}Ak+D2La$C7r8jj;mW$JscW!25)G zl1*+ZFJ=CoDb-K0Df8HKsVcKQFBGk0mb$06CGCuPrhXO}*UdDQp3C(zo5tVX7m6&I z$IUa(W$>3DNTr!qq~n#6947A@Uah2rxn;vM?m7lDc{sIfxgNLH+n&iD7>$-~a$tcb_#Y>rK7iye}pU$#~!3hHVSVv6WW)cH1@C4Q4bRR%l4{ zfw4#<_`;kd_qP&~LV<(qn>3sFBxke-G!fL zV>{XlYtWhUVVd`{nE_-UD*`qf!hR4b(2j<0AmPK@JCwhYLv800p47m)X2kUN5E z$mRWA^$!G(Vf$UdVOalAjCCc(@gBR+zBd@hDr9`X)&_e22=$K#tq*sM`4jdj)7WPN z+|St~_C>6&v+qM^!@dpRugBUejG^6R8>~8*)7p+6tX;NOhcufp>1;2}8nJp4{G3?5 zwX6OZbrx&q;OEBb-Y)z$>Q7?rPuZuS7>;NQwP&$*pGlzjGN#=C<8Nj zJb{G3^TQRt_`vj3M1`ii;qZpZRc3l-#dG-WlB{J6#`KzLxBO(eZ1ZaQ=Kg>C*GqG6 zBc_ZnbHixcUYPWrwM>7E+Vyp#vbm1nQvHwG&wlf-{`2S8{Cv5*Zg`cN?pnV2DuWo{ z7s}$m;6DmR#AHa(RPT2`~F0!byLQi2yi%*XKvlVSuDS|Y+GG7NpDuuWay1Lin^R4p>s6s zNEcd$n{LzeE}rvCD@MaqoejZ_0<2@VJu=`eMVw@rTq zQc>Zo`qWgp*|Lp_snYVmOh|>{^r=*P!=QI-@wjiG5HwrOw-+P}VriKo%@DKLxyO>Z1f9!jnLn5~9U zHPsE%@Y+yHDLI@ZZKP6jtcvMIa=>iNW<|G+XC`-FrAeh{SK1JW6Du@1B$yht_lFv@ zVzN6}SC2uQ7SLZtJ4w24=o==&g2SI0oLLeq*S~eA>6!2mW??B+$)Nk=ilP4RGO6pN zcb7z~S&YFD(aS4ZAN`9DV$23Qbk$S9Fq-txRx->iw9z;EeCeXQLOXLzU+LNw2Ky=C3o9%}FxNUOFbYC%=p1nhs z1b?Qww&S^E$M7uSxFwnIAuJZ=AD8{;NLge8O}Ng(Q=9yU`VQIRBF+b zl}ZV0S55d5Ng0{pXu0gPJl!^*n6{?eJLe;aASPg=<>CR?a$nK*#y@eB%5XDtW5cS5 zkkr$EgMuHKDv?ku8<4NBJ6o?(M1At^c@>pkjl60(?Nt$`YZ`D$v`z z>Km4AhDqAI!bB)+`_Q}C%eD}p{k@BrgT-d6z1SmI6sXED=K2U*J^Utct%^>BU`A2d zpDLHR*>Iki@Hd(sEL!A)@S0W|Ymmi5OaT$7PirQJ**mIhJVAst8!hjdiV%P$g+Q1_ z1VOQ`h+qkSV&)TQFHAr>|5#M%M?Rv12U$&;{YBM-WufG!$plRH>NJT44v2&8hUw2l zZ^vL90&%i!hERme7EtFS$7(9S#w1?jjMjNJQJTZ7=zvzs%{(Xmi z>hMj(Q=xmDDa3MZHkp>JRVo6B4Nn?tq$3dEj3Jip|Hg-e$4+6Bnh?tjTNgDzu}|QB z_WXr67S7@CJRwBd&=DvoPW8HerP8o5s9vj$4x_aW;^;AXJ#Ps@Tk!rEHincoprg85VBvDh#IS z;pF9-T+7oI4XpwerZ#NH@WKR4Crn_NVRGGZ>@b5xjO@@23vf0pV*NT!dTGIz|GC7^ zVs5@Uwpeo-=HixRI9pg1s*BhU@o;JzZ!$3&-iWiC$fJ8i}zVr0!RENOD1* zi2dc{Oafm8U-6fZ>N3^8?OR9yk0|5`@$BI!eV(W%sE2}FP$G`N#2@%l&xt->{5?fI zCv!4ifP58lA^K44r0^@wrDp;4q^?ZosAzXwPk70Cipf`yz!0Ux6$v~2(>0cne&zEJGy^=T>mZ0t3ADENF7n+QZMX(amJ*2Y`&CHIMfg%Wo$5 zpLhi-cgNY-w)|Y~$jwWrPk?8F|0R_GIfnNnE1;LZ@AWcHy`UwB6ZTYh&$iT|@r)t` zF&9uNJB?kxp6$~Cp7f?s+e@v_arSObdo%TYfXM{eP}l^WtC|IYRqSIiIyp?PXrmwA zc(8VBRll*kysD)xym|TZkCcgati z%_j&4T*X9bxrg^w?=I^P?r69C{Z|K{Yh;HD!%(%P@*>cMO@1Sv z5{g@4ZrDFRK^XJj5tzc!SPVi%EFwC^!Z2Z*%`oL+59evQVF(S=QCUk?+RV`My_Sm8 zh7^z+Ih`oNYPn%x)|5s|125xgGfW_SV(~gpq7|B`z;jKGeHk)L{h6PB$$aZ3@mj)pOnna_mGtm6s=!nFN0a_!Xph#HWNOWfO&i{X6#Xfl@~f za5Kw`D9LWoQy7f1*&rogn9yd;&oDVErV0Sc#4^mH>FfL2e#d8j?jaQyW$aIq0! zAQ9u`2nxp(FyWYeyi6QV&{F?@8sn7hjuIShMyVmCo`+i;!F!D5;9CEHQUOkil9v>` z@;#{vXc^}}r}jW5$J$3S5_E!%iOgWeo2*Zvz6alF!ea&DPIEE%*r08kk~D0hFHI-m z?XAyr_XW@dMo8m<1MZX9WKjy_c?u_rUW|=nisT>G17&BOiFRe zbZ3L4=wVno2wM5`L?_ircSeF#_Yld@Nd>7+hV7yJ<&UKx(^a>nWk_MkY-T3}eH>={ zzEY}6hr(U4JBl8UY)cf#AmQHn(fY9<3#{Y#o?x>t5}llYp6s3qXeLYBN{8m;bSLj2 zZyKZk%`+t9UL-$PAXRSbAmHWt!m{*esx!v+2V?poJAmi8m&)VShtVu5{CB|heEmZG zHE_TTBRf0C4x%@Vr2cv*<6UHjn2NEax|gD{Ab1ua@!_CA;n{nG!4B9_Bq_&HKGrKw zfLH#d!hh~v4#q(-iT4CJ9|zu4b3gi=tH0S@3dW(c$zJWPU;?$Ns8+ucOoGQ%c7mOR zG<&>j!FaI8o2$PK=yi7LMY1yu81-{Cm|}B-mePxKXD^^P*lE~Tg4euv>NmT~BpEa| zGWB=6?*&Cxp2f^yw0F?@KB#8#{@`Vj zU#EBXE?d}^LI01S|FFJ7Fm?u#FGBKSbE3zUU_GvJp1{8MGU;At=Yry9ns~DFgVKe5 zDaZ+$bCDdusHmOq)r#!GENou**8PYd{|)bZn9+1F66_7&AAr99d<2#;*O{j>`uPzo zZZ(*%KVYxTO7(leEPH)k+QyvH(uEnmC@gWUyM`rlKfBa9@KV`Qy^lameSG{}S(aWZ zkMbRO0_Hl&|2CNHJ`4^DJ$)kjo`y|Q&sPJ&`Sf|Fa|n`a+c?dm{{G7pucOyD=5Z&` zQGL1(X#ZDBzxhYC_(syX6ua!sa(yY`u5*!H*Y^EUZKY1g;5zI{~6 z`vqiXuhDD^8~d8Q3!NW_7Wembo94eA9EWZXviF2u#RhNdbZ{J6J<>Vh?d_U7Yz3Ty z6Op~VOh5kphmiE~*U+i3mtcQU>YT)eYolI8R6hA!W_MoD7VQ-5>V4?tR21jLCh3C} zf8n;sIS%&pbYf_j26C2kFxZj$pgg14wD5CSHT}$I*ly#nv-FMp$UnP#(}e6J7$6IU z?CtJd6JI3y!@oeX8_zG&`3R?(CH2dWkf(oM-MQ~d(%wmIqe_Wzgff*lH1f;nBuvr) z5P7_1RJ2i?bv+#i(3%r9^IIecc3bVGFFRsW=9dTCwx|%>x|}NQYP)(>98QdeJJ__- z-h%ESAK>dA!J~?m_t6}@U+&=UF_1S$IIWcuJ%3MmoWxdD4m17iD#~NreB8Eh+SVz1 zi~%VN=R&y|%B@}dpl@w=wB#R(iM(({?6%x%DD-iE1zFR-C@X&# zy&!Y(9=#I$4qp6ydVD~SojWyt7qEZ9w4d?6NkKG0 zE`4Y4z|<6Q29dkcw)8PV<6}$_O$cr}wGhGiali=G+6XR$K+gZ|5!mKfltjIrOfA2*b;RJFc5M{)zVI;P>2J^^LLKSIa z*F!pRtao)0UAl(3ien?yW$GKva1`D2w%o{LgGGeH$uRjEJwC!i8{3hV(h?Qy!sRgE zFwh4QD3o^%Q{uisOVvED9MH6e|7C20a{EwET ze4Q^Yc^6tpm=zv~&I(l3#{yB!jAJJo%BYpwcCqtnyiKMbH z-=y>i?tGAOas3h7sV;b1UML%(oC}kbnWNxjZqXsq+4OT{T&_pz#B97KEEBgCO(g6m z*7})54>tnv>QYTB?7Ys>QX;F41QAkKoGj}W=MLv@k-T&y#Mvd4nDlz4@E;xQ0iVq(iGVC4zh}MTT(Uv|Bii(g@-U1E&xYoxF3D@$Q{o zM|x>M{u(h%;quEy8lQ4Ww2p*IvlXUMVorl+fuSj$qfIoSKo(P^HX6HW)JBNftr=R1 zGQSNiC4PM%A|78MA;^NY^aIaq-Cf1y71znb`8{f))2WuD0a=C(|I%_g+*n6$)ibp` zT@k?940xKW6`~Si$75UvG_}zv8!nnPrP#zm zPD>c{iL9SCwkpQk`(cMR`2fokUC0=AAC!NG?hyL5-0(V;ldr{sO@2p0J=3eXd>&|8 zO8m@&;~0*fktAev+2L_*oXXLVkvpw;oZ@Vl8)nC~bd;J7aSe%_vrD&(TB6oo*NVGH zqo1818^YJA$0$eLOI(i-q(FIf{P+ZSxyYAtEi0lU`2lW`Vb@^_lV3HpbaWrAWyy4+ z#5eBWD4NDEf{~)Zq~iXv)uJReiHF!sV?fdaWj9P&p4o7(mt6<+>Ple_lnX)JFSN=uI2z zEj+j##t5PFXdTk2ZO@M7tA{a3)G%K9X;BX}9tRO&M$8M9u-kK?Q!09iYFK@7)3u1<7S7?wf7cSq;9A@-l)>lP8BC1k7$_!gD4QbazhD9Ra;+e#yS zOo`+PWV{MWUd}7UDDj*u0G?BbE~!idmQPKgJ(rqFq~#gp#?r`=Wxy}3%m9xlDLGwG zqBM0uNdqT`99bTaoRR?s64c;EN$WI9l+ZqbZ$GU~DeEHoA}&dkkStde(4Y=jPB{S> zWC1PxDfs5mUWEL_Gr6mUlHm%drtm(9v^tHFB%&-{L7sy&(=sP5g&95a6x~0%d~0oW zcx*wCQhUE5gK4WSti0u6^%{WzBV4&_So3}z9C!tf(gHt96MGuP}kMYnz;MHtv zUBnr&3v4~(HEO0MLz@t(*afjNERbDBL5>0@ZxBM3nid+CThWsAP|(1Ak9Ty>ovqlPm?NXr4dijvPFgb2UlKD7_`6`~~av6(B; tT5^pdvr9`cSw-|0+1UhaLRgfrJ6NCiBMh{&%!$kor4J})OOR>#e*nB`%+mk> literal 0 HcmV?d00001 -- Gitee From 63a0599e4e908649bce87e7b256665ee46aba970 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:03:45 +0000 Subject: [PATCH 40/59] my first commit --- .../contrib/cv/detection/DETR/d2/README.md | 39 +++ .../d2/configs/detr_256_6_6_torchvision.yaml | 45 +++ .../detr_segm_256_6_6_torchvision.yaml | 46 +++ .../contrib/cv/detection/DETR/d2/converter.py | 69 +++++ .../cv/detection/DETR/d2/detr/__init__.py | 4 + .../cv/detection/DETR/d2/detr/config.py | 34 +++ .../detection/DETR/d2/detr/dataset_mapper.py | 122 ++++++++ .../contrib/cv/detection/DETR/d2/detr/detr.py | 261 ++++++++++++++++++ .../contrib/cv/detection/DETR/d2/train_net.py | 145 ++++++++++ 9 files changed, 765 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/README.md create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/configs/detr_256_6_6_torchvision.yaml create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/configs/detr_segm_256_6_6_torchvision.yaml create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/converter.py create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/detr/__init__.py create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/detr/config.py create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/detr/dataset_mapper.py create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/detr/detr.py create mode 100644 PyTorch/contrib/cv/detection/DETR/d2/train_net.py diff --git a/PyTorch/contrib/cv/detection/DETR/d2/README.md b/PyTorch/contrib/cv/detection/DETR/d2/README.md new file mode 100644 index 0000000000..7f1d753190 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/README.md @@ -0,0 +1,39 @@ +Detectron2 wrapper for DETR +======= + +We provide a Detectron2 wrapper for DETR, thus providing a way to better integrate it in the existing detection ecosystem. It can be used for example to easily leverage datasets or backbones provided in Detectron2. + +This wrapper currently supports only box detection, and is intended to be as close as possible to the original implementation, and we checked that it indeed match the results. Some notable facts and caveats: +- The data augmentation matches DETR's original data augmentation. This required patching the RandomCrop augmentation from Detectron2, so you'll need a version from the master branch from June 24th 2020 or more recent. +- To match DETR's original backbone initialization, we use the weights of a ResNet50 trained on imagenet using torchvision. This network uses a different pixel mean and std than most of the backbones available in Detectron2 by default, so extra care must be taken when switching to another one. Note that no other torchvision models are available in Detectron2 as of now, though it may change in the future. +- The gradient clipping mode is "full_model", which is not the default in Detectron2. + +# Usage + +To install Detectron2, please follow the [official installation instructions](https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md). + +## Evaluating a model + +For convenience, we provide a conversion script to convert models trained by the main DETR training loop into the format of this wrapper. To download and convert the main Resnet50 model, simply do: + +``` +python converter.py --source_model https://dl.fbaipublicfiles.com/detr/detr-r50-e632da11.pth --output_model converted_model.pth +``` + +You can then evaluate it using: +``` +python train_net.py --eval-only --config configs/detr_256_6_6_torchvision.yaml MODEL.WEIGHTS "converted_model.pth" +``` + + +## Training + +To train DETR on a single node with 8 gpus, simply use: +``` +python train_net.py --config configs/detr_256_6_6_torchvision.yaml --num-gpus 8 +``` + +To fine-tune DETR for instance segmentation on a single node with 8 gpus, simply use: +``` +python train_net.py --config configs/detr_segm_256_6_6_torchvision.yaml --num-gpus 8 MODEL.DETR.FROZEN_WEIGHTS +``` diff --git a/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_256_6_6_torchvision.yaml b/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_256_6_6_torchvision.yaml new file mode 100644 index 0000000000..25d641845f --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_256_6_6_torchvision.yaml @@ -0,0 +1,45 @@ +MODEL: + META_ARCHITECTURE: "Detr" + WEIGHTS: "detectron2://ImageNetPretrained/torchvision/R-50.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + MASK_ON: False + RESNETS: + DEPTH: 50 + STRIDE_IN_1X1: False + OUT_FEATURES: ["res2", "res3", "res4", "res5"] + DETR: + GIOU_WEIGHT: 2.0 + L1_WEIGHT: 5.0 + NUM_OBJECT_QUERIES: 100 +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 64 + BASE_LR: 0.0001 + STEPS: (369600,) + MAX_ITER: 554400 + WARMUP_FACTOR: 1.0 + WARMUP_ITERS: 10 + WEIGHT_DECAY: 0.0001 + OPTIMIZER: "ADAMW" + BACKBONE_MULTIPLIER: 0.1 + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "full_model" + CLIP_VALUE: 0.01 + NORM_TYPE: 2.0 +INPUT: + MIN_SIZE_TRAIN: (480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800) + CROP: + ENABLED: True + TYPE: "absolute_range" + SIZE: (384, 600) + FORMAT: "RGB" +TEST: + EVAL_PERIOD: 4000 +DATALOADER: + FILTER_EMPTY_ANNOTATIONS: False + NUM_WORKERS: 4 +VERSION: 2 diff --git a/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_segm_256_6_6_torchvision.yaml b/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_segm_256_6_6_torchvision.yaml new file mode 100644 index 0000000000..ade490e6d5 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_segm_256_6_6_torchvision.yaml @@ -0,0 +1,46 @@ +MODEL: + META_ARCHITECTURE: "Detr" +# WEIGHTS: "detectron2://ImageNetPretrained/torchvision/R-50.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + MASK_ON: True + RESNETS: + DEPTH: 50 + STRIDE_IN_1X1: False + OUT_FEATURES: ["res2", "res3", "res4", "res5"] + DETR: + GIOU_WEIGHT: 2.0 + L1_WEIGHT: 5.0 + NUM_OBJECT_QUERIES: 100 + FROZEN_WEIGHTS: '' +DATASETS: + TRAIN: ("coco_2017_train",) + TEST: ("coco_2017_val",) +SOLVER: + IMS_PER_BATCH: 64 + BASE_LR: 0.0001 + STEPS: (55440,) + MAX_ITER: 92400 + WARMUP_FACTOR: 1.0 + WARMUP_ITERS: 10 + WEIGHT_DECAY: 0.0001 + OPTIMIZER: "ADAMW" + BACKBONE_MULTIPLIER: 0.1 + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "full_model" + CLIP_VALUE: 0.01 + NORM_TYPE: 2.0 +INPUT: + MIN_SIZE_TRAIN: (480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800) + CROP: + ENABLED: True + TYPE: "absolute_range" + SIZE: (384, 600) + FORMAT: "RGB" +TEST: + EVAL_PERIOD: 4000 +DATALOADER: + FILTER_EMPTY_ANNOTATIONS: False + NUM_WORKERS: 4 +VERSION: 2 diff --git a/PyTorch/contrib/cv/detection/DETR/d2/converter.py b/PyTorch/contrib/cv/detection/DETR/d2/converter.py new file mode 100644 index 0000000000..42882ce86e --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/converter.py @@ -0,0 +1,69 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Helper script to convert models trained with the main version of DETR to be used with the Detectron2 version. +""" +import json +import argparse + +import numpy as np +import torch + + +def parse_args(): + parser = argparse.ArgumentParser("D2 model converter") + + parser.add_argument("--source_model", default="../detr.pth", type=str, help="Path or url to the DETR model to convert") + parser.add_argument("--output_model", default="../detr_.pth", type=str, help="Path where to save the converted model") + return parser.parse_args() + + +def main(): + args = parse_args() + + # D2 expects contiguous classes, so we need to remap the 92 classes from DETR + # fmt: off + coco_idx = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, + 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91] + # fmt: on + + coco_idx = np.array(coco_idx) + + if args.source_model.startswith("https"): + checkpoint = torch.hub.load_state_dict_from_url(args.source_model, map_location="cpu", check_hash=True) + else: + checkpoint = torch.load(args.source_model, map_location="cpu") + model_to_convert = checkpoint["model"] + + model_converted = {} + for k in model_to_convert.keys(): + old_k = k + if "backbone" in k: + k = k.replace("backbone.0.body.", "") + if "layer" not in k: + k = "stem." + k + for t in [1, 2, 3, 4]: + k = k.replace(f"layer{t}", f"res{t + 1}") + for t in [1, 2, 3]: + k = k.replace(f"bn{t}", f"conv{t}.norm") + k = k.replace("downsample.0", "shortcut") + k = k.replace("downsample.1", "shortcut.norm") + k = "backbone.0.backbone." + k + k = "detr." + k + print(old_k, "->", k) + if "class_embed" in old_k: + v = model_to_convert[old_k].detach() + if v.shape[0] == 92: + shape_old = v.shape + model_converted[k] = v[coco_idx] + print("Head conversion: changing shape from {} to {}".format(shape_old, model_converted[k].shape)) + continue + model_converted[k] = model_to_convert[old_k].detach() + + model_to_save = {"model": model_converted} + torch.save(model_to_save, args.output_model) + + +if __name__ == "__main__": + main() diff --git a/PyTorch/contrib/cv/detection/DETR/d2/detr/__init__.py b/PyTorch/contrib/cv/detection/DETR/d2/detr/__init__.py new file mode 100644 index 0000000000..a618f82887 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/detr/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from .config import add_detr_config +from .detr import Detr +from .dataset_mapper import DetrDatasetMapper diff --git a/PyTorch/contrib/cv/detection/DETR/d2/detr/config.py b/PyTorch/contrib/cv/detection/DETR/d2/detr/config.py new file mode 100644 index 0000000000..9ea267dd6f --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/detr/config.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from detectron2.config import CfgNode as CN + + +def add_detr_config(cfg): + """ + Add config for DETR. + """ + cfg.MODEL.DETR = CN() + cfg.MODEL.DETR.NUM_CLASSES = 80 + + # For Segmentation + cfg.MODEL.DETR.FROZEN_WEIGHTS = '' + + # LOSS + cfg.MODEL.DETR.GIOU_WEIGHT = 2.0 + cfg.MODEL.DETR.L1_WEIGHT = 5.0 + cfg.MODEL.DETR.DEEP_SUPERVISION = True + cfg.MODEL.DETR.NO_OBJECT_WEIGHT = 0.1 + + # TRANSFORMER + cfg.MODEL.DETR.NHEADS = 8 + cfg.MODEL.DETR.DROPOUT = 0.1 + cfg.MODEL.DETR.DIM_FEEDFORWARD = 2048 + cfg.MODEL.DETR.ENC_LAYERS = 6 + cfg.MODEL.DETR.DEC_LAYERS = 6 + cfg.MODEL.DETR.PRE_NORM = False + + cfg.MODEL.DETR.HIDDEN_DIM = 256 + cfg.MODEL.DETR.NUM_OBJECT_QUERIES = 100 + + cfg.SOLVER.OPTIMIZER = "ADAMW" + cfg.SOLVER.BACKBONE_MULTIPLIER = 0.1 diff --git a/PyTorch/contrib/cv/detection/DETR/d2/detr/dataset_mapper.py b/PyTorch/contrib/cv/detection/DETR/d2/detr/dataset_mapper.py new file mode 100644 index 0000000000..f428a4939b --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/detr/dataset_mapper.py @@ -0,0 +1,122 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import copy +import logging + +import numpy as np +import torch + +from detectron2.data import detection_utils as utils +from detectron2.data import transforms as T +from detectron2.data.transforms import TransformGen + +__all__ = ["DetrDatasetMapper"] + + +def build_transform_gen(cfg, is_train): + """ + Create a list of :class:`TransformGen` from config. + Returns: + list[TransformGen] + """ + if is_train: + min_size = cfg.INPUT.MIN_SIZE_TRAIN + max_size = cfg.INPUT.MAX_SIZE_TRAIN + sample_style = cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING + else: + min_size = cfg.INPUT.MIN_SIZE_TEST + max_size = cfg.INPUT.MAX_SIZE_TEST + sample_style = "choice" + if sample_style == "range": + assert len(min_size) == 2, "more than 2 ({}) min_size(s) are provided for ranges".format(len(min_size)) + + logger = logging.getLogger(__name__) + tfm_gens = [] + if is_train: + tfm_gens.append(T.RandomFlip()) + tfm_gens.append(T.ResizeShortestEdge(min_size, max_size, sample_style)) + if is_train: + logger.info("TransformGens used in training: " + str(tfm_gens)) + return tfm_gens + + +class DetrDatasetMapper: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by DETR. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies geometric transforms to the image and annotation + 3. Find and applies suitable cropping to the image and annotation + 4. Prepare image and annotation to Tensors + """ + + def __init__(self, cfg, is_train=True): + if cfg.INPUT.CROP.ENABLED and is_train: + self.crop_gen = [ + T.ResizeShortestEdge([400, 500, 600], sample_style="choice"), + T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE), + ] + else: + self.crop_gen = None + + self.mask_on = cfg.MODEL.MASK_ON + self.tfm_gens = build_transform_gen(cfg, is_train) + logging.getLogger(__name__).info( + "Full TransformGens used in training: {}, crop: {}".format(str(self.tfm_gens), str(self.crop_gen)) + ) + + self.img_format = cfg.INPUT.FORMAT + self.is_train = is_train + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + image = utils.read_image(dataset_dict["file_name"], format=self.img_format) + utils.check_image_size(dataset_dict, image) + + if self.crop_gen is None: + image, transforms = T.apply_transform_gens(self.tfm_gens, image) + else: + if np.random.rand() > 0.5: + image, transforms = T.apply_transform_gens(self.tfm_gens, image) + else: + image, transforms = T.apply_transform_gens( + self.tfm_gens[:-1] + self.crop_gen + self.tfm_gens[-1:], image + ) + + image_shape = image.shape[:2] # h, w + + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + return dataset_dict + + if "annotations" in dataset_dict: + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.mask_on: + anno.pop("segmentation", None) + anno.pop("keypoints", None) + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations(obj, transforms, image_shape) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances(annos, image_shape) + dataset_dict["instances"] = utils.filter_empty_instances(instances) + return dataset_dict diff --git a/PyTorch/contrib/cv/detection/DETR/d2/detr/detr.py b/PyTorch/contrib/cv/detection/DETR/d2/detr/detr.py new file mode 100644 index 0000000000..95f89dff3e --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/detr/detr.py @@ -0,0 +1,261 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import logging +import math +from typing import List + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +from scipy.optimize import linear_sum_assignment +from torch import nn + +from detectron2.layers import ShapeSpec +from detectron2.modeling import META_ARCH_REGISTRY, build_backbone, detector_postprocess +from detectron2.structures import Boxes, ImageList, Instances, BitMasks, PolygonMasks +from detectron2.utils.logger import log_first_n +from fvcore.nn import giou_loss, smooth_l1_loss +from models.backbone import Joiner +from models.detr import DETR, SetCriterion +from models.matcher import HungarianMatcher +from models.position_encoding import PositionEmbeddingSine +from models.transformer import Transformer +from models.segmentation import DETRsegm, PostProcessPanoptic, PostProcessSegm +from util.box_ops import box_cxcywh_to_xyxy, box_xyxy_to_cxcywh +from util.misc import NestedTensor +from datasets.coco import convert_coco_poly_to_mask + +__all__ = ["Detr"] + + +class MaskedBackbone(nn.Module): + """ This is a thin wrapper around D2's backbone to provide padding masking""" + + def __init__(self, cfg): + super().__init__() + self.backbone = build_backbone(cfg) + backbone_shape = self.backbone.output_shape() + self.feature_strides = [backbone_shape[f].stride for f in backbone_shape.keys()] + self.num_channels = backbone_shape[list(backbone_shape.keys())[-1]].channels + + def forward(self, images): + features = self.backbone(images.tensor) + masks = self.mask_out_padding( + [features_per_level.shape for features_per_level in features.values()], + images.image_sizes, + images.tensor.device, + ) + assert len(features) == len(masks) + for i, k in enumerate(features.keys()): + features[k] = NestedTensor(features[k], masks[i]) + return features + + def mask_out_padding(self, feature_shapes, image_sizes, device): + masks = [] + assert len(feature_shapes) == len(self.feature_strides) + for idx, shape in enumerate(feature_shapes): + N, _, H, W = shape + masks_per_feature_level = torch.ones((N, H, W), dtype=torch.bool, device=device) + for img_idx, (h, w) in enumerate(image_sizes): + masks_per_feature_level[ + img_idx, + : int(np.ceil(float(h) / self.feature_strides[idx])), + : int(np.ceil(float(w) / self.feature_strides[idx])), + ] = 0 + masks.append(masks_per_feature_level) + return masks + + +@META_ARCH_REGISTRY.register() +class Detr(nn.Module): + """ + Implement Detr + """ + + def __init__(self, cfg): + super().__init__() + + self.device = torch.device(cfg.MODEL.DEVICE) + + self.num_classes = cfg.MODEL.DETR.NUM_CLASSES + self.mask_on = cfg.MODEL.MASK_ON + hidden_dim = cfg.MODEL.DETR.HIDDEN_DIM + num_queries = cfg.MODEL.DETR.NUM_OBJECT_QUERIES + # Transformer parameters: + nheads = cfg.MODEL.DETR.NHEADS + dropout = cfg.MODEL.DETR.DROPOUT + dim_feedforward = cfg.MODEL.DETR.DIM_FEEDFORWARD + enc_layers = cfg.MODEL.DETR.ENC_LAYERS + dec_layers = cfg.MODEL.DETR.DEC_LAYERS + pre_norm = cfg.MODEL.DETR.PRE_NORM + + # Loss parameters: + giou_weight = cfg.MODEL.DETR.GIOU_WEIGHT + l1_weight = cfg.MODEL.DETR.L1_WEIGHT + deep_supervision = cfg.MODEL.DETR.DEEP_SUPERVISION + no_object_weight = cfg.MODEL.DETR.NO_OBJECT_WEIGHT + + N_steps = hidden_dim // 2 + d2_backbone = MaskedBackbone(cfg) + backbone = Joiner(d2_backbone, PositionEmbeddingSine(N_steps, normalize=True)) + backbone.num_channels = d2_backbone.num_channels + + transformer = Transformer( + d_model=hidden_dim, + dropout=dropout, + nhead=nheads, + dim_feedforward=dim_feedforward, + num_encoder_layers=enc_layers, + num_decoder_layers=dec_layers, + normalize_before=pre_norm, + return_intermediate_dec=deep_supervision, + ) + + self.detr = DETR( + backbone, transformer, num_classes=self.num_classes, num_queries=num_queries, aux_loss=deep_supervision + ) + if self.mask_on: + frozen_weights = cfg.MODEL.DETR.FROZEN_WEIGHTS + if frozen_weights != '': + print("LOAD pre-trained weights") + weight = torch.load(frozen_weights, map_location=lambda storage, loc: storage)['model'] + new_weight = {} + for k, v in weight.items(): + if 'detr.' in k: + new_weight[k.replace('detr.', '')] = v + else: + print(f"Skipping loading weight {k} from frozen model") + del weight + self.detr.load_state_dict(new_weight) + del new_weight + self.detr = DETRsegm(self.detr, freeze_detr=(frozen_weights != '')) + self.seg_postprocess = PostProcessSegm + + self.detr.to(self.device) + + # building criterion + matcher = HungarianMatcher(cost_class=1, cost_bbox=l1_weight, cost_giou=giou_weight) + weight_dict = {"loss_ce": 1, "loss_bbox": l1_weight} + weight_dict["loss_giou"] = giou_weight + if deep_supervision: + aux_weight_dict = {} + for i in range(dec_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + losses = ["labels", "boxes", "cardinality"] + if self.mask_on: + losses += ["masks"] + self.criterion = SetCriterion( + self.num_classes, matcher=matcher, weight_dict=weight_dict, eos_coef=no_object_weight, losses=losses, + ) + self.criterion.to(self.device) + + pixel_mean = torch.Tensor(cfg.MODEL.PIXEL_MEAN).to(self.device).view(3, 1, 1) + pixel_std = torch.Tensor(cfg.MODEL.PIXEL_STD).to(self.device).view(3, 1, 1) + self.normalizer = lambda x: (x - pixel_mean) / pixel_std + self.to(self.device) + + def forward(self, batched_inputs): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper` . + Each item in the list contains the inputs for one image. + For now, each item in the list is a dict that contains: + + * image: Tensor, image in (C, H, W) format. + * instances: Instances + + Other information that's included in the original dicts, such as: + + * "height", "width" (int): the output resolution of the model, used in inference. + See :meth:`postprocess` for details. + Returns: + dict[str: Tensor]: + mapping from a named loss to a tensor storing the loss. Used during training only. + """ + images = self.preprocess_image(batched_inputs) + output = self.detr(images) + + if self.training: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + + targets = self.prepare_targets(gt_instances) + loss_dict = self.criterion(output, targets) + weight_dict = self.criterion.weight_dict + for k in loss_dict.keys(): + if k in weight_dict: + loss_dict[k] *= weight_dict[k] + return loss_dict + else: + box_cls = output["pred_logits"] + box_pred = output["pred_boxes"] + mask_pred = output["pred_masks"] if self.mask_on else None + results = self.inference(box_cls, box_pred, mask_pred, images.image_sizes) + processed_results = [] + for results_per_image, input_per_image, image_size in zip(results, batched_inputs, images.image_sizes): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r}) + return processed_results + + def prepare_targets(self, targets): + new_targets = [] + for targets_per_image in targets: + h, w = targets_per_image.image_size + image_size_xyxy = torch.as_tensor([w, h, w, h], dtype=torch.float, device=self.device) + gt_classes = targets_per_image.gt_classes + gt_boxes = targets_per_image.gt_boxes.tensor / image_size_xyxy + gt_boxes = box_xyxy_to_cxcywh(gt_boxes) + new_targets.append({"labels": gt_classes, "boxes": gt_boxes}) + if self.mask_on and hasattr(targets_per_image, 'gt_masks'): + gt_masks = targets_per_image.gt_masks + gt_masks = convert_coco_poly_to_mask(gt_masks.polygons, h, w) + new_targets[-1].update({'masks': gt_masks}) + return new_targets + + def inference(self, box_cls, box_pred, mask_pred, image_sizes): + """ + Arguments: + box_cls (Tensor): tensor of shape (batch_size, num_queries, K). + The tensor predicts the classification probability for each query. + box_pred (Tensor): tensors of shape (batch_size, num_queries, 4). + The tensor predicts 4-vector (x,y,w,h) box + regression values for every queryx + image_sizes (List[torch.Size]): the input image sizes + + Returns: + results (List[Instances]): a list of #images elements. + """ + assert len(box_cls) == len(image_sizes) + results = [] + + # For each box we assign the best class or the second best if the best on is `no_object`. + scores, labels = F.softmax(box_cls, dim=-1)[:, :, :-1].max(-1) + + for i, (scores_per_image, labels_per_image, box_pred_per_image, image_size) in enumerate(zip( + scores, labels, box_pred, image_sizes + )): + result = Instances(image_size) + result.pred_boxes = Boxes(box_cxcywh_to_xyxy(box_pred_per_image)) + + result.pred_boxes.scale(scale_x=image_size[1], scale_y=image_size[0]) + if self.mask_on: + mask = F.interpolate(mask_pred[i].unsqueeze(0), size=image_size, mode='bilinear', align_corners=False) + mask = mask[0].sigmoid() > 0.5 + B, N, H, W = mask_pred.shape + mask = BitMasks(mask.cpu()).crop_and_resize(result.pred_boxes.tensor.cpu(), 32) + result.pred_masks = mask.unsqueeze(1).to(mask_pred[0].device) + + result.scores = scores_per_image + result.pred_classes = labels_per_image + results.append(result) + return results + + def preprocess_image(self, batched_inputs): + """ + Normalize, pad and batch the input images. + """ + images = [self.normalizer(x["image"].to(self.device)) for x in batched_inputs] + images = ImageList.from_tensors(images) + return images diff --git a/PyTorch/contrib/cv/detection/DETR/d2/train_net.py b/PyTorch/contrib/cv/detection/DETR/d2/train_net.py new file mode 100644 index 0000000000..82f692922e --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/d2/train_net.py @@ -0,0 +1,145 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +DETR Training Script. + +This script is a simplified version of the training script in detectron2/tools. +""" +import os +import sys +import itertools + +# fmt: off +sys.path.insert(1, os.path.join(sys.path[0], '..')) +# fmt: on + +import time +from typing import Any, Dict, List, Set + +import torch + +import detectron2.utils.comm as comm +from d2.detr import DetrDatasetMapper, add_detr_config +from detectron2.checkpoint import DetectionCheckpointer +from detectron2.config import get_cfg +from detectron2.data import MetadataCatalog, build_detection_train_loader +from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch +from detectron2.evaluation import COCOEvaluator, verify_results + +from detectron2.solver.build import maybe_add_gradient_clipping + + +class Trainer(DefaultTrainer): + """ + Extension of the Trainer class adapted to DETR. + """ + + @classmethod + def build_evaluator(cls, cfg, dataset_name, output_folder=None): + """ + Create evaluator(s) for a given dataset. + This uses the special metadata "evaluator_type" associated with each builtin dataset. + For your own dataset, you can simply create an evaluator manually in your + script and do not have to worry about the hacky if-else logic here. + """ + if output_folder is None: + output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") + return COCOEvaluator(dataset_name, cfg, True, output_folder) + + @classmethod + def build_train_loader(cls, cfg): + if "Detr" == cfg.MODEL.META_ARCHITECTURE: + mapper = DetrDatasetMapper(cfg, True) + else: + mapper = None + return build_detection_train_loader(cfg, mapper=mapper) + + @classmethod + def build_optimizer(cls, cfg, model): + params: List[Dict[str, Any]] = [] + memo: Set[torch.nn.parameter.Parameter] = set() + for key, value in model.named_parameters(recurse=True): + if not value.requires_grad: + continue + # Avoid duplicating parameters + if value in memo: + continue + memo.add(value) + lr = cfg.SOLVER.BASE_LR + weight_decay = cfg.SOLVER.WEIGHT_DECAY + if "backbone" in key: + lr = lr * cfg.SOLVER.BACKBONE_MULTIPLIER + params += [{"params": [value], "lr": lr, "weight_decay": weight_decay}] + + def maybe_add_full_model_gradient_clipping(optim): # optim: the optimizer class + # detectron2 doesn't have full model gradient clipping now + clip_norm_val = cfg.SOLVER.CLIP_GRADIENTS.CLIP_VALUE + enable = ( + cfg.SOLVER.CLIP_GRADIENTS.ENABLED + and cfg.SOLVER.CLIP_GRADIENTS.CLIP_TYPE == "full_model" + and clip_norm_val > 0.0 + ) + + class FullModelGradientClippingOptimizer(optim): + def step(self, closure=None): + all_params = itertools.chain(*[x["params"] for x in self.param_groups]) + torch.nn.utils.clip_grad_norm_(all_params, clip_norm_val) + super().step(closure=closure) + + return FullModelGradientClippingOptimizer if enable else optim + + optimizer_type = cfg.SOLVER.OPTIMIZER + if optimizer_type == "SGD": + optimizer = maybe_add_full_model_gradient_clipping(torch.optim.SGD)( + params, cfg.SOLVER.BASE_LR, momentum=cfg.SOLVER.MOMENTUM + ) + elif optimizer_type == "ADAMW": + optimizer = maybe_add_full_model_gradient_clipping(torch.optim.AdamW)( + params, cfg.SOLVER.BASE_LR + ) + else: + raise NotImplementedError(f"no optimizer type {optimizer_type}") + if not cfg.SOLVER.CLIP_GRADIENTS.CLIP_TYPE == "full_model": + optimizer = maybe_add_gradient_clipping(cfg, optimizer) + return optimizer + + +def setup(args): + """ + Create configs and perform basic setups. + """ + cfg = get_cfg() + add_detr_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + default_setup(cfg, args) + return cfg + + +def main(args): + cfg = setup(args) + + if args.eval_only: + model = Trainer.build_model(cfg) + DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(cfg.MODEL.WEIGHTS, resume=args.resume) + res = Trainer.test(cfg, model) + if comm.is_main_process(): + verify_results(cfg, res) + return res + + trainer = Trainer(cfg) + trainer.resume_or_load(resume=args.resume) + return trainer.train() + + +if __name__ == "__main__": + args = default_argument_parser().parse_args() + print("Command Line Args:", args) + launch( + main, + args.num_gpus, + num_machines=args.num_machines, + machine_rank=args.machine_rank, + dist_url=args.dist_url, + args=(args,), + ) -- Gitee From bf6bffabd9fa11518f6063ebffb998eea7622bc7 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:04:06 +0000 Subject: [PATCH 41/59] my first commit --- .../cv/detection/DETR/datasets/__init__.py | 25 ++ .../__pycache__/__init__.cpython-36.pyc | Bin 0 -> 836 bytes .../__pycache__/__init__.cpython-37.pyc | Bin 0 -> 832 bytes .../datasets/__pycache__/coco.cpython-36.pyc | Bin 0 -> 5100 bytes .../datasets/__pycache__/coco.cpython-37.pyc | Bin 0 -> 5035 bytes .../__pycache__/coco_eval.cpython-36.pyc | Bin 0 -> 7198 bytes .../__pycache__/coco_eval.cpython-37.pyc | Bin 0 -> 7191 bytes .../__pycache__/panoptic_eval.cpython-36.pyc | Bin 0 -> 1760 bytes .../__pycache__/panoptic_eval.cpython-37.pyc | Bin 0 -> 1758 bytes .../__pycache__/transforms.cpython-36.pyc | Bin 0 -> 9336 bytes .../__pycache__/transforms.cpython-37.pyc | Bin 0 -> 10332 bytes .../cv/detection/DETR/datasets/coco.py | 166 +++++++++ .../cv/detection/DETR/datasets/coco_eval.py | 257 ++++++++++++++ .../detection/DETR/datasets/coco_panoptic.py | 99 ++++++ .../detection/DETR/datasets/panoptic_eval.py | 44 +++ .../cv/detection/DETR/datasets/transforms.py | 330 ++++++++++++++++++ 16 files changed, 921 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__init__.py create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/panoptic_eval.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/panoptic_eval.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/coco.py create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/coco_eval.py create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/coco_panoptic.py create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/panoptic_eval.py create mode 100644 PyTorch/contrib/cv/detection/DETR/datasets/transforms.py diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__init__.py b/PyTorch/contrib/cv/detection/DETR/datasets/__init__.py new file mode 100644 index 0000000000..571b126ea4 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/datasets/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import torch.utils.data +import torchvision + +from .coco import build as build_coco + + +def get_coco_api_from_dataset(dataset): + for _ in range(10): + # if isinstance(dataset, torchvision.datasets.CocoDetection): + # break + if isinstance(dataset, torch.utils.data.Subset): + dataset = dataset.dataset + if isinstance(dataset, torchvision.datasets.CocoDetection): + return dataset.coco + + +def build_dataset(image_set, args): + if args.dataset_file == 'coco': + return build_coco(image_set, args) + if args.dataset_file == 'coco_panoptic': + # to avoid making panopticapi required for coco + from .coco_panoptic import build as build_coco_panoptic + return build_coco_panoptic(image_set, args) + raise ValueError(f'dataset {args.dataset_file} not supported') diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..816728a01044b95762478d70b371822e2ffcd19e GIT binary patch literal 836 zcmZ8eO>fgM7`C0H$n9;ECS~<*zy?|jyNt+g6*Fq zgcI(R&caPRjJV6auPEv84)@{jb38*)u!Sx(-D0p2*4|Z{mpnC|S!T8i`y94B1G9Xp z0AYhC=>3y5R(ORJDV)MBybTfI3Kg9-x((kUoAHaFID+^o#KRff4e-cx8^WBZt3AQ0 zd8Wf$=9N^eN;B=LTBJAHYgJxK?Q*7=?#$XNnJMiX&m6k8=p(e%l@9h%ne@-=w4P)t zOO;8uDYy|4T>#UV{zPI3r{B-xn|hJO%Qjx-tX@LpJf2*xy`cNL047TF8iWVV5Qlhx#Y6B?Lz`J3)jfTe;H+=~^nD=Wks)Y> zH)OVp5s1Cy4tGnkaMlE)6*=S;$Y%P!Bi{MpZ15Kl7l`zyVgM6$-(Ia@Ro$pO6$U!O z!XV!r$Qj)YMpdmwvTd4LsEkLR?(N?|uk#W_u(I1%ieqTC{W^Th$~L(kMaeZ zXVggGGBKCQ@qH8hi*H!AREr02(*0e{9?Xy>-#yRE literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69983b6ba7fbe379137776a3c57350fb23ab5b16 GIT binary patch literal 832 zcmZ8ePjAyO6t|tE$EA%&WZGPXmmY&w_6#?XeyU;U6+QM@h7WE;QR>&(l3Jzk?{(Zy*2t2zeg?;mQi^G@iU0WbGRGek=fQhCFz?$ zX3c_|xKKr-bk^kD1X{@a)&!j{DrJ0@>CE)z-L>M{gx0f0-xYm^Hqn@9A64n_Qsm-_ zYo2SH@N;kzBKrVlF#Uz(7_2|Z^j6e-x-2rWgtEnyaV@8ZAyZ0=rqGn0wqMN2f@^B0 z(5x-!jTANA*F`W%xrN}7H^wm@Vfh$hxuwertmuKZx8SUB3ABAA@`)v8g*RlrixG&u zVjlA=QhRHH(TW`M8rEg|ejwKS>22^=SSk=0&g2LtnxVT|n>C`KII*R2Zu*`5*vj?-}r~xPhWWh%UpydhpNo2-upQmo;r?x?ckdU_Az0}=F gKDDl$`bvMFuoLZY?fTuQ50GIB1oM0Rb_hNWRL+H-a_tJ$5+ zIU`#Vvn6Dm=Aj5uT0$WWDfFq(zaS5#uLbA1bf6^AhoZMW^eKI4f8UwC$R-K#F7wTI zuHSdQ^Ihk1rQ!ugKMdYo(6oPQCw><2bv*Lt08DFYOlL-<_l%~Y6K+Ok(^PM3{I;7m z+E(QB3Qc!Br`YwHrB^g&Gv}_x9By~Z-O9*pR#<_#ceQ2}V@2j+%;R(DpJydjM*k^R zVO6|OGxL^KpF8~6+qk}Qy~cthkUXi~*$G=aHNlfT(U-Nb7j(EEvRXS9HGVsY_JSmg z`zxOJS}c?3pw@~92Tu1k__bc8;wqw?Cfo?wBlZ4aFE2JwbQsAN-S+` zNAY&!(&g>T&$ZhP!P{K$ev8Wn;|Xuk=8aa|iv7JLjO5DTz&nPJw7Q;IH-cnmlaO8@ zcU#mu`I%!xeisPWnmUAPyr;Ew2+g?bHcf?_%mQw`r&(Il#?kGpvH?ohCUjf&cpG}t zmh{Lc0TM0IQ*C5)jZ{yK5pF3pQ_VLME44-#@olDILw&REWVYNJa3Sc9GuQXSewg@v zR`L@O^d+R&liAsgxPP0Ags9($qXW4WzlI}ag@NFMKyYwoNglN`FYI;vb{sJ-GB@b= z{dO4f%sa7FIlm~ML?R)ZqVKn&KuX{LqxQwqjh(p18~b~W{V<64A=*yk+DltENz|e- zzHq6;4M7VBET5($6Qb#-bw@u0t0yG1GC5Dnpegv>C#ohoPKs)z|I8Q~Lx`$truvdL zgt$^D_V4mOD>YJcWK8Ekh`U8`6SM96Mlh>wK)l9g-N@XDMrI`lBuN#9`P&8Ab&;s0iKj?8V;Q4+pW_uCgW#4~mFNpFHC2vI`tBh~0#iPjgbNa-@ zgdxQXX%r!n5LYkT>a?wi?$hx|1{*ZxHvk=NXzst6YS*+kULRVC*3}_>QeSguqh+M_ zNFO?hnL1r4G1GT7v6?!pfO~QhyX&L`3(Pz+hQPZnvzUE^6{*=Rrp`#$wYR*V!VttE z=6SeR2jjTe{#mq3%pqEBbGnC_lDly?kBQ+GsH)A9g8@G#O+w3LVNd9U8!Maja#n0X z@swIOOKoxE;QDqK-a%##t=-H%M?Anazp-B3oL z@CVMKbn-GSrd>4s@bNEqRWnCG+|rNKl~hns{smU2dRK#Kn=pEVk@4G!PD{BE zW{{#Wa>`ojT+>Jy5zA(+0_K}3L7er-=-Oo8K+(h>Rb$zJq6Id0|StbBqaSDk=hfjAnva8>U6x^NIt zXM7ivVp=3e2SWqZVJUSmr^%`?E^kTO&0BLPR+Q99396G~MXBr2++jH_(^|AkX_=j8 z^VqXA<$0b?Gi?>9wQEh=#rsCRd~{tKR?f(JG@BHll#wvrIL7VPW$2(sQ9m8saRaX%^y9QgBofkJKzBTl~tAC9EgsLqX%hrgH7#40N+a{)K*|+~h@X z;gR14=-@bSTo@v6NcG)8rj8E;wG%E$_#Y>D^|Q;tfR4mypMia7^lv+W; zJFz%G9{G)UdIe<1JUukA@=34$ji~)-MlCDu@`FK)*dZU5uvjHosi19qy~JeYvVPi#S4!nBii z@b!ldXH>-+t)%76Vl2XrFT+EQgctDmfB1>37^+vqI!*fy0CF@H)di*ES)m=pL2~(- ztRQ$E{vBkzy@8+ExVPt@&BNM7T1=LmS+_%e2f1E9sn6%hpRH0zu}Gu(olM_R9s(jn zJ}F*QG*k|aaQ_r-Ll>g1H>>pbdj3>JGHaLf!R$Cn{5haJjdH-;q@PDiz7C-2o`KBG zaE&Vdi~3V~O|KbMBmyoHg0f*D!$b>df}>ZBb=@8RJ>A9Fq~&7VH5V|qj5W*pvhg)x zI_Cgo)62+xm%xIcd_Qfy(-w!ju(=9D= z*>b9VsC|UiJMjGQ_2)M`-rxQ(c+vZ8t@GF0fBty6@sG962OGbP{`KF#uXP@6vLAeQ z{nNG1!SR!q+&|r0>wJ1o{nosG^ zaB&mGW)P8aE!+(HEbhGub32TY@kTF4;hxAMsnAr9vV~r-%k$iLR=7%1EV5?}itXqZ%w?DW zVHT9yQkn8;(>1BD3S}2>VCFQ-rJR>^SE=4RKoZI}DX%nA)aWBkek*|jrpAXFGd_Y6 z;8IPLp$4HbvIyS-5OFDw zJhCVsg~F6J56=uld^=>E)zk--0X|pM4(Dnt=CX!LwZ``m&mp@$3K9C^(JZPZ$3142b(A?Aef2Y0!S`9*LMdDz@kHUa;y8En^Qw$n z$v^xMnXIIG^6y5?(%^v3mBcZEBy2_#PXIK}Oul2R%(N>P$rgq)cQTwG`6Bf>$kw3p q%Ae&zAns0_Xr{dGYF@lwA$^l)0id8zRr;3=&s>;yPt9L-YySuJp_hmN literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd72017b858cf9e469a04406af63d77a8f454623 GIT binary patch literal 5035 zcmb7IO^h5z74EA3nVz26AKM#e*Urxbk-(s^W5*#u;MkjN2sx0o6I&2Qi8{SgJ+r;r z(>Dc6};J70K#Y+f(ah8K5uhIxEY&mQ-7`T z*KXTrTd~uxwB7NXYTs+unDIkH*uuGG2q&~R>zgxsW_w0dgnP?q&tj}9JdAna9Qx-) zP1MnULd=L+d`}AVn$esae(0@US-sK_K^|mb-nh9Dbv7C@%(rEdHKKmd3;jqmx~Xi0 zH-dOO$fGn__Pm$UERS~^opiAK?Lm;`VWTV4eq$rg2if;ptzMLGY_BhO(tc~Oo2Rm~ z(Yg_3SlU{T)AiQ*3+orY)9too*bQZvbi%A9!aVHI=B-ZJN&W3SinHaxu6GDA8BM0F zt3kf8Mo2%%wmQ^1dCW1AeG^D%v>C+8-!-}nV&k{mwyAMbSir4!4a;cTIJ&K7Rzd0c z3Ehr8-iE$(OSEnF7(j01tT6WYJ}+3o4{%R~Ss1>VTZOfU5#JUDHe_o}N7>o-Ae55U zE7$jTrTC+b(zcsIMAzJw#G%0PyLK!)JV%EGv-yr|#v zyJ;*!soWq*{B9J7$~(4Iy}T%&M3O-^Rp0N#L6-UcAB@i)ZEd9eu(cBf=?>)DBN5Bi z_@Xl{W(Zh1VD%&&l@NoSWDXmG#bc6Lo}8n@X$qb@MASydaS2GNfFI3)@V2V*mBQG!?-;?XE{9n8T9YexqKT>oL8Q4>!>s8@;-r*z z&G&m@9_3*l0yiU($tsOi0GhTWgQVUafPPOoc_1-c(!rZd&e6IdE^d{k{2HKVjLe-k3gda>^;bt$ZtOEipVZbI*=X^? zK42p!Hw$MI$}8BGAukq=;J7Czw>O=FU`3d>`3U%?D=cB(#)`t+uNKY$W5%1_PhbFY zh&v0^rsHoV)Oholjktnp(&GxNUY$^ z9s-#7Ft~Cq%v%hB0Ji|AZ3zd`bft?)FC1AT36kDbycsDv?!fR7TuP_JB&l(eYa7t` zU1!lMd4U$wE(RMu{P`Yg>IaBhdPrMI12yHJV|Bqc4Oq4bo9BXT-_9B7T#rHPVBe*l z3G&58;XH5bL9-CTrmafO;57*1thf2TP2uc-k19n);6(3m1U0}ew~Zy^s1Koz`j(8R z;k7SL`#r3wz5_q9WNbQuw6%7d-1lU!`d#BzIgSuH<;VpuydYR1jz%v0n)@D;j+i|* ztD0AfD*3rRa|EiRTH#<$Tg<_>yd`6+Y|S5AQPU?Ss85O&wM~!aj_O67))uw>T2aUO zPr!C-M?9aP(;T(x)Y`I++SNNoy>fKL7|j%wVg`Fx3Riz8r}%T>>9$wY$Pa_=M@3Ed z)FNZVv5Je6kf@9ADSXSb{|BqMSU{WZl;K;hhK^x17jqX8GWWT-Z+c$b|E%sieMjG1 zF-z32=6>ulH^vwBdNZ$6fCV)7SdSmTvxucH965vai|6*=k*6hNEZsWRmmPGfGr^h+u{-~^H=;$3tMqjiL#DNZwl?CwCSLeFP~Fp zy1sdrDShT$jH4_^-ZQv#iRchf#~5DtLW}AI2IUi&C&{Sq;!r+`z7@P<)(~vS*;j2XLUV)f&${VP%X(Uh91ZiTyp6agf=SHkic5||F${L97u z@|2!@5V*2&VIjg>9IS_NCR^ycSL(__${2hl^3jz1=Tm85g4V6Yu~9g3vDC2yRz5KZ zc^=b_8^f0$dUHxuuFy(a-ma!H>iJnT43X>tHfQ9sH25e0vNIjfB_-dg(v8y~zwm^r z$gqt34szW7z*jbI>!~Nph(-Zbl2I$`MikyemY3ws`7-UZb@C_|X_VbmY(qN-2o9N~ zBr8=Foi!uUKTg}wHRwxKGs$+}KT-^3ZH3`rdI&A_5>SRl*!^h6MfBfij@8gg3 zb2<7H&M%_)5j6;`5O{;YTLktA{FT6`1gLI|8UV;pWq<fU00P=#ELWvm3uwCUgojYHPnegOt!J`N|1=O|01kqm?CS9Ux=eYyI%HQ3u9!* zZEp|=AGU${?b-1N@Ku=(l=oxQ?BNfjOl8wl)v}60ov*4;{V@$EgU&$rr#hjvd7&R{ zg=Ka-tvGYi4>C&w`suI-%rnqr!Yp3C9{xP1RK!AnDYTYYjm+jRPN|^%YIQsrAaj> zf;ns)lu<$_YHPz&1DW24L?{~ipO69mXJ`!P8X^s|21?CF zxPyoc*(E3?=&y=)Rp09q=gvQ^6O7>;RrgUs9m|_pnzSppXKD>+{4u8ck%GUBQ_EKX zfRizmsd+_0JKz(`QUdgqqlso}_Sr1MUj-=N^&RTkPYYE$DO3VK#X<@Jy-Zm$O>_J` zfEF(sq>hr?ovxV%O7kXdIaR0R5(r&PuF zMcP3vTOskp(Yofi^w#sb5L+()u^}?5rhCdieA>0aE}bh+Q$#p&BJvS{_NmDq5zABU zDn&y1nQ|w?<&n=(pM!i03a{N#DFou~*om|Tm#+9?S$$t3ealV*SmcJaORYmi7v|j) J^Ut~8_&@(1ju!v` literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0a4bb3ac128cd0c9e7f43d79e205e0142980752 GIT binary patch literal 7198 zcmb_h%aa>N8K3TXXf&&p*X#A#4&V$VKopYQKtd9XNes3F=CJ|WA(V)jnpKaqqm^c~ zJuO=+q@oHLY*S?(l?zm*xIqC&4xITDIM0bHPAQJuxWMo0k!E+Dg`_BCPj^puPxsef z|Gwuwzr5^)55F2d{xQS&n{nu8p?wKY_EQwXXc~eE-eXOsYu@C#Hk+odt){JOr|FFGA_Ep=;|Jgy9smO23wD0dIHOMY(Zgyg|Gu+;2rTxvpJ{D~4 zY~GHuI88RUd+GLOB--M$;Y*t`YDY3kT2ZzsA{Dh1&1j{qG$2wo2K(O4oe0~v=hOT+ zD@ap2_>SIsQI;K05htC3j=?^yqSCaNc{l*3`%xnNw5^-%xEDR+hnWxBcGBc|zZE8a zHXIBjtu~NxqB8#uj)0XD6^&Fc-uB+b-5FaAmb=6&q7QA0+Niy-*9$r!k=3Gkiyueh z5}xcPipao?8^vvd|E4g7h1wFfa8TQ#B3#sts0t5tMJ$OLYF8|aI_j!8Ay!a(;-pwb zy(CWI{?_v4%Q))Q#nrV;>_k5TosH)*o@@t&GA3Lx#oK(wvsGpOkSVKcPuPr2*@R7u zz?pEAmDoV(j?Pc`730?W)KpcB0-6so+I1)9MPu)UH;oR>@&p&=U95Y>n9&ML!T?DY zaj6og!<+kqs0DQCAF+aHx`8KKMM3-A@}>+cvA6l2b;J0*AuAYY@SJZvm$O~{ri*`m zW^*U)N1LPJ<|q!+5pdGkymIyCb;6C@6rl>UNM%Q`^H1-hh+7I69K7-|+LS18jPcV) z__YxPaT2Q_*tpnBTVXGIW#izpMlJUM+ayyVK$=@U2tsbF;h-1g^)F^wBnh!srA%dR zku2n{h@ydRRWyz}DzAbQfdVVCysG;MJCcY2llq+?78szu0B>2t%B?KwwI#7go}yXq zVu1sGNKX$eXg`BD%~OM|GLJdjVGinb)Z_KzkJ};w=vT!zXs% zEB%|&-4j~)j4@;MOT0Ni$4{pwS{1^_j884)s%qC0Y{q}~!qiq!P!oH`es-Q26HB($ za<`7zCnna6Cze{C8B^y?sKaj=*NyKO2QAE5nV9E{X=UO}D#1y@Eznizt_G*n+N3gL zZ}BOgx|nyG=5^O6E>X4jG_3*r@vWTQX`GSEI84ryLGBEbcyEaN+l}@!d78Q%NHsJ< z!Qcu6e-KJcNC#1p^EAsVBD$Rnd%fH{PyzV~8HrdzO$C_}=}<|MwPvN4hT?jZ6*sm< zT@Kyz6_qGr zGD#qA7O)aZ(nfljM^WJn2at#8NsMQ-fWg*Tjn`Pstg{+>hIv2*-uQU!C^CxAj>xNn zuQe`SLt}Ia95-<5FHRXD1aIbqor7VT&D$3ZWuX^3nvu>Ox%uJ-p;!Oy+%dz zUVaGedr@Df0J`)Pt|ZWD@Q(-k(j=9=cwqb&GHwx6_iWSYd|DA8 zl1Ip->6*OHpfP&p5P9bzL6B($$Aj*WBpfozyVenc`PR68Eq%l9L&e6e3;y?~4ZApi z9jgeiwza9uopvhwArxIU?DstuAz@eBe#i7Ktd6-)0BP9$|@qnXk&D$DI z_l$e&9&el4!*P{OUy-Yz)ph+%%mMQ&TatW)+{Oh!D4eX7LgVF5FWioL*%3sOV30}7 zKA{PQ|1d}LeRdxRfY#ynam&z2#yi$8v`^7su&8d_4?7VoEi?^;t&_@qyvP^zwt7D; zZs50zE=eXaH$e_MUZwU@D&tO^kck%Z6`G<&l%x}026+YA%Xka#g_q%sFCNSOWAJjw z?<8jSYu?BtG0X{J0+Q>sxi>_?JRh+POBp(U@iNMp_| zNABVF=axl&0mF;a(&6M4D)gbWw)q@hU&k}I9Tq%_8spq0n*pm~@w&5Wj-UFV$14Fx z94i6$N4#jj{jLPuJO3GQ8dw0?1ET;KKnJIJ*V5nvAW6|K=IHi35Smy7wy6zY*4DnP zGjW7V02Nhu6llYj9S{t=RtZ%PArwIsp38fnx>P{5;Fd;)-4C{QZdXhXINk%4jM5I|t;wG=5fa{{I8wr~HDivR&;_E1K z3%h0VJM=!MEzO5zl#^dy=1*~B#zpI@LHqw334sB z>JT{#9xiMY0*uEf8#8qJ}&^74f7~ldhS%Iw_q&?8m zX)jj?5s2@qdKaB|khidE5%6C_T1S2yuaewWxkANB8eIyC{J>SjgU~cOP@>TGxUgPs zVBpVqvKk5l0TVnjMDi}vBPICB@gM=ve*JUMp#=U1)_uSQUVK%)pj#9qofl@p1x5mn zkO&$aBRVKzhSKN5I}$51h8Zr}Rr$TL-<$V)Y6&AXJ*Rdb!GA#eA+^H{{l#6BUSf%* z4tqd7fZHz5eyv*;X!tE=*B`KCMfW)9`4f6hJYZx!+FWNuX>WI)U|hlI$p;)gHBo~@ zXx`)Z%zG9h49c0h_Yr&Jt{84-m4dx45=QsYgVO4rp4W6H6LZ$PXRsPZLG@YMV}cz4 z`Q`D*^YObrO#DG4eH}Y2a^r~LeEjdkwZ#Sr;(f7^d5^vOe~d^5S- z4N9U@Qg4I6OmNT}Tqp$Ug1?m}k?twfj-&*I4o=1|4YTM5qDTKJ5y9qD@FK8U<|nDr zpKTDC3FW=NKRraFHf|**2n(bTOJ_@`Ue8YvG(lmkP5FIF00aG0w)xxUUTqKQ*aHOy zaFTk;ZJo!;ZP0N|z!7zH1Bna@fAHr4WDAOqw8C4$Cx6GI6`sR68-JM2Px-GyHn7am zmgJINA>Ub(OJ-5~&QF6}uJcdlK``&aJHNF$H4*obP~r$P>lfDE*}Zy6ItZ*Fs~28R zI$H7?RuSF}u%yhWNKrt$FD=Nz)#MIT_Z4J3x-{Xy0$s)HKk(>`g~K>h%(2HQW4L$! zg`d1lJDk(f->l3pQ_@|?=c%A5zfc!idy>x7r0ThsLnQt7(1n;dmALt4rJyZqy2xhf zl%=kg3R|qR%Q}Hje-R%$*QmevLNJM=!OIOwhMTKlN#x4h`EzJL~23 zVsc2Ynko8nKZ)1B;K`mtVN@JAt7~u|R*`t7fYE^)yUN#~o{)})qNf*tGx(qlnXUZ8dp<#AP5J5$lZ&Dv-1i`Mn=Jb YMaHN2M=m_hTQgUFV64DPS$9|d4PXDIIsgCw literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6c4d22531816b7721e85752cdb07aa03d9f7468 GIT binary patch literal 7191 zcmb_h%X1vZd7tii?CfB%1VIoaWr?0ximjz3K(_TVMN5=P%3)$d2`NRcjp(Vt_UvMS z-I+yCFS%IlmdXZglB$v&moIjeE44TzmE@R%&;Ao~o|CI`QstPNFZS>2*&P5BY^hRS za(enT-CuwG`=0pr;-Y8Z_rpscgELa4D?PXAGWy2qR(oF4({Pcy^tuDyNjZ9lpjb_bzKrLVR^ z<=;!?PUgpnFXBwecypj4;rCJzt$E(}(oA*t{8rlEd%hoLD)QSh?fF|u^|Qz!C_ z4K~+WX>Yy1hXq?(>v!WUPLuV`Zo0W1iMDttd}Uom?MOyRE6UbIq@tFh8LhOH21LqQ zf6seoE5i2e*)%`S3ewaTzN1z*%CbW$;-pj1(chz0RGM}(4+p?>KT3q3wso}~ccbV0 zF!Mp%R+@anZ-t4U4f=gatMz4^sLa2IBVgr3MMKq%H@#1Ad&WkCu)E=_~Heq8nHUekNQC4CDr8~Sh=GToos}oaIF$!>gjL~g(Y+f;TUwqH#&@4}I zVcy5O*NuHzVL=!m$s#UQ;&kxNUO#F9UHV6?AewID$(B*jK6ktc!%FNIeAl{d{D&bc z7-;aEuf3479o^E!?=P)yrM+l<7>DT)An6cZb51Z zMxLQr?tFnmK1fnCxjU!*99o)}23uwxbGXACyjSrat)6_`77;+dDxL*A8Xvz!MHz}s zIZ(o;<^k^WfbFvh-8rFzA24PNe~Fd@RQz;ej*V%BaB;vVmU2~f+Y@Y`|Lnzytrn)W zv3qcwkU@wA zqzWqKdFrk}8x6WDlLX>s0V|;-ZKRiZ92L%>4|#~5!FWas7;Kf*c#YM}I;*kgm3%$@K7a-Hy&V0tc ziQ93@+xdi`*QjXT%P*jwFM!D{(43U4IoFaA8UduYmnL}4Y(a5sy21w6T+=*yoCQy1 zdr50crb#@Cg3UAQXp zDmHFi_CJ_3?BW1+tRleL)+RD{+Ntb?P;}X#*9#?hj)M}Si@Q1LNH1aZDxTRjtgt#8 zojZQ0aLEx~{*6IbeQeLy~Wh+qeJ-g_E^X zXuRC%hMQ40JBDZy3^Iw?mo>rg8)is;$Q}X#&^r7fZW%hs__g(O?MpNmEUFv#!cGKB z3rz!I>!flI4f(R(R_~|94g7Y|CCMb_Cdfg@8&qFNW!#AqGSNc5PE)jql61n$Ag@4s z8SkTAcp1*<%8Beh0WU}Vj&h-fBS~g+vkP-{3d->=JdHr5!Eo>1R_>^jX3C2+r5YuJ zUL;9{*oDv$TJll)Bq`eY z99^FUKx2!*HnHKu+S-S8#*T0aprU%fs16^tPcWQXB~(3xP6So^1gI_)P%XHnkzx13 zY$xmBIQaeUjS{V7;)tEv%O&QLpR6q>G?U_FIRyoC?8g-Lp+uk6hVbBRlQ;dbd4L%g zj2Vh8EhBJj-hx+b3(`cnok|hOhLcx`h+v@NF8rFLg~+Tht9B@5xR+Ow=w3LCp|1zf zX|KZ65Xy6lzAAuRqFItQxre8KF&n+{8TWitwVj0I9InuS6sbK>?dM-D`9o@lcKv@y zv%HG$D9kj^(q!hW)pBA~#spFy9VGkfI+ZN>^Uc5PxDV7=+B_;HLUv)5&R^Iqo~uG z9iGF|9Bp6uoJZL}fQ%qC342ixUd*y%q$!gRonS$P3s{ewB)KD#)RghWsad2lSc1UBjdZax+c@Vt>kE z8*h zxzJ>AF6bI`QVeiG7-a>va**~w4Q>RIWIBjId{@=G=){A(gH?-w|0dEp@~_d9U#4P- ziqkZ@5ES`=8;A#?X>_1Oq3uawyiid=Ln-Wg zWIeRD;b!bq<%_yTLDEHGCR|`7;0THADb3MA5i^uNAKFN)>@&=8QLoA$l>OeU-%|@1 zsp&bjhY0=y61GJOBlQ>eQF;k%Lx(-09ym5TJo^`2vp~bYfx7x5mMrO>3VMEno>Pw) zS&uf?nQPkHoh25RFnanCM^8;Sa0tx@{DJwvLWDuNZTBH!Z+&mhbeV#^DH28x(Sy?3 zK0AB6h(Ao+?dtR#@kZa^dD>%)9Rc~;Xz2O)-5Ml*Ka#$V9p*W4L~uU-cjCJdVnT{4 zGCxE{IfG(|{F%S)KO6ewp;Mlj-0lV?&nc<5Mqnm5=nXCx0(IHnNRvqS6lzCOfJX-5HoU_rB!~{x$9WjJuqP8TLlncdSnq0Dt+JU|Va=A`IT?D~=8`0#Q z<%x;lk0cXE*tdRe?VjJQm!yNh3Q~LFt)!#HuVEE2-Y%Auii$o8sQ86BS-79vf&RXZ zw8u70I5a~y(EL9m0Wrp$Qoa=AzWK9c6icAjZvr3rFoPS_)D+G?;q7C$c zO9dno^vyF*JsPK@!mQEGdU?H=yh}~Z3Vo$tLG!=yWG|pFDh~YA75EOz$T?H2=)i+r z<}1)l$V5X&)lB4~IrNqN46B=xzHY4Ml^_slD+uIYp)H@Lg6PU^gtPs5r&&vHbu5 literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/panoptic_eval.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/panoptic_eval.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f011b0b88ed8100cdf362fe3682d349a156ea783 GIT binary patch literal 1760 zcmaKsOK%%D5XVXGL%S>ak<@VWE>N@wJs52g)ae2#v0l>hZnO`R zR4ohZlY9;OIr@Qm;kBoJg`PS?+lgEUrNH5c%jb{(nJ?GYlI-md*`EO+f030k0RJge z{Su5If=)?#-qMu1|E%;<|6~(<+0-+9RGfRXZ~>$G*$rf%6oq9)Xbr_f47T3nsiXn!<`AI3uYq zLeYaZ5Rr(X4MhU05A2N}v$|DlwE5ty10bC+=pecV9TI>3spY`)6tG_ z(lM~qz&)2lJv4q9lws#BsYbtt9wnUzW1Ztw<&itE>J|nLm zpE5%XErDxCO21=*-X=>X*pmIeed={y8FZ`+1@J|}d!V|X$aC_F9EE$p*(boBK7t41 zJxAQ%wJ}oQbyi8sj`!1~ttVU?Rn(JVWc{X z&{rEc4t+~oOAj2!bMp~)hO-v1$&bO;5f}nI?$HNqaqBvf^PxG#VNj3DVuLOJ_BIDI`0wm9BspmQHAVMTb}B{OnC6=lutl3Qmr05-N618t)w?A zsU}ja_NlwD!hi2|U!T%aA1 zfO;_3?AkuWNMkY=LW|86Mk5!g^mxcr2&YM1tX)LyaNY4%_|y*cdmX7;*oPNUs0MW# z3}_3ZJ`l62)d~YaL0%OK%)S5bo}IkJk?p%jAuO;IKLD<`yYJ$O=S=iv>i&VI*jcr`zk9>^$7vi|y6; zfZ`Jp;xDX^{2(rU<-}j$L{-~P);OX^Q(aTjkLs_#ub!{2rUcqAkAKbo3<&uPKdxE~ z&NjUIdoY3sIwjdfPc!PiS?y&$B_|v3#il_WW+5fJq}lX|j9#Z|n-Y9}APsv(z9PaC z{uvRzjNf=!BIrvp4E~0-$D56lGz` zw$Vf1`dUu+ZJaj^-!G@qrp|Nws8z`S529{C_s?j2(AIK%T;}aDOr4B{G-@nzlWS@8 z_!=6c?!?BNmrZFnUqd9&5E{Z0(*aGPpWnNI>F8=o!EiVz_TZn-!nn)l5-~58}|Fwsn>gppl6Fv0IrDaF7WMJ@*R0a4#Qo**C$of)2D0kd6U=DvZK8$?V1VKMwQKE7+b&XWTO&DKw&!8uiCP4@Yvu;l}1{xYdeLf z9QtY<$Dwa&XX(Df@50>1U^wdlHu)j=8pS~Xj|cP-o8P&K$i>it;xK5}*!2VghAex&Ckd zOJEPym>s(U+GtJgq%+@G!f50aN{@z2g^*3=GsHJ_Gu#e1Kb+;$i$7`;_kM`xBQ&3& zd6$p#PzMUjB%a-g0Fp`KD2`lon5vI)1YWUCu-J|2QWw@o>(I=$y36vLcjX8xFjQqI wOAau*@^#&{%Dhkt3U%1-RkAQ{;rHcX literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..763db09f26c435b163b41b135013ab5df796043a GIT binary patch literal 9336 zcmbVSON=DRS$O@XeE8+?-4`tDpRFT57w?-$NkdsyWHqfwMRrd$ zl|1dHEl;QE$kXk5O<&$hU9TH7%XoKsmHtw5NwKHftM+Tn8s5Fge_%E1Q7H=WJQe+1 zRE{bSRC766imIqN{VP^fi|P-os2;COo#q*oPescpFH3n9<yp+$Qd?C7s@1R18Rg4TzKHU3(eo%jFXc-per8RlwXLhpnOHj&qekvYyIlsXDhdLI7qgKx}P+{LDY!CG;DNMk)3_QEu7 zJ8X;JDWnq8O{C<u8#GjN>r|XXWacRe% zsxjnv!;0+4iQFA$5=7ou?fLqrsP`^dlQPpQvGL|d{zH3G8Jk*f?2eIk+^ITm zO_uPzG+yckQ9TNxVCsx3Q+u+6nldH(Q1Ly227S|p%;mZ5T(F9|^1SZ6)vch-68cnS z#8XjqYLlWH*0_qNgJ;lPioB_UCe=vsM@qknFSQF8nR9?PtnnP*Jp7gTCi9`qtanX% z%X&Xj$y>!Ze{Xnz5x-}+fHf_fHOVSL)6=bE&y`i!liFa_(AKR*rK##d>!)f`+kbu> zyk&i`HmQ%*xDH-`dcNJTK2mr8{A<=2(%Yt1o!-jaq;nW&elNTe_ma#FbsT1;PSV!H z`%&gWVd4Zj;zjAfD1HPzupZgSG~ZC~^g9DSqE7!24LbAsVX~Jz`XBuCG8-~mKb5I> zG8bcxkPk-(nU@ZAdpGmbc#sTrRt}Ta?6tJr8;0rY*L9V{m2{l!hNC$1+P$!k-XlGX zGCv%R;z5)-VXv2=Lztp7N`nK;bS9>N6e^Q*fT$D_=X4nMT?OQ`$!^=rGsetd2J-nIQsINZl% zJJ;TN`}X_SqBzypXr)PuiF!6;=*JdnrsB) zgei`oItr!;HJ+)R7;EJ1xZCyxD+T02%^WDGLsMLpnla3oi8be>FGqo)`!eS5q2yn% zV9;fO03Rj+0~2MdYzc?;63EjPW};3+98I{H>x)sEl0J>v%-h}Wbw--nrOz;1Wwypo zF2Ah%`V*{aAZylGQ%G5Vk{_-zqijW7CVQF1Pa*ppQo<1};WN1(s2eJP99C>;(dNwj zkw)^E1SeNOJKP3b)&upsa2?+UG!p6_l)+~gJm1@1tVjnGk_^PN*Qb;D{ql~e~D~t&D7EcQ_A{wYNZOu1mzP3 zcqoH~MHvhkdzXM?tC)PxMSBNq0}7+;UouRS{(DMCM7TXWNg)ac7I~TEIpZqc|L85)e}-fz&#vb){C=EK5>b>mCq*C}l9p0b*nhoM`6h;k`k$Ud}24 zG6Ew;aeo2pooi$BG+?te)SaEqAnX~mIZfPIsLJf&eWcxEYMWgItVy5N{Z6{u3X@UX zPFtEGz^{?Fmyubb$IHNvWwmNIM4w$~v`p7)mZ9mp0uwWrQ4)WXj=TgKqU88~21(itRaIB3V2 z+v_CYTzYTRi`O0fI)w}9>L0}{oGgZi;dY7!PTFSZ$r3B#+w$j(ocHPPBjaM_3WPT!IIwqBm%Qtr7 zLA*cGZw@&=f!VbVFC3?;Xk`$I3oy$ix@4t63M%OY29R&i>Mm2)oZU&8lh9YaAK3J>XvMkU#jJfUk* zx|>x^VK?sV?547WW_8wU?j@T`vnmD?27U#IDaBl(0>EED#k)w!1~SX3tk(Uys?r78 zmqld>#%r^;syyK$m95^eHTxFYNzE*x)o-C-+#lc>T1Y2{900ZL0)%1h0>ZDrjR^?n zk*q|oh)PA1J!-PJFvNs#c?8pMFjDQ75C>zw0l1ruEzjBL`~vvH9D)`8mPpVjKZ%?> zy7j=gH)j+^M}#KwV#~}P46@4hy+NCyeb`%f=c3C>=oEDZJIyLza*-WVN&OCn`8HBQ zv$M!j{@E9izo_IZji~rhWjB!oKw=9UG0SYiuxWC{w^>a$@?aHyR(T&gkYWEOVyYqr zVSlEZ9E1KH5MZ?o0EiuNS?e|;&bYU2v`_5%h;g@Cok1sUwf>Mpa4bt5o|#Xzk$+zh zJcpGz349bh{4HiitOH@gML?ej8hGgLKukxeQ>F?+U z-t-I19;XwQ(rdN8hmQ&8M+A4AQqzLYB51V+VLxuQvP!E3v%c5kd$rZNdoS#nFPdhc z-(xnPt6+=}{0I0bb3ulV)GSEP!RJjjc@^6g_1kMCm-LUx<(1cQjoB}#2K^JG(o%?h-PZJQBA9|6La2u%DzJqYWBXm~D`D9T#k?tlFK40N zFY4X2)D5}_a~LZi1VWgWBK%=wkt@v-b`hfQ*pJ`|S8rBR#cY`;$S6zA{24HQ1s_C2 zngmIxjxbf&P%K~`4xFT8q6>e5&Iuz|OP$Bx%1NBb)vT<4om0Kf>=v`vn31=c^WK-< z&Ac!1OdxZkPCI?FqTNROKSr7<+A_i{lok|?vfpGAnp>`D5(u9F6E|@-5$pdCOc-v_ zwocNQKP3UAga9$0q|lT?I~a8qo`LoSov>zcC&F#K;pBpG45aU!2d2*=$&%DLWB*8< zi3x29xxKLuZ$eP9w{}Peew#oKAy`;{9W_Ss zH<0%fJLs~!_2TXHIA^%NKSN3?$grWo-zxr&aVXaq@3_YNYz+DUL;jG1bBTgMtXWVE zLc=Cf@;tIaHA;d$3O)qh1HQzKd_kO5W945&(0e$<5mxUV?A`tiJ@Na==bzfGWC~zq zIJ}LX$9cj)=I7up&_{v&6;ct%R@C8>2+YN%bj+cjQF;;j`~_z+(yKU{HEpEg$K`G! z$#M(n`Gxf0H6T9)`6-cK)o}8Ci9ojsi{mhyL^yW^FGqI@$x7;ci1-PxzO?IeV1c#|2K3 z3pbKS5)e0b1aKYPB62fob=|3J2=+wSncvCMI=cD5o$73i{X2d8Nr8iba=-qCstQ^awVtN zHo_1KS39^G0Mz3K@`7ddG?9a|abF*z2hOjslf{`i0vKHJaF@#+uIzHT=PgI>L;I4s z!a-}CTIB}mZ^2*B7q|qfNq zITg5?H47@o^=z^UX?`37;eX|n3}-7=)A-*YE8QISN5dq3`oO6&_i%aa^xK&4BxUmz zh#^%#!R6URg`yRT&IK`vp3!&hU#7w1mUsmxqBCD*4#V#TxFODUK@RRXb?#LQp=IjC z{b!lvP@@^o;>6+zNk%LSfLM_9COVKHhI6=ARQp#^vO(TOs(EC8n*VBm8#cIAMC2^p zT_u^3Yreq1y{(Vci{vbrO@E?AKTH_Tb-n^Y3Hb8=@_0To;lfb~xBa&WrxxuCXOUasP=vb=<} KyzHTT=6?Y|gnO<4 literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..55f3cbecf4b2d8b549aa7a097884fd136156f530 GIT binary patch literal 10332 zcmbtaTWlQHd7d+u-Pv7|q9{rtCE2EA#n$>F*^Y`jjbO8p;?!x%xR#ovTL+8L&X8Pk zZ#pxyNM<)cOgT3|snMr2C<^MK1@h7YDbR-&=wllceQ1FqD5iZdP@sig^XP}B-}j%L zS?-ETsb-0DX8tqh`rrTm@N?7CWedM=z3~0uk1tu)e-g-kE?$;V;;$*o3ay$Is?ct! znv$nov*qd39C^BJuU1f|PqFRUma^J@tt2f@%Wjuz<#wf35!h`_wWn*-^6s@}+Q(|g zh9hTd$HPKcyl2(sLO(3wc_O?NmczMywcs4u^ z__)9efak&!fKLd#2zWj`3HYSIX8@lHPXj(J@L9kM;UeHgfzO3!!n0WCDYTyppF;an z(*88yr^9CeKO^unfS1DafX@rO1o%SuEZ}DaJ`ea}_$k1j68Hk(=fdX!KQHjJp?%Y; zR(C&BzNLdsyxP_6cq!;$&L9bvf~~c7)JcM*+3hUh+tTfBvcA-82W!#Ni%Yj}cegK> zACgonRXg?GW_wza#|<6at*UemUFzFA+dK87TW@SPcJ8jDwbJY)k?wU{K@v3_j>qp9 zN)hEcO8l>&c4BR)zO}0Q_Mh56um(){AuQat2jxCm_r1Q?cUjil2Wr6@ zRM1xJSN8p|2>SaEoPPO%J*f068>O%mmhUURhv3E!?Qr;9br!9>tz^EjZNb_A--J^b zGc}xX`hm?IU9;=rbHW zBctWKAF238KF>cH9AL&D8Z3;?#8YlyPe(h)rYong2h*K-v$y@}u<}6d&#)d$Z$I7l zZ&+VHHNXk_*m-{%1#j1^kJO!?eAViMdaLBLV=Ji}H+Q47&wI zZkT#tm?#F1cww^Biyr<0zlSzhLW3hLv^(u)=VA69Hro%WAyTg$#G7&TSgPJiUCh)& z-R!xq2@O>*lj+rfMTq>#4e%7FL^4 zD~vU@UFx@k?K&|r8X-s5KEJw~QPfL48SrCEzl63=Uc9p2ZAVwOn?ZLQE3I7#qeNe! zoW@ZSUrBQLz1-W;^B8d)CBBNvQnR+NrkpvujHj>W)hYZXa=rG>Ccr3Mjd7{t&cKC$Iuy_lq6oq3fl(9$4@^5` zG7-(tTXR?KMJsXlJ*ZiSg@QLPEqw?t3tLXhSk3~mF3OzW2UJ|N_94C^@I?p!#7S7o zUlGjeX>3nZn+VH1q<+y;tmaTvi~0l}skgq`YW6hGrBAXt#cF|{Tz;t*^m(>iKvkP& zOHNb$JU=|gij)-MnCfMMS5RF;iAe%WuuGi#>YDO_{W)8DG|0C6WonPxj9&+1!;HYh z-BaI#Dftf6AJy5t60qw6(>$%S=M3xw%CHXziV0#LTARN99Pvy{Bu)s--UB}S9!6l) zo_)jGQ+sMq=odh%;=l*3JfW4r>rV_`2PG)P!oH|Is=gr=%Kb81p*(#SLoF!JGDm?n zC85pF0F^**C`b01x8^M2Q!$*?@#w5#f@(~c7m$N}8P6Rnhbjhrmx0i&=UGTXJt+V?A6f2b6q7IrS(C zq#gw|rs_*Ti9{tfsAcPOAUiFi`VjW@ruALW7C5!{oSSHcg|Sye+N%yQ6E{0?5_B3- z>b9COkd|!qT2a-}zlK>y*s3c1Ivco_rVyuow#^2hm|DV8Git`(U3lDi8nj9LWxJ2@ z9#ehvt*@T}_d!+i9D52ixbCd=6?mt2z>nB7=&bas=!;$3mtRR$oEBoC&;&s~DP3EO zI?;Adzp>5vsh?eIcX4u8c`s2>4}XeoVpz*^j6<&78%VN-6C`9A8ev@A6MiFLK{MT%5c@Agi7brliA=;v_ zA~ZC%Pb{98H!*v@F?&ug2oI1=D8^2y-RY#|)vZp0u71#}x+CJHMGOj?owam|FB#D$ zI8%QS)BKPanMGw~NVU%ZnU|D&r7Yz?a_%yUsF27)Kxdg?4ndD2HqWZLp$Fk8q~&)J zk#yVF;aTM#3CFVt5om{@1gyGI0U}O37NQb2j#{gR|3thG$auZp>@<^l{m;=;B*{{{ zCq_$M&c2Tkp2U|msrneAzz$>O4Tr>rkpK(OLK~s>QgusHQKn@olwP2Nc5F67?*LeS zhaa70drdqI{Vc!<#HN0|4q0i{>p#ZFnCruzo7}1CL34rbouD1n>uI@OhhT5D_&!yy z-`NUU=8OJaF7tb=MmWh4BOCrRe3Z2y434ynQP08WWp<%7Wz>tD2x0i2fcZn`o?CMx z4-8z0ilK`(dL(WQAr&y=r!xl4H=$2RRnW<&nG*d%`<0`d3*}@cXBjdB$yOaKW6*5t z8op=_gl;EFM-2x~D0+ZL|4Wo1A`5Ew^!Q$KvR7?=i&I@dmG6gK#c~}bUO{DKe*-aP zUv1Du^Y-YJIb-dGyxmQT8~#2V9eN9Jg5bU-aHHs3WK2^*EFt`ks02?Kdvh8IR6Nm* za@4diRE_Uq8$u#Ye@7g5xUYhS`~V||;4mH&Uid2>iF(aa3;3Hm3^N&#t!3yA*{~q~Di$+zFHO)LxfW(+GLvcm!#8P# zp=x~ylm#%B*O&JMQFKY(TG47Ui5Y5-ze9J@V8!DfJd3iMM#zbS*scf|u8#d(}2H^>(N9E=p=xy1PZ-V-bZ zV>hb~U(48wg8mHp>fc}`-t5e8#&M+S`g~q%J^pl0)#*bh&Wy3~3#gSl;!Kk(2?zv{eo)9}NG}gP#Mx#PveVK!g z;6s&+-U7qsOu^wS!W85XQK+LhD=jjagcD^_w}n2F2cXm;+mRc_xqO{Rg^mK33mp+b zPa07#)#K2w&LKBfmdxZ)2$B6^A_m@(n8)|2TL1 zz2tw??%2*VGx>zdGQn)WKpA;AY#Ae(gN9{xxqvFSkMwJNiH&I)CC6LyB%&&%LkoE` zY9T|n`$2gLKZE7g&Jto^w8l%fqvUQBb(ThV1+U6w-w1x~B^k+V-xBUXZU?uUt!A>b z)RXJKQ9m>8(yL1^FZHSu-(Hg`M&6ZJ9GrvCoRyewH>@PKnb}mUo@c|Hv~pmrgTQOz z@(d$q%_NxKVYR_(LT<=*xYW%ESU6ys4W1t5J|dp`6n1|OFJr#gnZ70AJ~$EHn6=Mr zuDsJ~4p~k%lJDzC`hI@2N93e(%bf2q<1?l2Vn>MW#)Gf4Sg(iOM!o)j#P#oEFz&() zI(*l)VW!zO#(5;rGKySZj5rTB5)yJj#N5=s)x9MbY7L@`KR#Kmqr?nUCHz%LY2^;9 zZ8A_LTm+jdxFezFEceSjA^19;Eeo<`C~+rn@%V7_8<^uvHk+?B;RYwpe)-9dpeUz} zXnkG>NHMRC<$j5VFQdeBsN@bCNcaK0&=Gg*ckxXRP^EA^E#`m(Roa z`FwT+Amr58E)HkrF{*%NA#?|+qVzYhizAWZ;xAh{qsJ4fLZaXq&P$m?=G76TjFE*0 zSVkES^ms)^hAiZP0=0iJEv?``7=jku_c7iiZLf2ZSyTtrP8Jd%QP_?GT<*q6m|^1s z^woQ;4$&*sY}XdG%3Qu|PK8mFLi52Q8{nivu7E{@RYS@Q-V%63^28VEtH`a9z3<|J z4zV9{#Q63ha=!4HaLpPzxRWaAU5r5182KaIRpBlP*R0GfF|Q=KC1#pRp?lvxYpzkz zo1V~6AU!y%IzQmU|G_o8_#bdvE7&1CmlJrTrA~&YX%Z~t5R6YVH+(WXKaO9+YgD;U$Zg-k{FgGiVHH=i zrNL#hpN*Ggl=vd5N7Wy$VkIrj>(!y@_0!_@ZoAiwqoeyw<&kmA+@|khy-Cc_eqG~! zxhu%~>~Nu+$8sJg8CZnHi02K7AN zg=ncdH0ARqw?k%T2#FNofh`%bjOoJMk9z|HxS#q{uOq`q#|e#T#Vn s_$n$lznYuF{fE%T@n7_hmw(%zo-NHz;jcVfnXUMXcpjhi(01Z~0b7hdng9R* literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/coco.py b/PyTorch/contrib/cv/detection/DETR/datasets/coco.py new file mode 100644 index 0000000000..e94e8d5026 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/datasets/coco.py @@ -0,0 +1,166 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +COCO dataset which returns image_id for evaluation. + +Mostly copy-paste from https://github.com/pytorch/vision/blob/13b35ff/references/detection/coco_utils.py +""" +from pathlib import Path + +import torch +import torch.utils.data +import torchvision +from pycocotools import mask as coco_mask + +import datasets.transforms as T + + +class CocoDetection(torchvision.datasets.CocoDetection): + def __init__(self, img_folder, ann_file, transforms, return_masks): + super(CocoDetection, self).__init__(img_folder, ann_file) + self._transforms = transforms + self.prepare = ConvertCocoPolysToMask(return_masks) + + def __getitem__(self, idx): + img, target = super(CocoDetection, self).__getitem__(idx) + image_id = self.ids[idx] + target = {'image_id': image_id, 'annotations': target} + img, target = self.prepare(img, target) + if self._transforms is not None: + img, target = self._transforms(img, target) + return img, target + + +def convert_coco_poly_to_mask(segmentations, height, width): + masks = [] + for polygons in segmentations: + rles = coco_mask.frPyObjects(polygons, height, width) + mask = coco_mask.decode(rles) + if len(mask.shape) < 3: + mask = mask[..., None] + mask = torch.as_tensor(mask, dtype=torch.uint8) + mask = mask.any(dim=2) + masks.append(mask) + if masks: + masks = torch.stack(masks, dim=0) + else: + masks = torch.zeros((0, height, width), dtype=torch.uint8) + return masks + + +class ConvertCocoPolysToMask(object): + def __init__(self, return_masks=False): + self.return_masks = return_masks + + def __call__(self, image, target): + w, h = image.size + + image_id = target["image_id"] + image_id = torch.tensor([image_id]) + + anno = target["annotations"] + + anno = [obj for obj in anno if 'iscrowd' not in obj or obj['iscrowd'] == 0] + + boxes = [obj["bbox"] for obj in anno] + # print(boxes) + # guard against no boxes via resizing + boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) + boxes[:, 2:] += boxes[:, :2] + boxes[:, 0::2].clamp_(min=0, max=w) + boxes[:, 1::2].clamp_(min=0, max=h) + + classes = [obj["category_id"] for obj in anno] + classes = torch.tensor(classes, dtype=torch.int64) + + if self.return_masks: + segmentations = [obj["segmentation"] for obj in anno] + masks = convert_coco_poly_to_mask(segmentations, h, w) + + keypoints = None + if anno and "keypoints" in anno[0]: + keypoints = [obj["keypoints"] for obj in anno] + keypoints = torch.as_tensor(keypoints, dtype=torch.float32) + num_keypoints = keypoints.shape[0] + if num_keypoints: + keypoints = keypoints.view(num_keypoints, -1, 3) + + keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0]) + boxes = boxes[keep] + classes = classes[keep] + if self.return_masks: + masks = masks[keep] + if keypoints is not None: + keypoints = keypoints[keep] + + target = {} + target["boxes"] = boxes + target["labels"] = classes + if self.return_masks: + target["masks"] = masks + target["image_id"] = image_id + if keypoints is not None: + target["keypoints"] = keypoints + + # for conversion to coco api + area = torch.tensor([obj["area"] for obj in anno]) + iscrowd = torch.tensor([obj["iscrowd"] if "iscrowd" in obj else 0 for obj in anno]) + target["area"] = area[keep] + target["iscrowd"] = iscrowd[keep] + + target["orig_size"] = torch.as_tensor([int(h), int(w)]) + target["size"] = torch.as_tensor([int(h), int(w)]) + + return image, target + + +def make_coco_transforms(image_set): + + normalize = T.Compose([ + T.ToTensor(), + T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) + ]) + + scales = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800] + + if image_set == 'train': + return T.Compose([ + T.RandomHorizontalFlip(), + T.RandomSelect( + # T.RandomResize(scales, max_size=1333), + T.pad_resize(), + T.Compose([ + # T.RandomResize([400, 500, 600]), + T.pad_resize(), + T.RandomSizeCrop(384, 600), + # T.RandomResize(scales, max_size=1333), + T.pad_resize(), + ]) + ), + normalize, + ]) + + if image_set == 'val': + return T.Compose([ + + T.pad_resize(), + # T.RandomResize(sizes=(640,640),), + # T.RandomResize([800], max_size=1333), + + normalize, + ]) + + raise ValueError(f'unknown {image_set}') + + +def build(image_set, args): + root = Path(args.coco_path) + assert root.exists(), f'provided COCO path {root} does not exist' + mode = 'instances' + PATHS = { + "train": (root / "train2017", root / "annotations" / f'{mode}_train2017.json'), + "val": (root / "val2017", root / "annotations" / f'{mode}_val2017.json'), + } + + img_folder, ann_file = PATHS[image_set] + dataset = CocoDetection(img_folder, ann_file, transforms=make_coco_transforms(image_set), return_masks=args.masks) + return dataset diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/coco_eval.py b/PyTorch/contrib/cv/detection/DETR/datasets/coco_eval.py new file mode 100644 index 0000000000..9487c08fd6 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/datasets/coco_eval.py @@ -0,0 +1,257 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +COCO evaluator that works in distributed mode. + +Mostly copy-paste from https://github.com/pytorch/vision/blob/edfd5a7/references/detection/coco_eval.py +The difference is that there is less copy-pasting from pycocotools +in the end of the file, as python3 can suppress prints with contextlib +""" +import os +import contextlib +import copy +import numpy as np +import torch + +from pycocotools.cocoeval import COCOeval +from pycocotools.coco import COCO +import pycocotools.mask as mask_util + +from util.misc import all_gather + + +class CocoEvaluator(object): + def __init__(self, coco_gt, iou_types): + assert isinstance(iou_types, (list, tuple)) + coco_gt = copy.deepcopy(coco_gt) + self.coco_gt = coco_gt + + self.iou_types = iou_types + self.coco_eval = {} + for iou_type in iou_types: + self.coco_eval[iou_type] = COCOeval(coco_gt, iouType=iou_type) + + self.img_ids = [] + self.eval_imgs = {k: [] for k in iou_types} + + def update(self, predictions): + img_ids = list(np.unique(list(predictions.keys()))) + self.img_ids.extend(img_ids) + + for iou_type in self.iou_types: + results = self.prepare(predictions, iou_type) + + # suppress pycocotools prints + with open(os.devnull, 'w') as devnull: + with contextlib.redirect_stdout(devnull): + coco_dt = COCO.loadRes(self.coco_gt, results) if results else COCO() + coco_eval = self.coco_eval[iou_type] + + coco_eval.cocoDt = coco_dt + coco_eval.params.imgIds = list(img_ids) + img_ids, eval_imgs = evaluate(coco_eval) + + self.eval_imgs[iou_type].append(eval_imgs) + + def synchronize_between_processes(self): + for iou_type in self.iou_types: + self.eval_imgs[iou_type] = np.concatenate(self.eval_imgs[iou_type], 2) + create_common_coco_eval(self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type]) + + def accumulate(self): + for coco_eval in self.coco_eval.values(): + coco_eval.accumulate() + + def summarize(self): + for iou_type, coco_eval in self.coco_eval.items(): + print("IoU metric: {}".format(iou_type)) + coco_eval.summarize() + + def prepare(self, predictions, iou_type): + if iou_type == "bbox": + return self.prepare_for_coco_detection(predictions) + elif iou_type == "segm": + return self.prepare_for_coco_segmentation(predictions) + elif iou_type == "keypoints": + return self.prepare_for_coco_keypoint(predictions) + else: + raise ValueError("Unknown iou type {}".format(iou_type)) + + def prepare_for_coco_detection(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + boxes = prediction["boxes"] + boxes = convert_to_xywh(boxes).tolist() + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "bbox": box, + "score": scores[k], + } + for k, box in enumerate(boxes) + ] + ) + return coco_results + + def prepare_for_coco_segmentation(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + scores = prediction["scores"] + labels = prediction["labels"] + masks = prediction["masks"] + + masks = masks > 0.5 + + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + + rles = [ + mask_util.encode(np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F"))[0] + for mask in masks + ] + for rle in rles: + rle["counts"] = rle["counts"].decode("utf-8") + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "segmentation": rle, + "score": scores[k], + } + for k, rle in enumerate(rles) + ] + ) + return coco_results + + def prepare_for_coco_keypoint(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + boxes = prediction["boxes"] + boxes = convert_to_xywh(boxes).tolist() + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + keypoints = prediction["keypoints"] + keypoints = keypoints.flatten(start_dim=1).tolist() + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + 'keypoints': keypoint, + "score": scores[k], + } + for k, keypoint in enumerate(keypoints) + ] + ) + return coco_results + + +def convert_to_xywh(boxes): + xmin, ymin, xmax, ymax = boxes.unbind(1) + return torch.stack((xmin, ymin, xmax - xmin, ymax - ymin), dim=1) + + +def merge(img_ids, eval_imgs): + all_img_ids = all_gather(img_ids) + all_eval_imgs = all_gather(eval_imgs) + + merged_img_ids = [] + for p in all_img_ids: + merged_img_ids.extend(p) + + merged_eval_imgs = [] + for p in all_eval_imgs: + merged_eval_imgs.append(p) + + merged_img_ids = np.array(merged_img_ids) + merged_eval_imgs = np.concatenate(merged_eval_imgs, 2) + + # keep only unique (and in sorted order) images + merged_img_ids, idx = np.unique(merged_img_ids, return_index=True) + merged_eval_imgs = merged_eval_imgs[..., idx] + + return merged_img_ids, merged_eval_imgs + + +def create_common_coco_eval(coco_eval, img_ids, eval_imgs): + img_ids, eval_imgs = merge(img_ids, eval_imgs) + img_ids = list(img_ids) + eval_imgs = list(eval_imgs.flatten()) + + coco_eval.evalImgs = eval_imgs + coco_eval.params.imgIds = img_ids + coco_eval._paramsEval = copy.deepcopy(coco_eval.params) + + +################################################################# +# From pycocotools, just removed the prints and fixed +# a Python3 bug about unicode not defined +################################################################# + + +def evaluate(self): + ''' + Run per image evaluation on given images and store results (a list of dict) in self.evalImgs + :return: None + ''' + # tic = time.time() + # print('Running per image evaluation...') + p = self.params + # add backward compatibility if useSegm is specified in params + if p.useSegm is not None: + p.iouType = 'segm' if p.useSegm == 1 else 'bbox' + print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) + # print('Evaluate annotation type *{}*'.format(p.iouType)) + p.imgIds = list(np.unique(p.imgIds)) + if p.useCats: + p.catIds = list(np.unique(p.catIds)) + p.maxDets = sorted(p.maxDets) + self.params = p + + self._prepare() + # loop through images, area range, max detection number + catIds = p.catIds if p.useCats else [-1] + + if p.iouType == 'segm' or p.iouType == 'bbox': + computeIoU = self.computeIoU + elif p.iouType == 'keypoints': + computeIoU = self.computeOks + self.ious = { + (imgId, catId): computeIoU(imgId, catId) + for imgId in p.imgIds + for catId in catIds} + + evaluateImg = self.evaluateImg + maxDet = p.maxDets[-1] + evalImgs = [ + evaluateImg(imgId, catId, areaRng, maxDet) + for catId in catIds + for areaRng in p.areaRng + for imgId in p.imgIds + ] + # this is NOT in the pycocotools code, but could be done outside + evalImgs = np.asarray(evalImgs).reshape(len(catIds), len(p.areaRng), len(p.imgIds)) + self._paramsEval = copy.deepcopy(self.params) + # toc = time.time() + # print('DONE (t={:0.2f}s).'.format(toc-tic)) + return p.imgIds, evalImgs + +################################################################# +# end of straight copy from pycocotools, just removing the prints +################################################################# diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/coco_panoptic.py b/PyTorch/contrib/cv/detection/DETR/datasets/coco_panoptic.py new file mode 100644 index 0000000000..b24f615c2f --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/datasets/coco_panoptic.py @@ -0,0 +1,99 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import json +from pathlib import Path + +import numpy as np +import torch +from PIL import Image + +from panopticapi.utils import rgb2id +from util.box_ops import masks_to_boxes + +from .coco import make_coco_transforms + + +class CocoPanoptic: + def __init__(self, img_folder, ann_folder, ann_file, transforms=None, return_masks=True): + with open(ann_file, 'r') as f: + self.coco = json.load(f) + + # sort 'images' field so that they are aligned with 'annotations' + # i.e., in alphabetical order + self.coco['images'] = sorted(self.coco['images'], key=lambda x: x['id']) + # sanity check + if "annotations" in self.coco: + for img, ann in zip(self.coco['images'], self.coco['annotations']): + assert img['file_name'][:-4] == ann['file_name'][:-4] + + self.img_folder = img_folder + self.ann_folder = ann_folder + self.ann_file = ann_file + self.transforms = transforms + self.return_masks = return_masks + + def __getitem__(self, idx): + ann_info = self.coco['annotations'][idx] if "annotations" in self.coco else self.coco['images'][idx] + img_path = Path(self.img_folder) / ann_info['file_name'].replace('.png', '.jpg') + ann_path = Path(self.ann_folder) / ann_info['file_name'] + + img = Image.open(img_path).convert('RGB') + w, h = img.size + if "segments_info" in ann_info: + masks = np.asarray(Image.open(ann_path), dtype=np.uint32) + masks = rgb2id(masks) + + ids = np.array([ann['id'] for ann in ann_info['segments_info']]) + masks = masks == ids[:, None, None] + + masks = torch.as_tensor(masks, dtype=torch.uint8) + labels = torch.tensor([ann['category_id'] for ann in ann_info['segments_info']], dtype=torch.int64) + + target = {} + target['image_id'] = torch.tensor([ann_info['image_id'] if "image_id" in ann_info else ann_info["id"]]) + if self.return_masks: + target['masks'] = masks + target['labels'] = labels + + target["boxes"] = masks_to_boxes(masks) + + target['size'] = torch.as_tensor([int(h), int(w)]) + target['orig_size'] = torch.as_tensor([int(h), int(w)]) + if "segments_info" in ann_info: + for name in ['iscrowd', 'area']: + target[name] = torch.tensor([ann[name] for ann in ann_info['segments_info']]) + + if self.transforms is not None: + img, target = self.transforms(img, target) + + return img, target + + def __len__(self): + return len(self.coco['images']) + + def get_height_and_width(self, idx): + img_info = self.coco['images'][idx] + height = img_info['height'] + width = img_info['width'] + return height, width + + +def build(image_set, args): + img_folder_root = Path(args.coco_path) + ann_folder_root = Path(args.coco_panoptic_path) + assert img_folder_root.exists(), f'provided COCO path {img_folder_root} does not exist' + assert ann_folder_root.exists(), f'provided COCO path {ann_folder_root} does not exist' + mode = 'panoptic' + PATHS = { + "train": ("train2017", Path("annotations") / f'{mode}_train2017.json'), + "val": ("val2017", Path("annotations") / f'{mode}_val2017.json'), + } + + img_folder, ann_file = PATHS[image_set] + img_folder_path = img_folder_root / img_folder + ann_folder = ann_folder_root / f'{mode}_{img_folder}' + ann_file = ann_folder_root / ann_file + + dataset = CocoPanoptic(img_folder_path, ann_folder, ann_file, + transforms=make_coco_transforms(image_set), return_masks=args.masks) + + return dataset diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/panoptic_eval.py b/PyTorch/contrib/cv/detection/DETR/datasets/panoptic_eval.py new file mode 100644 index 0000000000..9cb4f83409 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/datasets/panoptic_eval.py @@ -0,0 +1,44 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import json +import os + +import util.misc as utils + +try: + from panopticapi.evaluation import pq_compute +except ImportError: + pass + + +class PanopticEvaluator(object): + def __init__(self, ann_file, ann_folder, output_dir="panoptic_eval"): + self.gt_json = ann_file + self.gt_folder = ann_folder + if utils.is_main_process(): + if not os.path.exists(output_dir): + os.mkdir(output_dir) + self.output_dir = output_dir + self.predictions = [] + + def update(self, predictions): + for p in predictions: + with open(os.path.join(self.output_dir, p["file_name"]), "wb") as f: + f.write(p.pop("png_string")) + + self.predictions += predictions + + def synchronize_between_processes(self): + all_predictions = utils.all_gather(self.predictions) + merged_predictions = [] + for p in all_predictions: + merged_predictions += p + self.predictions = merged_predictions + + def summarize(self): + if utils.is_main_process(): + json_data = {"annotations": self.predictions} + predictions_json = os.path.join(self.output_dir, "predictions.json") + with open(predictions_json, "w") as f: + f.write(json.dumps(json_data)) + return pq_compute(self.gt_json, predictions_json, gt_folder=self.gt_folder, pred_folder=self.output_dir) + return None diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/transforms.py b/PyTorch/contrib/cv/detection/DETR/datasets/transforms.py new file mode 100644 index 0000000000..0419a81236 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/datasets/transforms.py @@ -0,0 +1,330 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Transforms and data augmentation for both image + bbox. +""" +import random + +import PIL +import torch +import torchvision.transforms as T +import torchvision.transforms.functional as F +from PIL import Image, ImageDraw +from util.box_ops import box_xyxy_to_cxcywh +from util.misc import interpolate +import numpy as np + + +def crop(image, target, region): + cropped_image = F.crop(image, *region) + + target = target.copy() + i, j, h, w = region + + # should we do something wrt the original size? + target["size"] = torch.tensor([h, w]) + + fields = ["labels", "area", "iscrowd"] + + if "boxes" in target: + boxes = target["boxes"] + max_size = torch.as_tensor([w, h], dtype=torch.float32) + cropped_boxes = boxes - torch.as_tensor([j, i, j, i]) + cropped_boxes = torch.min(cropped_boxes.reshape(-1, 2, 2), max_size) + cropped_boxes = cropped_boxes.clamp(min=0) + area = (cropped_boxes[:, 1, :] - cropped_boxes[:, 0, :]).prod(dim=1) + target["boxes"] = cropped_boxes.reshape(-1, 4) + target["area"] = area + fields.append("boxes") + + if "masks" in target: + # FIXME should we update the area here if there are no boxes? + target['masks'] = target['masks'][:, i:i + h, j:j + w] + fields.append("masks") + + # remove elements for which the boxes or masks that have zero area + if "boxes" in target or "masks" in target: + # favor boxes selection when defining which elements to keep + # this is compatible with previous implementation + if "boxes" in target: + cropped_boxes = target['boxes'].reshape(-1, 2, 2) + keep = torch.all(cropped_boxes[:, 1, :] > cropped_boxes[:, 0, :], dim=1) + else: + keep = target['masks'].flatten(1).any(1) + + for field in fields: + target[field] = target[field][keep] + + return cropped_image, target + + +def hflip(image, target): + flipped_image = F.hflip(image) + + w, h = image.size + + target = target.copy() + if "boxes" in target: + boxes = target["boxes"] + boxes = boxes[:, [2, 1, 0, 3]] * torch.as_tensor([-1, 1, -1, 1]) + torch.as_tensor([w, 0, w, 0]) + target["boxes"] = boxes + + if "masks" in target: + target['masks'] = target['masks'].flip(-1) + + return flipped_image, target + + +def resize(image, target, size, max_size=None): + # size can be min_size (scalar) or (w, h) tuple + def get_size_with_aspect_ratio(image_size, size, max_size=None): + w, h = image_size + if max_size is not None: + min_original_size = float(min((w, h))) + max_original_size = float(max((w, h))) + if max_original_size / min_original_size * size > max_size: + size = int(round(max_size * min_original_size / max_original_size)) + + if (w <= h and w == size) or (h <= w and h == size): + return (h, w) + + if w < h: + ow = size + oh = int(size * h / w) + else: + oh = size + ow = int(size * w / h) + + return (oh, ow) + + def get_size(image_size, size, max_size=None): + if isinstance(size, (list, tuple)): + return size[::-1] + else: + return get_size_with_aspect_ratio(image_size, size, max_size) + + # for i in target['boxes']: + # draw=ImageDraw.Draw(image) + # draw.line([(i[0].item(), i[1].item()),(i[2].item(),i[1].item()), + # (i[2].item(), i[3].item()),(i[0].item(),i[3].item()), + # (i[0].item(), i[1].item())], width=2, fill='red') + # image.show() + + size = get_size(image.size, size, max_size) + rescaled_image = F.resize(image, size) + + # rescaled_image = F.resize(image, size=(1280, 720)) + + if target is None: + return rescaled_image, None + + ratios = tuple(float(s) / float(s_orig) for s, s_orig in zip(rescaled_image.size, image.size)) + ratio_width, ratio_height = ratios + target = target.copy() + if "boxes" in target: + boxes = target["boxes"] + scaled_boxes = boxes + torch.as_tensor([200, 200, 200, 200]) + scaled_boxes = scaled_boxes * torch.as_tensor([ratio_width, ratio_height, ratio_width, ratio_height]) + target["boxes"] = scaled_boxes + + if "area" in target: + area = target["area"] + scaled_area = area * (ratio_width * ratio_height) + target["area"] = scaled_area + + h, w = size + target["size"] = torch.tensor([h, w]) + + if "masks" in target: + target['masks'] = interpolate( + target['masks'][:, None].float(), size, mode="nearest")[:, 0] > 0.5 + + # print('pad:',target['boxes']) + # for i in target['boxes']: + # draw=ImageDraw.Draw(rescaled_image) + # draw.line([(i[0].item(), i[1].item()),(i[2].item(),i[1].item()), + # (i[2].item(), i[3].item()),(i[0].item(),i[3].item()), + # (i[0].item(), i[1].item())], width=2, fill='red') + # rescaled_image.show() + + return rescaled_image, target + + +def pad(image, target, padding): + # assumes that we only pad on the bottom right corners + padded_image = F.pad(image, (0, 0, padding[0], padding[1])) + if target is None: + return padded_image, None + target = target.copy() + # should we do something wrt the original size? + target["size"] = torch.tensor(padded_image.size[::-1]) + if "masks" in target: + target['masks'] = torch.nn.functional.pad(target['masks'], (0, padding[0], 0, padding[1])) + return padded_image, target + + +class RandomCrop(object): + def __init__(self, size): + self.size = size + + def __call__(self, img, target): + region = T.RandomCrop.get_params(img, self.size) + return crop(img, target, region) + + +class RandomSizeCrop(object): + def __init__(self, min_size: int, max_size: int): + self.min_size = min_size + self.max_size = max_size + + def __call__(self, img: PIL.Image.Image, target: dict): + w = random.randint(self.min_size, min(img.width, self.max_size)) + h = random.randint(self.min_size, min(img.height, self.max_size)) + region = T.RandomCrop.get_params(img, [h, w]) + return crop(img, target, region) + + +class CenterCrop(object): + def __init__(self, size): + self.size = size + + def __call__(self, img, target): + image_width, image_height = img.size + crop_height, crop_width = self.size + crop_top = int(round((image_height - crop_height) / 2.)) + crop_left = int(round((image_width - crop_width) / 2.)) + return crop(img, target, (crop_top, crop_left, crop_height, crop_width)) + + +class RandomHorizontalFlip(object): + def __init__(self, p=0.5): + self.p = p + + def __call__(self, img, target): + if random.random() < self.p: + return hflip(img, target) + return img, target + + +class RandomResize(object): + def __init__(self, sizes, max_size=None): + assert isinstance(sizes, (list, tuple)) + self.sizes = sizes + self.max_size = max_size + + def __call__(self, img, target=None): + size = random.choice(self.sizes) + return resize(img, target, size, self.max_size) + + +class pad_resize(object): + def __init__(self, sizes=None): + # assert isinstance(sizes, (list, tuple)) + self.sizes = sizes + + def __call__(self, img, target=None): + + img, target = Pad_img(img, target) + return resize(img, target, size=(1280, 1280)) + + +def Pad_img(image, target): + # assumes that we only pad on the bottom right corners + + h, w = image.size + pad_value = int(abs(h - w) / 2) + + if h > w: + padded_image = F.pad(image, (0, pad_value, 0, pad_value), fill=0) + else: + padded_image = F.pad(image, (pad_value, 0, pad_value, 0), fill=0) + h_, w_ = padded_image.size + target = target.copy() + if "boxes" in target: + boxes = target["boxes"] + scaled_boxes = boxes + torch.as_tensor([abs(h - h_) / 2, abs(w - w_) / 2, abs(h - h_) / 2, abs(w - w_) / 2]) + target["boxes"] = scaled_boxes + + if target is None: + return padded_image, None + + target["size"] = torch.tensor([h_, w_]) + + return padded_image, target + + +class RandomPad(object): + def __init__(self, max_pad): + self.max_pad = max_pad + + def __call__(self, img, target): + pad_x = random.randint(0, self.max_pad) + pad_y = random.randint(0, self.max_pad) + return pad(img, target, (pad_x, pad_y)) + + +class RandomSelect(object): + """ + Randomly selects between transforms1 and transforms2, + with probability p for transforms1 and (1 - p) for transforms2 + """ + + def __init__(self, transforms1, transforms2, p=0.5): + self.transforms1 = transforms1 + self.transforms2 = transforms2 + self.p = p + + def __call__(self, img, target): + if random.random() < self.p: + return self.transforms1(img, target) + return self.transforms2(img, target) + + +class ToTensor(object): + def __call__(self, img, target): + return F.to_tensor(img), target + + +class RandomErasing(object): + + def __init__(self, *args, **kwargs): + self.eraser = T.RandomErasing(*args, **kwargs) + + def __call__(self, img, target): + return self.eraser(img), target + + +class Normalize(object): + def __init__(self, mean, std): + self.mean = mean + self.std = std + + def __call__(self, image, target=None): + image = F.normalize(image, mean=self.mean, std=self.std) + if target is None: + return image, None + target = target.copy() + h, w = image.shape[-2:] + if "boxes" in target: + boxes = target["boxes"] + boxes = box_xyxy_to_cxcywh(boxes) + boxes = boxes / torch.tensor([w, h, w, h], dtype=torch.float32) + target["boxes"] = boxes + return image, target + + +class Compose(object): + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, image, target): + for t in self.transforms: + image, target = t(image, target) + return image, target + + def __repr__(self): + format_string = self.__class__.__name__ + "(" + for t in self.transforms: + format_string += "\n" + format_string += " {0}".format(t) + format_string += "\n)" + return format_string -- Gitee From f2a449eb51db62a47811bb013fa69e6363b9d3df Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:04:35 +0000 Subject: [PATCH 42/59] my first commit --- .../cv/detection/DETR/util/__init__.py | 1 + .../util/__pycache__/__init__.cpython-36.pyc | Bin 0 -> 128 bytes .../util/__pycache__/__init__.cpython-37.pyc | Bin 0 -> 124 bytes .../util/__pycache__/box_ops.cpython-36.pyc | Bin 0 -> 2749 bytes .../util/__pycache__/box_ops.cpython-37.pyc | Bin 0 -> 2700 bytes .../DETR/util/__pycache__/misc.cpython-36.pyc | Bin 0 -> 14494 bytes .../DETR/util/__pycache__/misc.cpython-37.pyc | Bin 0 -> 14120 bytes .../contrib/cv/detection/DETR/util/box_ops.py | 88 ++++ .../contrib/cv/detection/DETR/util/misc.py | 489 ++++++++++++++++++ .../cv/detection/DETR/util/plot_utils.py | 107 ++++ 10 files changed, 685 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/util/__init__.py create mode 100644 PyTorch/contrib/cv/detection/DETR/util/__pycache__/__init__.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/util/__pycache__/__init__.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/util/__pycache__/box_ops.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/util/__pycache__/box_ops.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/util/__pycache__/misc.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/util/__pycache__/misc.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/util/box_ops.py create mode 100644 PyTorch/contrib/cv/detection/DETR/util/misc.py create mode 100644 PyTorch/contrib/cv/detection/DETR/util/plot_utils.py diff --git a/PyTorch/contrib/cv/detection/DETR/util/__init__.py b/PyTorch/contrib/cv/detection/DETR/util/__init__.py new file mode 100644 index 0000000000..168f9979a4 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/util/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/__init__.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/util/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ce80ead7a8df0dfa09aa3e9fcd147d99516d47a GIT binary patch literal 128 zcmXr!<>gB3^-6SSU|@I*#Bjg}WH|tFF$<7LVF+f>Wb|9fPy`Z25WjTwGxBp&^(#vC zD>4)FD>C!*()C?jLxS{EQcH^TOG`3y^yA|*^D;}~g`kf<&#ZM0X(l7{q}AMj*ohh>KZ(L<&PNgC?WjN`@kkFoO7{s-Kaco2p-t ynV4UZnV*-gpORWqq+eQ+nWG;cpP83g5+AQuPY8wLp literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/box_ops.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/util/__pycache__/box_ops.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38266ba3db12c3b64edef438c8b1c9e8d99964e5 GIT binary patch literal 2749 zcmZ`*+iu%N5aseFO15R!O>4OhQ1%8yjUm}dpMqWzBtcrBG0-$Y0VRe?YgZCwiBy+V zOc~Tml|1IT-_YOa7xb+Fp9&Nx;9uxN+A~X9z91#Q9dbCFot-&*W^b;nRIWF_^H@dG z{?g7}I-YOij_zULT1(@M>qFLJkj79S87+fp+~n2^tz~kXmmn?f@G_*$D|`vE#FzOB zq{H>^wR-jVX5~rj5B=B|k=qTWdk{_n?gu?QB<{!y{PASy#eNvLUclW4--J&Zl@m;_ z?bTW4z^Ls>;dL0PaM{Iu7c%-5L#hoB(wsdrXKbd=jG37t=+wBP)wEQ{lc}Bx-Nr(< zq4ms}6|;e!THNBsD+<@%JE4hsxN-gY6+$q$KRWprm$aa&8<{-`4*Y;;RvgMsKeM9P z>m1hYj3t?ov^$wG#lVg-*3a0%ubN!NZ~tsJ`{775lSwo2y)eOYd(C_IA3thx5zFQT z8EVo}!*SFYPqTG;?j)V*QNJCB?PQuv%kUC87&NxduCN-DZ{hhuS|~o17OLKx(vlh* z8rQdwg48UYjFcfwW~$fToDONg*9qaG&ZG&UvQVT#QZZzS28yh1ONuhnlbuYT;!buU zcV(3fC=21(qFeEB|#+` zS}&0`yk=Hs=#9qpN@fEsBHAtFjm)AAvzv<>nLdm&T^?lm5wPk9v5=WH2>`&%@WaXF zx|9fc(9;YNMWZo?IeLRN^fk7v*K~);PoQ0tg$hqFP-S@+p(ClF3rdPA04@1cgFToR z;LsXqb1mA5+0P8+xom%`xorO<*~fZH1=pYHwA*82V4m}z|B)9x_qy<*if#<7bJlr; z=dBkp!}_%7HaECg6#djEtWT(AOa!1p?@s6 z3$Am!G90=6I37p0n#~?E-vFNkAWGg4eA29_pC0!G(Ipr(>W7mdcMpW?2kz5kXWO0P zPIe)8_laLeUY!5)NW>EvM6TyP-P?8_?z>~pmq&giK)SK?lWq5?FQjny+%Mdr2(INI zs;l{ki65TV-^leB0IE=!1GJPG-f);z??jQ1#O3#;45b1}P7eepxkB8qo|JSR81Dw7LDtP=rGo`jTsiR8aNQROqBm z1Zo{P9P|Vt0GCg+&QIjD-&;brwEW z3JcK3|M;jD78S%*1s*G@otAqhVj#L%J^uS{I2uo4;l>JIXMGt!^Cb0Y^<^x>=ROBs zM_zOoogq@aUPNRLK!v!kx7{E17Z~#+EPyr$wgt77)2%>TP7&$}H7~HL&Bb%SBNWrp3B%&? zbSxB%Ro8#<$1iv4y812p)y%?j#4zV&>sZJzlGpJIjs$&gEHc}Z zUeFVn)g6XjoH-*A^?TCiS$PsfKTU)<7Fnr_!w=S?Hb=HZDHXik^|71GgD@QCn@n}r zrOZ+pmn)b`v8GpXotvH6`OVCJytGs?@C0xF z7%Yi#E*I=8MGRUvcD{HotHNwD-6zeLs|Fa>N@&8{pwz_d&Y-M2(XYjDI#YHN&hB%Kcax5Ms^}$H3 z(!7amNK(#WsdM>JD@vqi3G^!Vz@#w+As2k zyej-7FV6Y3!sIn@TC^iQgiP8+`}Hd(DLnfDVFhKNR>~%34rFrUnMosO&&;;9j@3Dj zIOe)BHODrjt#i_lxg=e13TY>Ia=U6A*xUpd*x1e8xlY=x*yke8roFEk;}U>2vx)Ym zlg~?4W9;Rn+}kk5<-DAGZ339HD_??NlHg3>>W9&Zo;#8bK_4`@K_=Jmshn0X=nrZY z%Aus#Dab8l_EKfaePtd2nPHR)shnX1#3?I`hnM40BD{fL#SY0T27`Ib zGwZBwuCPtBYI;n*1@1g76!CdjuEAU!DoO$!iVgs|e9A$d6#+POCdSlAwo~>!LwGLB zpK>nCe@pTyf{cKhk4?(x)SB4m^cQ~=K`*qX|SZanO9e_!}vS+EfR*bsyh`$(+@Se&8g9_=CJnFA;M4Fr z24xeBFXQv&&e{1jXIQ-n>4d6>v9Qgy+43B%vIgF~FfQPOYLzY^e`7+0aSCi6n({i; zdh`S1z$&UeW;QjmXTUFzMrDQ_pcmu})aCATiwchB&`QCfqJpUB_LR^`-3A49;!y`e zApo8xWnP-HNrih;;AP!FA#o-Puw!v*TrqfgYR)8<<`O07u;x0v-20CXl#l<>QGs2{ zz+)wM^K!?A4ODI_$A8|7`-5RB{8Z!XtoH(FfuvrnR|YbE5^~_RA0&s#86x%PMMM?= z)QJ0R)Bk2~jxVXC=uUlvoy2VD(HvLMCl05etKJH$p+^sY!-D;znKd&`v z@_ebi+&F<+p;POwG*b(<*cR7f6G=+s0$Cg##3>v{jv7braiu>p8S@h z0(J3$cW5_0uAWjK!zjfnKd&~_72Gdl z%oh$V^(l2#ed2+Qkv7z))l2GSKz>Czx2*bikHX4Zt)v-j4|~nD)$b*jgI2HE8LC!q zCs3^<)vc{z8mnM8?hIm`Y*gOrCu!#(X!ZvOFASn2je~97?*?gKH+S#05{wb(cstf{ zuNkBJ2R!}Fx|3B^yd4cYscJRTtf1mM!x*jp+XGG;bu#Zw%vQIv;_bMX^mS%$d;lD= zcKE(!oxg!cC&ONRFO0i`^dL;77aD6}f0z!2X_&M|antlRrHk7QT*&}MYyrtEA{dbC zG3BWOHo?R3l$4KJL6ubnTULxq=E$*GKI(GdsFzVcp;jIMWf~RKPpVa^S5ZHu)}&rj zr_~vZv7}sV?b&SUR=3|zcVqR1s56X5$!#4q_kt)$V%>_9puZj5<%1-MdMX&`{@s>} zgQ$sZOR$|mlD|Fe)xR6-AgToT+-vnz|6b770frBDI{mGvGaWWUw`eEckd7a$VdPOWJ$b(yt5(#z zw(;`zB)D|nOddvecci7x6YNI&ow%2uoMmaAW8Kr7IV+_76sKKi_J_SRbGN%`-Oaot z?rdjOvn)ACy@JlaV(C-3eOSM;+waC#_J>#YTTy=>__A~5wbyUIb4A6ezA{W(ohw}+ z@y6gF^TV*!Yo%d$(ZLdHc-yjl+p}wS5#Q07BS*3^?W}$DJF-h;RI*D~Q5jopt8I@# zKsM^0)yAHD%5u0j{N;}Av9o4Li`gf8bGqDOG#JD^rB7fOy^Nyn>XW!-1v&f7?#U8O z!Nbam!-0xY&<5ZyqQL^}%l2sPDCDM7lRL`~I$H%?0&)YD)H<}1in4E6sSR)mq}BFz z9Q-OMN4XE(iJcbPr9;cMaQE}Y+3oV7g(nrDf=B+yqes1P77GtOj#n>^t_f}t5#Nn; z51is!p!&e$UOx?zgI;r2_j@2lK~5p`i?bItGp{*R5h%Z)(t|;q6;*t<)riZm!qXO*ba z33aT7P0W|}JACN2dTHG;XJMzu58;W0WrSOCdM}Q9AxM<839iPoA%a?75xDg@FsyF3zf9EN_Lok;czG~t51+?a!W#vYwKy3u$( zgO@aczGgeg0Z6{M>~s0{IYNknH-M7VcznLfoU6UgX*rzbjMk4%I`8!8xhr@HOcgU9 zl5TKtPUeC)$`(9&W=(eU*@Y>OtrKs5^juu0a?Wy zo;tcl;ZjGYEa3&g)BqJu3C>84n6nt%3%H6viUI%uA`}GUG{ud+!h&nmoEOMJD$0u6 zecg@HkM8&AQ!0faP(KVmPw)+^Cpnt$==9Nr`C+_&#y02V)UYJ9EMpX47+_b z?66)7!#l&MlRv>ss^1JleH!oRvn-xPk@;W=(2CN7j|5%6#LjbcmKyoIuA(?k$ob26 zJ*VPV3jR{9fUn1|^q6h^5_-+H{$=1eIXtOO*?W#&8(TXwUJsVxq;A{8K{%+ps29iX zq1`Si{`Td>lN}J~ z`)iQrAX3HCrh@1EgmcFuk1#TKVIUC4Ot64gIU83qJ6u3sV(N#L>x8t~?+#w+PWeXB z8eN!g=f?F;zZrFsS5TRDJpv=8iNLr%g>i*2Ld@BRz&!(>%nOJ#WZjj6P~-u4;8R>yKs2J8>G`>+6mZ)iWy!2E@=%hTTCT zTar~jbCAZjpm;#F&Z<0>aBH{?GbgL?Np7<^=pa(L&>MDRXt_AFomgKa*rJ4(b2S|B z3|k9g&18ik)R9kTZZm>rGHMM_8w@j!0-d&$_O~!ck` zMgvAg4#KRe%xw#KAej#_Vj8OknWb4@BS57=q-gwQWzJz9$o9>7n6R(!L+1uJoP6E# zi_m|HZUwsPio0g7iq7g6Jm^Ii@MN7Wj%Tl-w&rNQd%H<&<%e92%us~N&!8fl4JtM` zo26IBKo#n+ht`j$!Up-RS}0-alH!KJN!zk}EA?Z(29VE?K9>?sVrWZ2Dl z2nSdky45p^Hs+-z4Gtzug8u#U?4tc%fHkm_RmHueYHA60AMP#sJ>UUR%OHVM zS?w(t-L2-E{T*<&5AE`dj(NbmZgNtdMGCqT3le8Ib0LUn%x&h){{W;6G$-(kSAby- zv?F`$rcT>Ebo4XeAuzP^wnAFmb`G87(`o6;RB`<=bc;6lgK(7!cmp+Utf!I{XmKT3 z1@5mN&;yXj$Jxf9!Y{O}&4r<&iP4$Y?zejRK{xzPKS^lBWFC1$RuF19wqh|M{vB6pPMc!X zv(GrAwPWiPV`#QU9yc#pV}Q#3;|uGniNOw`iq+KSsO%c(6}pRsGbP4*Y&eF5&!X+O z2^Nvcvf)Q~?g&8N+4({C#1HSh1WY=}=x!u7UZ2sIEkNatGkq6*^dsQO%1~TF*}&f= zR9w3SkKYlPW?eR$0yVmj;;4Q!aV38Y`L19hjoqZC$Q%F)7(yYx7gK~Kl=l$E&#dOF zLJD(>7qP*7AA{oXm^CShGQFX?(FW@}KrN7e-73U1rwdW+?9>e@2oa)57wdZ`( z)(_Ah<`>(>-nzABO_s107=P)~cnSSh9^%PldAbt3(MXra6&QN?NaJNXgAVBu!a7j; zD7o&HLl|A>EOqiB+HI@%-2eSOwfYb%#TqukEv)MFrf46TH7ehqd8=)4KzwLl6SvdI{nGubqvGVs{ZCBZ zA639*K+-5DBOhLG%BjhymbSVvnt8)7D#`f8+hlYadUeo^yJ$e!*9X@&u5M3ufLQo^ z^M>&_*jXuwoBbXX`9kJmbi*7y4yY-jC=>ZIn8X-A4Mk2^v73C%pP<{iK%nh~7Fyk-6~88_`6wFRQ`PDn3@4yTYIJkbNm%#8;E~SXs=8 zvNB6ZX#h-PPlVRn&bKBYP*XjJ5ta9hWBvf^$WDVR@o zE2}_c>NJ#DTsX@cWxgT5p>@y-;iE=58AQ!EM8u#`>hyQQZnBd(Z+#|nA$uBCY-E`4 zD*`n90TBcB2j-Ae(F!M znERj-{(OlAd9mJS@sP!wQ%jA?nBGS5&$uo!S**tMOFloI@5(dsKlxB!VjptlaBTJTH34VZ`fdLw)I0+&-tRImd5taTWWdi zpajAHy!B6t^aVj5S=pS)KX7tUcen!y9SvN-*a8@ES3{~pl0)gE663-f2!s0`A`4N{ zY_;khP;$7Hn@L%%xf?h4@+c$Y70teas3v|5@L~zJMi(Ip9N9N8i`jebc;fy8rVX3|bPw?Bz^I{NL;6?-g`BbVFJjmu$4bn+ zi7R;;1+9#+^$iH12{cvU5Oh{r9SRx*!~p7|ryA4KGcgF71KEXMjOew>Bgx=O@aZhL zLl9;f4mvSTxUdzah-uA)*6_B$Q$xp2F;41i{_x)odhOAJ2)nB>Gu0R90~42aM- zucLnZS^{E=52zprB*w|{mDNJB^R0t0+HWOmT$?DK z4o!?zM7#9A#+*%Dvk&@MH*h6nP*Ynoc4LGPmHjPoG9$$ycQV7qG&Uz9?_zWMs)Dst zv)ZN{i1DRi-@R^>>@_(yKfD8B%s#H5LQ^FabrF$j(y;_sO6CX?{p;v5)fxW{>e6s* zslrPyVB}nfh4FBD5s0}#$(i}#NfMHqxRM(vj80AMi43fbQ-h9Y?UnV5=AJy0x-cJI z6F;7KJx@`ufkwtNHBc9{0Qee)Gf;L}`!y6GFUnByV`kMR(k*fnm41z~v~Zh_FN|Y7 z#f6cRTMonDB6UDzqZ3D#nD%F=N?e+4@nYm`I|tM;fXdcBIwk`H zADMC1<8di*F&kIj1dxJ0EXpv6`f*(&Go z8*KYc7W1j3Cc>0w^8}~zQ2||eeidR=PR!QtqHVS`?tcl_OUHRjskL>4wVf_^3xSyU-WqJ+L6`7OxR58r!18n1byNED6j$7O= z`r9Ogu{;Y8919LyLJx@I-DrS>3F8%XTiu<7Gn3Box z^8f-*T{lp?F7s$Sn>!fWL3#gF!@k$SwPQ5aH&a!^y#qQ>+K_x={pG@ zVX^ujaGLq7;+iZV_Caag#{ii}udz5F^>3jsvuXwh2xNYNc}2#xA~?A$IH|wDK2?!s zR6~$`VQ*!2GqZOy``*Gt3}F~X^zZowLjoaoHM6_NH};qC{AF&eh;e7x_Fdmy#k~eq zw*sHQI=(O3&p>u)_QfueI zLK~PpjR9cHS{oUK|WKQzpabzTw|L`0}A<$eknj?4S!}g|+7P*Fk=UQ2n;WsFyAh)DZ%YRV7Cvq_Qix?rR%sc7dXT$U@(FEqy%hR78 zrJUk_M8n(Mp{G!>pbMO_H=t$vMQYv^sCJj^inDG%4OuB~ra$r0(Udb~kX#aI2 z2caz%;a6=$?-kV7#OM1W%=l#J953kDR4LSSXb&4{=y$ zxuE|H^(GspA1Ze@aY{TXZ@onvSn-mvnulh zyal?OYeY|=<9$LQwgQw*I!!!Vv;EON3T+=Jc9W!ja+%t#5cVJ0ZK6Z zKgMfXa9<8`cAD8TJAlRKnE#WUpUj$Dtp5`C8CT%War1BDO1_OkNwx_hNN*_d8L~~We`uWneTRkKYFbtj z`B%`1!wP8~Zg$k9UlnGVc?wM-tf6nYy@WRA4=f)dONseLk_AhrwuP)G>bA&h#O12$ zLoa^@cQ$G#rM0#vI)!kRCFHce^XldgW|eo}e&@~C!dsthyq;CweEa6BZ-#7d)NZ}` z_uqLd{M|=HZsBl@A8CR ziS5oX*@dqPX|ByZxJ|RFWIn@Dz1vqN^6Uu)3v#dOpTpr~rMP#uh4fq_l>Qr?lq_vwN66owGd1|fKpv(~R^x*(2SWV8h#IqLs5{xJz?8ev5L53D2b#i1 z_!CG#Fz^`iotAeNrej)O{4>NsiX97{p%r`?0$+jW2n-JaaWFmH>33bmw_@x6fu2p$ z20!$G&zvy7iXSElEV*+RNG&uoaN5X@k~}(?=g_eb6hXE$tuW#Oa_A`Cly~SsNum8E z`O|Vo{&c&Hr=>#&_Lk_Nd4AmWu>%eBWoULKRT!7)>_Jb2NT6c^*JShxLODwasVoD3 zJp?U`-q(YZ#|OVCk3M-5CN=(rfxiVH-G89tru@(1#$EtnV=^)ZXhTMZn*CW|JJ>TW zFOwrCJoWbG4mZQHe&~k&1a4B)Pw3ZKzrf-@u@F^7l%VVQiMVWku&256nFGhYcefSa zgR5zOz`X1z(H%ZLNF6fW{%sxq06KS~T?QXTwT;sy0b%dO$`;n{ZRVPA)KLx26c zt^XJMmwIu8VbdnjfFClg8RPng_(4Zv8^#5X#IPpiA_1v`qxZnPy**D~qdQyTc=SFw zFs!{pWVw?4klg}lV3;F`(rQ=f7FJ`+7&1Z0DzwAI8_ zBe744KVf5mDeC{uhZk88jb&5K_Wlij{w<2^#DW)M1KVUaZ0%U<2KP?37DOxAs*;-{ zX?l`ZB=I3}!;5R%I45Yvh)til?N*Avt@MUTti)sx=5CZJDq}{^q23~@gK%d3!~99g zl#N{2`neL~UpML4ulU%gfQuzqA5y38*`K+Row#ecqXZl&R``9;6H a^4RJ*gb&X9ClKT+;_LYZ|11J=tN#ZxS0mT} literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/misc.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/util/__pycache__/misc.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b3b4b214ac31f0fde1248254220e74a874f5f27 GIT binary patch literal 14120 zcmb7rd5j#_d1qbSU46`OW;mpVB1IJySz^o7P^KN(VHAN9CFYVwj>uSX+eB?oznYnz z>5H$b$(d$#4wGJcfzVbq3uFO1*cl{2oC7(KW3h0M#Rdo#*!-2f5LB?h0=qyUAiG%X zA3@@rzu)(2x_f3RFVcg0_3G7E@BHrXo7d;&Di(g$zrJc&|AP;QKM$2_xZ;Pl zWhtv+DO))myJ5@SX*hCs8!ql{$Lkgvg>JD?v<+OTQIhw5!^hq0l)IHirCV)O^Y^tz zP2SIS=Nl)at^bZjkhKYwK=1MjwHW zHX;ovf;&jpneEsCFyK3M#rYjL_=8GvKhzPUgLZ(du?qycPB0zRv8` z4}c}sCcn3=bJy@{G3-V6!l*k)_QFJ<&{zxm!(=c_!ni$(S_aybE^gOw#RC+P1vImW zVnDAal&1>V1P{kkQa)-0RaO;jSurY^Bgba>sLO$)UPk?dT6hSQX;e^OR7+B?qJB~> zOTDH}snh6VPPy3HGuhnDZoi*wMd~xn&M+Fqw{)|$6EuT3((Nb?`WwMrzKDZnPXzO0P`b*(nhI;9r|)+o z)o%8#u6|-;99+C_1`nHeH>IV{6YMs3J5et`Im^;K$GWFEa#l$C2~NAv>JNKK=5BP8 zx|?}%)Y)iM&9dYm^#Xu@$ee1&It$sJUyxVT}cY!RMmsOPL%fqDIx!eWvt`7Dx zKMdQwb`pk9ENlL z9OXW8$97WOFCAF6g}a|G&fYH{Sa?$b3V7s;JaW_vXRz?lV}JGH=&Ilp5%9gH?tw#G z4OAca+v_Jmyw_`O>3$DHC`c&;eqr*#TITi!LlAsHC3}M?E2`*jyA=s)$SKsT*;DN} zB+CIZHQSvKR3T^DZg$YABIba_WJN>y)1A#q8=ZbL`Pj>u$FH-}dQ*eIbXIA0I-!o# zu!ZrGeuppJb}y+r<|yps_!&Gkvx;y%O72BbF9cOcE2^FUlmHwYsgJ z1Z=%DovS8Fo2iqgCIY}WI(c+? z!k>M}P`P+Xj1>G8fbfxQf7sg3O^_5MgYmpSRfd&ftX2dU!SL}=ry}h7-p3)?Do~L!+I?Y z?+lxr{0&A@{Z<%i3Q)bm;yDzV4{iWGDB1hasp~5Qo?^1p$l-Mr1&JeOpT6rk6~9vO z=V}G~dHfTI$=1&i3YTpCRiHO{I|)wNJC0sXtxe(b``{Sh7Ea=Bd;5jdLETN<{bK4K z*!v~LXAkfE)FY4IgouX}aK;5ufh= zAWECo*x!AQt+&sQ%c;LVkDgDYr41YR1@v5cgi&ql&QHE)rG>P#1l zTKl4N#u__m74xz8pWq&F7Vq14EX<<{q^ynport1$hrNJOVY3OvNe9qXn|i&S=%(Ha z2JO~Pr@7vV0?6TJaFJ527m@WhbsHM5kvnl+I(kiS##3>~53WLuo9(2(z8$p^GisC! zb#Dfq%=>fv==Rlsb?SmLsF@ekZPBTtW@{_Z&EAgmBIv8n03(3PXK|Zg3Yi8~hMTxz zN|H^G>Fs66bC9aynNzWIe#*JyVJ}=GbZ0sOiA*F5n3c2oQf7xUC`|1AR^=7WwAJqp zUftn1X3)|3>2|KZ(&@LFo%mH$4ndF1k(!Fkk@O^UT;Lob=j;PupqZbHiy1u&m_rHi zDb%4PLJc@JD}5Djy14S`;5$;7Y!4Rt-K!du4;GJ>MPm&W|V~Y`nsb;<;;qL z0TEP`VRsPAmSok}_mb!)G!LlMS(T>}t`9e0-eeWN$!!$}03wzPy}AwWJM!(VmdH8u zK-p5pB~0#Lq9RNVdN!CGxM&J|0ZBUt)>ji@2rig{w|gmpT1(}Ktn2ok=Bf9b)72#s)F+|fwc8qFd*kC^q4cla_XiYSq<6Ydtib;WW9@p zrbUoVsrfQ^@MDO-4aGKr;@V1ri!AHl1f7m3!MS~RuZ`#R2zD`bQ6+r@_~oWraB;og z@8Gk0TamCO7=Nlic?&Bq9(Hp^!VYG;Zug9~jd5v4gN+I2-~=XmYY_JV)J$YrlUC_) z3saB|@~J4yF@}*IuLkcLrWG{d2EfwpwUaG@j3GLMjds*gFkEBlWx4~Gg62j7#Fv$F zM!6ZvRNEx34~LMm*58bX__YaWaqA`OU8`60_W<6Ie5uzDH+55GMYtj0`pCSUXfM`=Y~utr zyb!j2hF?&Ts+Ss-=@w+Ad@uD6;T^~r`xR$Jg9^HVwni;S1ewsWHDuM9$vy;hOvYlB zNn@la&-R=p+?QZ?@W}^_X$6QD!47}i)@!)p??W+heJqj);OuY>y2JfZR3$hp3d&by zIzLoJRdFw=nwrDihi{8c54bUTsMGGke9_?FqL^ALb(u3(VyenWBWkhpT`x8qgB|gk8*VK$Zq700&X^cyiXhj zuj7HIvTf}YsYt?368k~gY_wfO8x&326bU%-zMH@k^Z+gdxC@*I_7|X3VD--0=)&{& z$Ik~w)d{wmbOi*>BzGDHdlCiLxHAezD8o> zD--&%Wm38KOb-F0A2Xk<48CgK(2syZ>Qy>IU>q)>X^w{I86b7^_MY-X31Rz%@R z=RSi|g=KcU9V*gktZv7N{~|4z*{X#VYsY!t*88cy4SS3$pq;jCjpwnV>UJ%eOXm?I zS$u>yV@QGo?XblN=bW|9TH}SoHKLch$%(YOy>QrjdV;>J2XqyltE3KWi z(gn5j2ybkw_rm}GOLg)QEKsbhpcX!9jSGioa_1XR@Z_A*;)*q1oXmT1GVi72gj(KT zfU38YE}B)IwGKefuyV=9POXV%l3Anj{fQgf7C*(e?5pCK8o8gl|I(;9zI^{<Jf) z-x$Q=Iw$ME@1@s_pTf>can$Pfpz{|q7lRo->al-KX-4tMzTiY3#9%7-(Pk((WsM-C z3-`ytEjrwTe0Fr%8`pgtTn^evy}qOcyvlE46qyU(y^&xv5wjY+vf_}XX)J6^CxoRK z67NptW04X2&&n(z%rV(#`Fe*P6b2erj$wt%G(OCw*;j2bD9yD7~h%iH= z)ah@A-FP!|-hMrEA+s7)>}r^AGNL+sfavON>L2C2D-x+NBg{|hggM3yvogDm5O~@9 z1AG~8ps>KE`RC)0qUoH7CkLf7`K!1^)DK&H*hupb?zM9%W8Xvv!3A*}1;Q;;SsC%wB5bk~%{1uW$*U z#+p38NU}0?{Lg`PY2Fb<8 zM0PG9wJ;6%y`dOD8Q6El>+Qm~2%G&LLKMxo)o#~4VCQf>x3scaYb$E)c1NeFvmx}0DS&)b03QS#7xAd} z;)~7ob#{YT?x=t`3!A&`_zAA9f@xb0t&}B%&GfI~sl}$l9}najuJ~mXhvsVJ zMhHbJ`|IMeMv6r4vWA&z%vMD1#ccIe1#78hwKX{qkds-MKaAz~3f`1+o4OfE=owzlpju99ydJ(hFQ6IfX@rd}?+QQ=3#M7$7`Ju2yDo z4Hcu7)07|s+s3J-E@JX!eZ}14=MondsB7ZTW3T5a_^V)$I6PCV8#ZV`%LLh{nPrEy zUq^vgi>#kTkyTqr*T_*+;2LFV;Wiy#82kDN7e=kzawvyH>Oe+~P8?Zc(x0Fz;HOT2 z*$~`DK2gU^RJQi<5gxb#=)UR4(MudXXMCrRj5q;#{J=!EWAQgpY-DUO64{WcgcD)} zlhV-uM&-w6Mm>I%24byVJ}yyt_`1kKWUHLRZ}IWBSxkqT3R9jXdy{1NhV5C!@R;~drJ4>?uh(MCb!1}uSkIwdH1*d`bOY!b-J z&wFGH&U&jlW$6YL!x76%eH<@ao(<<$0<)?EW3g>`f{R_x( zA38kjhcDR4f{aOgsuL~&O@wWRC_Jo(BX(@m6zyWF+HQ zblcs{nKL7LNR;9AL(C{zqbt2{%lfDCc^U{o_tQWGPhEc&#f{ONBu@+n1MxN17vT&* zcp|GxvV+N|g4Y8&ID%9yy5^q^`@P;SGNHbm@DUcPe~iOSM`esBDzUok?5)hcH!~1DmWC1iYkt9~ zLWo_>?C$Z6{RO=LBsW%^oX))MyM6(#&AA%X-ip0|kim*~#eNR5LlYLeNb1OnTD;Q} zDk06&f)OhDkEqb6gUi|d1}Wk8KHPfn?kKR?jW5aP{GpET z_z$m`Rt$}bj)cj##mxAUjQ}%QqHp2}koDZm!srbNUEJDg$CKnoCOp#qj1fD5$qH&? zNTw(GO$fAL^3aS9<0|sGXoYJnj4MCRgxu$8T=6p~=x^qXfIQBZ?RXjH%K0yYIj8nl z4&4v#8vMh^Y>IBsiWu|<%!d;XFyor~k5M2dJGVY;cQ7MlBVlG)84STp$551P-vfYt ziQ7j}#Pl)W9z7d_PzE6efcupuhh%e38I*K~{{6iFShE-CJ^uwax?2y&4dsQ51|s~= zcqvCZ3 z+KpS7RsdlA<0!IuYyngL^7OD7&N3CV=K%Os&O#JW><8^}&(>^z{5<&WYz`e~0d#TJ z48^(fshNdDjHVbo9hEPBW_k|HP?7M`KXycP!IR)*WyH*<;lq4@$1XMl+|M2xmTiw8 z)@Vr&fSo&}2P{yAvo$|?T>se#!8t)eNr?M7j%5@y{in0zu+7Ew4la?w+)%w_5)x-H zADXKxUz)g^7b9>6-1<-WObhN$H}@pkvUvhP`sJAaiyWWSM!j4AH{2&&flDC~T*npv z9~4SbArRTPp~Mlv8eE02=nu`kkWO4agZWF`%FCZ21mwBUyQU~87#~UUchgi!Bw8lL z?E(`kPlLY0LT@R73MEcPC|Y5e76#nEs0myJy1vA#q5fE56|iL_-m0f@*0=y0} zTJSsE>C2m!NUi>JK>m`+L7Uj5R9Pm<{V(Ie#3v=e>pba&+AB^L$vcwl1*?yaEvg|@ zW#F_z<|Wh+26Vjx4>mv5dP(nP3(4A6#L0X4~C-w1XN8I5Qf6c4xbODNYiQQ-I4YxNWQ4OEQ&@_Vca=w;Tv z8wG-e{lSj@2&)df!rt9>bPvvp-2qdEo3U{_yGWWaI*!*Jc0%|Eisl~$kP8g)ia7TF zMh6Wz33hk%KV{#N0^?yFssX2*#(I!1jC=?av@hpP;VkzRzT_ike+cb-?+=L*qPw^? z96X$)4xP0tFq-KAUa|FGa?Yh*)I_(*Bnm}VH9-STg+8fmXCL|{{!c|1l5&v_(=kae z%G>euReDAxszgVI!^D*Xq+cdDM5-4+6xevk?gQ(fyMzaf_)8@POM)F(5bLc#2LeW# z+nz^K%#nT+Q(pYh?}(Q3=AB_1`3q4m;r}J{daF}aauwQ0qz8R^vMvQsE=;0GYz!Jf z1j%#?#A*nbhJ=3#3GfrmOEP2NIPWD$y+n@PdW2k;+!hLY&7^%dV(VV;>AhRb{ziTY z{#PP_(k6YhI!*kqM61~gHW7Ee*W5E*Yi_0p>>Q`gT@$1elWXmRzhjE*3BeNmZ+IY3 zbKtw@zpfxWU5E!AWCTtz@ISD#KN7f^#cB#r;|-+gq?yEosKNk775^szF&lWuKph5S zYcUez;Oc9(VAlbH9^g6TwQu&o(sYkW5Vga9W1(R+BY=rw`fpME39)Md6|1x&%H6y> z4{sVYV!o~0x4^#s7i-?yn*J+P*6J_mmvI*+YrL;19^nUq=Ml(_^~RdQTC#=pPq6kX z0g4F5GYmrEs_C&ZOn6aj($yJ8sQ-*F0~W+#*-f*rf6d3gL6M!9v1C`VKeAJe= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + iou, union = box_iou(boxes1, boxes2) + + lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,M,2] + area = wh[:, :, 0] * wh[:, :, 1] + + return iou - (area - union) / area + + +def masks_to_boxes(masks): + """Compute the bounding boxes around the provided masks + + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = (masks * x.unsqueeze(0)) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = (masks * y.unsqueeze(0)) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) diff --git a/PyTorch/contrib/cv/detection/DETR/util/misc.py b/PyTorch/contrib/cv/detection/DETR/util/misc.py new file mode 100644 index 0000000000..e08159ebe1 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/util/misc.py @@ -0,0 +1,489 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Misc functions, including distributed helpers. +Mostly copy-paste from torchvision references. +""" +import os +import subprocess +import time +from collections import defaultdict, deque +import datetime +import pickle +from typing import Optional, List + +import torch +import torch.distributed as dist +from torch import Tensor + +# needed due to empty tensor bug in pytorch and torchvision 0.5 +import torchvision + +if float(torchvision.__version__.split(".")[1]) < 7.0: + from torchvision.ops import _new_empty_tensor + from torchvision.ops.misc import _output_size + + +class SmoothedValue(object): + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float16, device='npu') + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, + avg=self.avg, + global_avg=self.global_avg, + max=self.max, + value=self.value) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + + # serialized to a Tensor + buffer = pickle.dumps(data) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to("npu") + + # obtain Tensor size of each rank + local_size = torch.tensor([tensor.numel()], device="npu") + size_list = [torch.tensor([0], device="npu") for _ in range(world_size)] + dist.all_gather(size_list, local_size) + size_list = [int(size.item()) for size in size_list] + max_size = max(size_list) + + # receiving Tensor from all ranks + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + tensor_list = [] + for _ in size_list: + tensor_list.append(torch.empty((max_size,), dtype=torch.int8, device="npu")) + if local_size != max_size: + padding = torch.empty(size=(max_size - local_size,), dtype=torch.int8, device="npu") + tensor = torch.cat((tensor, padding), dim=0) + + dist.all_gather(tensor_list, tensor.char()) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger(object): + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, attr)) + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append( + "{}: {}".format(name, str(meter)) + ) + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, batch_size, print_freq, header=None): + i = 0 + if not header: + header = '' + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt='{avg:.4f}') + data_time = SmoothedValue(fmt='{avg:.4f}') + space_fmt = ':' + str(len(str(len(iterable)))) + 'd' + if torch.npu.is_available(): + log_msg = self.delimiter.join([ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}', + 'max mem: {memory:.0f}', + 'FPS:{fps:.4f}' + ]) + else: + log_msg = self.delimiter.join([ + header, + '[{0' + space_fmt + '}/{1}]', + 'eta: {eta}', + '{meters}', + 'time: {time}', + 'data: {data}' + ]) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0 or i == len(iterable) - 1: + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + fps = 1 / (float(str(iter_time)) / batch_size) + if torch.npu.is_available(): + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time), + memory=torch.npu.max_memory_allocated() / MB, + fps=fps)) + else: + print(log_msg.format( + i, len(iterable), eta=eta_string, + meters=str(self), + time=str(iter_time), data=str(data_time))) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('{} Total time: {} ({:.4f} s / it)'.format( + header, total_time_str, total_time / len(iterable))) + + +def get_sha(): + cwd = os.path.dirname(os.path.abspath(__file__)) + + def _run(command): + return subprocess.check_output(command, cwd=cwd).decode('ascii').strip() + + sha = 'N/A' + diff = "clean" + branch = 'N/A' + try: + sha = _run(['git', 'rev-parse', 'HEAD']) + subprocess.check_output(['git', 'diff'], cwd=cwd) + diff = _run(['git', 'diff-index', 'HEAD']) + diff = "has uncommited changes" if diff else "clean" + branch = _run(['git', 'rev-parse', '--abbrev-ref', 'HEAD']) + except Exception: + pass + message = f"sha: {sha}, status: {diff}, branch: {branch}" + return message + + +def collate_fn(batch): + batch = list(zip(*batch)) + batch[0] = nested_tensor_from_tensor_list(batch[0]) + return tuple(batch) + + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + # nested_tensor_from_tensor_list() does not export well to ONNX + # call _onnx_nested_tensor_from_tensor_list() instead + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + # TODO make it support different-sized images + + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + batch_shape = [len(tensor_list)] + max_size + + # batch_shape = [len(tensor_list)] + [3,640,640] + # h,w=tensor_list[0].shape[::-2] + # pad_value = int(abs(h - w) / 2) + # if h > w: + # padded_image = torch.nn.functional.pad(tensor_list[0], (pad_value, pad_value, 0, 0)) + # else: + # padded_image = torch.nn.functional.pad(tensor_list[0], (0, 0, pad_value, pad_value)) + # padded_image=padded_image.resize_((3,640,640)) + # tensor_list=[padded_image] + # + + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + # print(pad_img.shape) + # print(img.shape) + # p + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], :img.shape[2]] = False + else: + raise ValueError('not supported') + return NestedTensor(tensor, mask) + + +# _onnx_nested_tensor_from_tensor_list() is an implementation of +# nested_tensor_from_tensor_list() that is supported by ONNX tracing. +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + # max_size = [] + # for i in range(tensor_list[0].dim()): + # max_size_i = torch.max(torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32)).to(torch.int64) + # max_size.append(max_size_i) + # max_size = tuple(max_size) + + # work around for + # pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + # m[: img.shape[1], :img.shape[2]] = False + # which is not yet supported in onnx + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + # padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padding = [torch.tensor(0), torch.tensor(0), torch.tensor(0)] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop('force', False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if 'RANK' in os.environ and 'WORLD_SIZE' in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ['WORLD_SIZE']) + args.npu = int(os.environ['LOCAL_RANK']) + elif 'SLURM_PROCID' in os.environ: + args.rank = int(os.environ['SLURM_PROCID']) + args.npu = args.rank % torch.npu.device_count() + else: + print('Not using distributed mode') + args.distributed = False + return + + args.distributed = True + + torch.npu.set_device(args.npu) + args.dist_backend = 'hccl' + print('| distributed init (rank {}): {}'.format( + args.rank, args.dist_url), flush=True) + torch.distributed.init_process_group(backend=args.dist_backend, init_method=args.dist_url, + world_size=args.world_size, rank=args.rank) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +@torch.no_grad() +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + if target.numel() == 0: + return [torch.zeros([], device=output.device)] + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corners=None): + # type: (Tensor, Optional[List[int]], Optional[float], str, Optional[bool]) -> Tensor + """ + Equivalent to nn.functional.interpolate, but with support for empty batch sizes. + This will eventually be supported natively by PyTorch, and this + class can go away. + """ + if float(torchvision.__version__.split(".")[1]) < 7.0: + if input.numel() > 0: + return torch.nn.functional.interpolate( + input, size, scale_factor, mode, align_corners + ) + + output_shape = _output_size(2, input, size, scale_factor) + output_shape = list(input.shape[:-2]) + list(output_shape) + return _new_empty_tensor(input, output_shape) + else: + return torchvision.ops.misc.interpolate(input, size, scale_factor, mode, align_corners) diff --git a/PyTorch/contrib/cv/detection/DETR/util/plot_utils.py b/PyTorch/contrib/cv/detection/DETR/util/plot_utils.py new file mode 100644 index 0000000000..0f24bed0d3 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/util/plot_utils.py @@ -0,0 +1,107 @@ +""" +Plotting utilities to visualize training logs. +""" +import torch +import pandas as pd +import numpy as np +import seaborn as sns +import matplotlib.pyplot as plt + +from pathlib import Path, PurePath + + +def plot_logs(logs, fields=('class_error', 'loss_bbox_unscaled', 'mAP'), ewm_col=0, log_name='log.txt'): + ''' + Function to plot specific fields from training log(s). Plots both training and test results. + + :: Inputs - logs = list containing Path objects, each pointing to individual dir with a log file + - fields = which results to plot from each log file - plots both training and test for each field. + - ewm_col = optional, which column to use as the exponential weighted smoothing of the plots + - log_name = optional, name of log file if different than default 'log.txt'. + + :: Outputs - matplotlib plots of results in fields, color coded for each log file. + - solid lines are training results, dashed lines are test results. + + ''' + func_name = "plot_utils.py::plot_logs" + + # verify logs is a list of Paths (list[Paths]) or single Pathlib object Path, + # convert single Path to list to avoid 'not iterable' error + + if not isinstance(logs, list): + if isinstance(logs, PurePath): + logs = [logs] + print(f"{func_name} info: logs param expects a list argument, converted to list[Path].") + else: + raise ValueError(f"{func_name} - invalid argument for logs parameter.\n \ + Expect list[Path] or single Path obj, received {type(logs)}") + + # Quality checks - verify valid dir(s), that every item in list is Path object, and that log_name exists in each dir + for i, dir in enumerate(logs): + if not isinstance(dir, PurePath): + raise ValueError(f"{func_name} - non-Path object in logs argument of {type(dir)}: \n{dir}") + if not dir.exists(): + raise ValueError(f"{func_name} - invalid directory in logs argument:\n{dir}") + # verify log_name exists + fn = Path(dir / log_name) + if not fn.exists(): + print(f"-> missing {log_name}. Have you gotten to Epoch 1 in training?") + print(f"--> full path of missing log file: {fn}") + return + + # load log file(s) and plot + dfs = [pd.read_json(Path(p) / log_name, lines=True) for p in logs] + + fig, axs = plt.subplots(ncols=len(fields), figsize=(16, 5)) + + for df, color in zip(dfs, sns.color_palette(n_colors=len(logs))): + for j, field in enumerate(fields): + if field == 'mAP': + coco_eval = pd.DataFrame( + np.stack(df.test_coco_eval_bbox.dropna().values)[:, 1] + ).ewm(com=ewm_col).mean() + axs[j].plot(coco_eval, c=color) + else: + df.interpolate().ewm(com=ewm_col).mean().plot( + y=[f'train_{field}', f'test_{field}'], + ax=axs[j], + color=[color] * 2, + style=['-', '--'] + ) + for ax, field in zip(axs, fields): + ax.legend([Path(p).name for p in logs]) + ax.set_title(field) + + +def plot_precision_recall(files, naming_scheme='iter'): + if naming_scheme == 'exp_id': + # name becomes exp_id + names = [f.parts[-3] for f in files] + elif naming_scheme == 'iter': + names = [f.stem for f in files] + else: + raise ValueError(f'not supported {naming_scheme}') + fig, axs = plt.subplots(ncols=2, figsize=(16, 5)) + for f, color, name in zip(files, sns.color_palette("Blues", n_colors=len(files)), names): + data = torch.load(f) + # precision is n_iou, n_points, n_cat, n_area, max_det + precision = data['precision'] + recall = data['params'].recThrs + scores = data['scores'] + # take precision for all classes, all areas and 100 detections + precision = precision[0, :, :, 0, -1].mean(1) + scores = scores[0, :, :, 0, -1].mean(1) + prec = precision.mean() + rec = data['recall'][0, :, 0, -1].mean() + print(f'{naming_scheme} {name}: mAP@50={prec * 100: 05.1f}, ' + + f'score={scores.mean():0.3f}, ' + + f'f1={2 * prec * rec / (prec + rec + 1e-8):0.3f}' + ) + axs[0].plot(recall, precision, c=color) + axs[1].plot(recall, scores, c=color) + + axs[0].set_title('Precision / Recall') + axs[0].legend(names) + axs[1].set_title('Scores / Recall') + axs[1].legend(names) + return fig, axs -- Gitee From 5a9e8ca2fe1550d3426b87c64aafb72963c1d87e Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:05:18 +0000 Subject: [PATCH 43/59] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PyTorch/contrib/cv/detection/DETR/models/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 PyTorch/contrib/cv/detection/DETR/models/.keep diff --git a/PyTorch/contrib/cv/detection/DETR/models/.keep b/PyTorch/contrib/cv/detection/DETR/models/.keep new file mode 100644 index 0000000000..e69de29bb2 -- Gitee From 288fd90d007b7e8bbeb46038509ff3da000d4097 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:05:36 +0000 Subject: [PATCH 44/59] my first commit --- .../cv/detection/DETR/models/__init__.py | 6 + .../cv/detection/DETR/models/backbone.py | 118 ++++++ .../contrib/cv/detection/DETR/models/detr.py | 368 ++++++++++++++++++ .../cv/detection/DETR/models/matcher.py | 95 +++++ .../DETR/models/position_encoding.py | 91 +++++ .../cv/detection/DETR/models/segmentation.py | 363 +++++++++++++++++ .../cv/detection/DETR/models/transformer.py | 297 ++++++++++++++ 7 files changed, 1338 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__init__.py create mode 100644 PyTorch/contrib/cv/detection/DETR/models/backbone.py create mode 100644 PyTorch/contrib/cv/detection/DETR/models/detr.py create mode 100644 PyTorch/contrib/cv/detection/DETR/models/matcher.py create mode 100644 PyTorch/contrib/cv/detection/DETR/models/position_encoding.py create mode 100644 PyTorch/contrib/cv/detection/DETR/models/segmentation.py create mode 100644 PyTorch/contrib/cv/detection/DETR/models/transformer.py diff --git a/PyTorch/contrib/cv/detection/DETR/models/__init__.py b/PyTorch/contrib/cv/detection/DETR/models/__init__.py new file mode 100644 index 0000000000..a3f26531be --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/models/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from .detr import build + + +def build_model(args): + return build(args) diff --git a/PyTorch/contrib/cv/detection/DETR/models/backbone.py b/PyTorch/contrib/cv/detection/DETR/models/backbone.py new file mode 100644 index 0000000000..9976e2e358 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/models/backbone.py @@ -0,0 +1,118 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Backbone modules. +""" +from collections import OrderedDict + +import torch +import torch.nn.functional as F +import torchvision +from torch import nn +from torchvision.models._utils import IntermediateLayerGetter +from typing import Dict, List + +from util.misc import NestedTensor, is_main_process + +from .position_encoding import build_position_encoding + + +class FrozenBatchNorm2d(torch.nn.Module): + """ + BatchNorm2d where the batch statistics and the affine parameters are fixed. + + Copy-paste from torchvision.misc.ops with added eps before rqsrt, + without which any other models than torchvision.models.resnet[18,34,50,101] + produce nans. + """ + + def __init__(self, n): + super(FrozenBatchNorm2d, self).__init__() + self.register_buffer("weight", torch.ones(n)) + self.register_buffer("bias", torch.zeros(n)) + self.register_buffer("running_mean", torch.zeros(n)) + self.register_buffer("running_var", torch.ones(n)) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + num_batches_tracked_key = prefix + 'num_batches_tracked' + if num_batches_tracked_key in state_dict: + del state_dict[num_batches_tracked_key] + + super(FrozenBatchNorm2d, self)._load_from_state_dict( + state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs) + + def forward(self, x): + # move reshapes to the beginning + # to make it fuser-friendly + w = self.weight.reshape(1, -1, 1, 1) + b = self.bias.reshape(1, -1, 1, 1) + rv = self.running_var.reshape(1, -1, 1, 1) + rm = self.running_mean.reshape(1, -1, 1, 1) + eps = 1e-5 + scale = w * (rv + eps).rsqrt() + bias = b - rm * scale + return x * scale + bias + + +class BackboneBase(nn.Module): + + def __init__(self, backbone: nn.Module, train_backbone: bool, num_channels: int, return_interm_layers: bool): + super().__init__() + for name, parameter in backbone.named_parameters(): + if not train_backbone or 'layer2' not in name and 'layer3' not in name and 'layer4' not in name: + parameter.requires_grad_(False) + if return_interm_layers: + return_layers = {"layer1": "0", "layer2": "1", "layer3": "2", "layer4": "3"} + else: + return_layers = {'layer4': "0"} + self.body = IntermediateLayerGetter(backbone, return_layers=return_layers) + self.num_channels = num_channels + + def forward(self, tensor_list: NestedTensor): + xs = self.body(tensor_list.tensors) + out: Dict[str, NestedTensor] = {} + for name, x in xs.items(): + m = tensor_list.mask + assert m is not None + mask = F.interpolate(m[None].float(), size=x.shape[-2:]).to(torch.bool)[0] + out[name] = NestedTensor(x, mask) + return out + + +class Backbone(BackboneBase): + """ResNet backbone with frozen BatchNorm.""" + def __init__(self, name: str, + train_backbone: bool, + return_interm_layers: bool, + dilation: bool): + backbone = getattr(torchvision.models, name)( + replace_stride_with_dilation=[False, False, dilation], + pretrained=is_main_process(), norm_layer=FrozenBatchNorm2d) + num_channels = 512 if name in ('resnet18', 'resnet34') else 2048 + super().__init__(backbone, train_backbone, num_channels, return_interm_layers) + + +class Joiner(nn.Sequential): + def __init__(self, backbone, position_embedding): + super().__init__(backbone, position_embedding) + + def forward(self, tensor_list: NestedTensor): + xs = self[0](tensor_list) + out: List[NestedTensor] = [] + pos = [] + for name, x in xs.items(): + out.append(x) + # position encoding + pos.append(self[1](x).to(x.tensors.dtype)) + return out, pos + + +def build_backbone(args): + position_embedding = build_position_encoding(args) + train_backbone = args.lr_backbone > 0 + return_interm_layers = args.masks + backbone = Backbone(args.backbone, train_backbone, return_interm_layers, args.dilation) + model = Joiner(backbone, position_embedding) + model.num_channels = backbone.num_channels + return model diff --git a/PyTorch/contrib/cv/detection/DETR/models/detr.py b/PyTorch/contrib/cv/detection/DETR/models/detr.py new file mode 100644 index 0000000000..44209c8242 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/models/detr.py @@ -0,0 +1,368 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +DETR model and criterion classes. +""" +import torch +import torch.nn.functional as F +from torch import nn + +from util import box_ops +from util.misc import (NestedTensor, nested_tensor_from_tensor_list,_onnx_nested_tensor_from_tensor_list, + accuracy, get_world_size, interpolate, + is_dist_avail_and_initialized) + +from .backbone import build_backbone +from .matcher import build_matcher +from .segmentation import (DETRsegm, PostProcessPanoptic, PostProcessSegm, + dice_loss, sigmoid_focal_loss) +from .transformer import build_transformer + + +class DETR(nn.Module): + """ This is the DETR module that performs object detection """ + def __init__(self, backbone, transformer, num_classes, num_queries, aux_loss=False): + """ Initializes the model. + Parameters: + backbone: torch module of the backbone to be used. See backbone.py + transformer: torch module of the transformer architecture. See transformer.py + num_classes: number of object classes + num_queries: number of object queries, ie detection slot. This is the maximal number of objects + DETR can detect in a single image. For COCO, we recommend 100 queries. + aux_loss: True if auxiliary decoding losses (loss at each decoder layer) are to be used. + """ + super().__init__() + self.num_queries = num_queries + self.transformer = transformer + hidden_dim = transformer.d_model + self.class_embed = nn.Linear(hidden_dim, num_classes + 1) + self.bbox_embed = MLP(hidden_dim, hidden_dim, 4, 3) + self.query_embed = nn.Embedding(num_queries, hidden_dim) + self.input_proj = nn.Conv2d(backbone.num_channels, hidden_dim, kernel_size=1) + self.backbone = backbone + self.aux_loss = aux_loss + + def forward(self, samples: NestedTensor): + """ The forward expects a NestedTensor, which consists of: + - samples.tensor: batched images, of shape [batch_size x 3 x H x W] + - samples.mask: a binary mask of shape [batch_size x H x W], containing 1 on padded pixels + + It returns a dict with the following elements: + - "pred_logits": the classification logits (including no-object) for all queries. + Shape= [batch_size x num_queries x (num_classes + 1)] + - "pred_boxes": The normalized boxes coordinates for all queries, represented as + (center_x, center_y, height, width). These values are normalized in [0, 1], + relative to the size of each individual image (disregarding possible padding). + See PostProcess for information on how to retrieve the unnormalized bounding box. + - "aux_outputs": Optional, only returned when auxilary losses are activated. It is a list of + dictionnaries containing the two above keys for each decoder layer. + """ + if isinstance(samples, (list, torch.Tensor)): + samples = nested_tensor_from_tensor_list(samples) + features, pos = self.backbone(samples) + src, mask = features[-1].decompose() + assert mask is not None + hs = self.transformer(self.input_proj(src), mask, self.query_embed.weight, pos[-1])[0] + outputs_class = self.class_embed(hs) + outputs_coord = self.bbox_embed(hs).sigmoid() + out = {'pred_logits': outputs_class[-1], 'pred_boxes': outputs_coord[-1]} + if self.aux_loss: + out['aux_outputs'] = self._set_aux_loss(outputs_class, outputs_coord) + return out + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_coord): + # this is a workaround to make torchscript happy, as torchscript + # doesn't support dictionary with non-homogeneous values, such + # as a dict having both a Tensor and a list. + return [{'pred_logits': a, 'pred_boxes': b} + for a, b in zip(outputs_class[:-1], outputs_coord[:-1])] + + +class SetCriterion(nn.Module): + """ This class computes the loss for DETR. + The process happens in two steps: + 1) we compute hungarian assignment between ground truth boxes and the outputs of the model + 2) we supervise each pair of matched ground-truth / prediction (supervise class and box) + """ + def __init__(self, num_classes, matcher, weight_dict, eos_coef, losses): + """ Create the criterion. + Parameters: + num_classes: number of object categories, omitting the special no-object category + matcher: module able to compute a matching between targets and proposals + weight_dict: dict containing as key the names of the losses and as values their relative weight. + eos_coef: relative classification weight applied to the no-object category + losses: list of all the losses to be applied. See get_loss for list of available losses. + """ + super().__init__() + self.num_classes = num_classes + self.matcher = matcher + self.weight_dict = weight_dict + self.eos_coef = eos_coef + self.losses = losses + empty_weight = torch.ones(self.num_classes + 1) + empty_weight[-1] = self.eos_coef + self.register_buffer('empty_weight', empty_weight) + + def loss_labels(self, outputs, targets, indices, num_boxes, log=True): + """Classification loss (NLL) + targets dicts must contain the key "labels" containing a tensor of dim [nb_target_boxes] + """ + assert 'pred_logits' in outputs + src_logits = outputs['pred_logits'] + + idx = self._get_src_permutation_idx(indices) + target_classes_o = torch.cat([t["labels"][J].to('cpu') for t, (_, J) in zip(targets, indices)]).to('npu') + target_classes = torch.full(src_logits.shape[:2], self.num_classes, + dtype=torch.int64, device=src_logits.device) + target_classes[idx] = target_classes_o + + loss_ce = F.cross_entropy(src_logits.transpose(1, 2), target_classes, self.empty_weight) + losses = {'loss_ce': loss_ce} + + if log: + # TODO this should probably be a separate loss, not hacked in this one here + losses['class_error'] = 100 - accuracy(src_logits[idx], target_classes_o)[0] + return losses + + @torch.no_grad() + def loss_cardinality(self, outputs, targets, indices, num_boxes): + """ Compute the cardinality error, ie the absolute error in the number of predicted non-empty boxes + This is not really a loss, it is intended for logging purposes only. It doesn't propagate gradients + """ + pred_logits = outputs['pred_logits'] + device = pred_logits.device + tgt_lengths = torch.as_tensor([len(v["labels"]) for v in targets], device=device) + # Count the number of predictions that are NOT "no-object" (which is the last class) + card_pred = (pred_logits.argmax(-1) != pred_logits.shape[-1] - 1).sum(1) + card_err = F.l1_loss(card_pred.float(), tgt_lengths.float()) + losses = {'cardinality_error': card_err} + return losses + + def loss_boxes(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the bounding boxes, the L1 regression loss and the GIoU loss + targets dicts must contain the key "boxes" containing a tensor of dim [nb_target_boxes, 4] + The target boxes are expected in format (center_x, center_y, w, h), normalized by the image size. + """ + assert 'pred_boxes' in outputs + idx = self._get_src_permutation_idx(indices) + src_boxes = outputs['pred_boxes'][idx] + target_boxes = torch.cat([t['boxes'][i].to('cpu') for t, (_, i) in zip(targets, indices)], dim=0).to('npu') + + loss_bbox = F.l1_loss(src_boxes, target_boxes, reduction='none') + + losses = {} + losses['loss_bbox'] = loss_bbox.sum() / num_boxes + + loss_giou = 1 - torch.diag(box_ops.generalized_box_iou( + box_ops.box_cxcywh_to_xyxy(src_boxes), + box_ops.box_cxcywh_to_xyxy(target_boxes))) + losses['loss_giou'] = loss_giou.sum() / num_boxes + return losses + + def loss_masks(self, outputs, targets, indices, num_boxes): + """Compute the losses related to the masks: the focal loss and the dice loss. + targets dicts must contain the key "masks" containing a tensor of dim [nb_target_boxes, h, w] + """ + assert "pred_masks" in outputs + + src_idx = self._get_src_permutation_idx(indices) + tgt_idx = self._get_tgt_permutation_idx(indices) + src_masks = outputs["pred_masks"] + src_masks = src_masks[src_idx] + masks = [t["masks"] for t in targets] + # TODO use valid to mask invalid areas due to padding in loss + target_masks, valid = nested_tensor_from_tensor_list(masks).decompose() + target_masks = target_masks.to(src_masks) + target_masks = target_masks[tgt_idx] + + # upsample predictions to the target size + src_masks = interpolate(src_masks[:, None], size=target_masks.shape[-2:], + mode="bilinear", align_corners=False) + src_masks = src_masks[:, 0].flatten(1) + + target_masks = target_masks.flatten(1) + target_masks = target_masks.view(src_masks.shape) + losses = { + "loss_mask": sigmoid_focal_loss(src_masks, target_masks, num_boxes), + "loss_dice": dice_loss(src_masks, target_masks, num_boxes), + } + return losses + + def _get_src_permutation_idx(self, indices): + # permute predictions following indices + batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) + src_idx = torch.cat([src for (src, _) in indices]) + return batch_idx, src_idx + + def _get_tgt_permutation_idx(self, indices): + # permute targets following indices + batch_idx = torch.cat([torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) + tgt_idx = torch.cat([tgt for (_, tgt) in indices]) + return batch_idx, tgt_idx + + def get_loss(self, loss, outputs, targets, indices, num_boxes, **kwargs): + loss_map = { + 'labels': self.loss_labels, + 'cardinality': self.loss_cardinality, + 'boxes': self.loss_boxes, + 'masks': self.loss_masks + } + assert loss in loss_map, f'do you really want to compute {loss} loss?' + return loss_map[loss](outputs, targets, indices, num_boxes, **kwargs) + + def forward(self, outputs, targets): + """ This performs the loss computation. + Parameters: + outputs: dict of tensors, see the output specification of the model for the format + targets: list of dicts, such that len(targets) == batch_size. + The expected keys in each dict depends on the losses applied, see each loss' doc + """ + outputs_without_aux = {k: v for k, v in outputs.items() if k != 'aux_outputs'} + + # Retrieve the matching between the outputs of the last layer and the targets + indices = self.matcher(outputs_without_aux, targets) + + # Compute the average number of target boxes accross all nodes, for normalization purposes + num_boxes = sum(len(t["labels"]) for t in targets) + num_boxes = torch.as_tensor([num_boxes], dtype=torch.float, device=next(iter(outputs.values())).device) + if is_dist_avail_and_initialized(): + torch.distributed.all_reduce(num_boxes) + num_boxes = torch.clamp(num_boxes / get_world_size(), min=1).item() + + # Compute all the requested losses + losses = {} + for loss in self.losses: + losses.update(self.get_loss(loss, outputs, targets, indices, num_boxes)) + + # In case of auxiliary losses, we repeat this process with the output of each intermediate layer. + if 'aux_outputs' in outputs: + for i, aux_outputs in enumerate(outputs['aux_outputs']): + indices = self.matcher(aux_outputs, targets) + for loss in self.losses: + if loss == 'masks': + # Intermediate masks losses are too costly to compute, we ignore them. + continue + kwargs = {} + if loss == 'labels': + # Logging is enabled only for the last layer + kwargs = {'log': False} + l_dict = self.get_loss(loss, aux_outputs, targets, indices, num_boxes, **kwargs) + l_dict = {k + f'_{i}': v for k, v in l_dict.items()} + losses.update(l_dict) + + return losses + + +class PostProcess(nn.Module): + """ This module converts the model's output into the format expected by the coco api""" + @torch.no_grad() + def forward(self, outputs, target_sizes): + """ Perform the computation + Parameters: + outputs: raw outputs of the model + target_sizes: tensor of dimension [batch_size x 2] containing the size of each images of the batch + For evaluation, this must be the original image size (before any data augmentation) + For visualization, this should be the image size after data augment, but before padding + """ + out_logits, out_bbox = outputs['pred_logits'], outputs['pred_boxes'] + + assert len(out_logits) == len(target_sizes) + assert target_sizes.shape[1] == 2 + + prob = F.softmax(out_logits, -1) + scores, labels = prob[..., :-1].max(-1) + + # convert to [x0, y0, x1, y1] format + boxes = box_ops.box_cxcywh_to_xyxy(out_bbox) + # and from relative [0, 1] to absolute [0, height] coordinates + img_h, img_w = target_sizes.unbind(1) + scale_fct = torch.stack([img_w, img_h, img_w, img_h], dim=1) + sc = torch.ones_like(scale_fct) + value = torch.max(scale_fct.half(), dim=1)[0] + value = torch.unsqueeze(value, dim=0).t() + scale_fct_value = sc * value + sc_value = (scale_fct_value - scale_fct) / 2 + sc_ex = torch.unsqueeze(sc_value, dim=1) + boxes = boxes * scale_fct_value[:, None, :] + boxes_one = torch.ones_like(boxes) + boxex_ten = boxes_one * sc_ex + boxes = boxes - boxex_ten + + results = [{'scores': s, 'labels': l, 'boxes': b} for s, l, b in zip(scores, labels, boxes)] + + return results + + +class MLP(nn.Module): + """ Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +def build(args): + # the `num_classes` naming here is somewhat misleading. + # it indeed corresponds to `max_obj_id + 1`, where max_obj_id + # is the maximum id for a class in your dataset. For example, + # COCO has a max_obj_id of 90, so we pass `num_classes` to be 91. + # As another example, for a dataset that has a single class with id 1, + # you should pass `num_classes` to be 2 (max_obj_id + 1). + # For more details on this, check the following discussion + # https://github.com/facebookresearch/detr/issues/108#issuecomment-650269223 + num_classes = 20 if args.dataset_file != 'coco' else 91 + if args.dataset_file == "coco_panoptic": + # for panoptic, we just add a num_classes that is large enough to hold + # max_obj_id + 1, but the exact value doesn't really matter + num_classes = 250 + device = torch.device(args.device) + + backbone = build_backbone(args) + + transformer = build_transformer(args) + + model = DETR( + backbone, + transformer, + num_classes=num_classes, + num_queries=args.num_queries, + aux_loss=args.aux_loss, + ) + if args.masks: + model = DETRsegm(model, freeze_detr=(args.frozen_weights is not None)) + matcher = build_matcher(args) + weight_dict = {'loss_ce': 1, 'loss_bbox': args.bbox_loss_coef} + weight_dict['loss_giou'] = args.giou_loss_coef + if args.masks: + weight_dict["loss_mask"] = args.mask_loss_coef + weight_dict["loss_dice"] = args.dice_loss_coef + # TODO this is a hack + if args.aux_loss: + aux_weight_dict = {} + for i in range(args.dec_layers - 1): + aux_weight_dict.update({k + f'_{i}': v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + + losses = ['labels', 'boxes', 'cardinality'] + if args.masks: + losses += ["masks"] + print('losses',losses) + + criterion = SetCriterion(num_classes, matcher=matcher, weight_dict=weight_dict, + eos_coef=args.eos_coef, losses=losses) + criterion.to(device) + postprocessors = {'bbox': PostProcess()} + if args.masks: + postprocessors['segm'] = PostProcessSegm() + if args.dataset_file == "coco_panoptic": + is_thing_map = {i: i <= 90 for i in range(201)} + postprocessors["panoptic"] = PostProcessPanoptic(is_thing_map, threshold=0.85) + + return model, criterion, postprocessors diff --git a/PyTorch/contrib/cv/detection/DETR/models/matcher.py b/PyTorch/contrib/cv/detection/DETR/models/matcher.py new file mode 100644 index 0000000000..7a4483e4c4 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/models/matcher.py @@ -0,0 +1,95 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Modules to compute the matching cost and solve the corresponding LSAP. +""" +import torch +from scipy.optimize import linear_sum_assignment +from torch import nn + +from util.box_ops import box_cxcywh_to_xyxy, generalized_box_iou + + +class HungarianMatcher(nn.Module): + """This class computes an assignment between the targets and the predictions of the network + + For efficiency reasons, the targets don't include the no_object. Because of this, in general, + there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, + while the others are un-matched (and thus treated as non-objects). + """ + + def __init__(self, cost_class: float = 1, cost_bbox: float = 1, cost_giou: float = 1): + """Creates the matcher + + Params: + cost_class: This is the relative weight of the classification error in the matching cost + cost_bbox: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost + cost_giou: This is the relative weight of the giou loss of the bounding box in the matching cost + """ + super().__init__() + self.cost_class = cost_class + self.cost_bbox = cost_bbox + self.cost_giou = cost_giou + assert cost_class != 0 or cost_bbox != 0 or cost_giou != 0, "all costs cant be 0" + + @torch.no_grad() + def forward(self, outputs, targets): + """ Performs the matching + + Params: + outputs: This is a dict that contains at least these entries: + "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits + "pred_boxes": Tensor of dim [batch_size, num_queries, 4] with the predicted box coordinates + + targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: + "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth + objects in the target) containing the class labels + "boxes": Tensor of dim [num_target_boxes, 4] containing the target box coordinates + + Returns: + A list of size batch_size, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order) + - index_j is the indices of the corresponding selected targets (in order) + For each batch element, it holds: + len(index_i) = len(index_j) = min(num_queries, num_target_boxes) + """ + bs, num_queries = outputs["pred_logits"].shape[:2] + + # We flatten to compute the cost matrices in a batch + out_prob = outputs["pred_logits"].flatten(0, 1).softmax(-1) # [batch_size * num_queries, num_classes] + out_bbox = outputs["pred_boxes"].flatten(0, 1) # [batch_size * num_queries, 4] + + # Also concat the target labels and boxes + + targets_labels = [v["labels"].to('cpu') for v in targets] + targets_boxes = [v["boxes"].to('cpu') for v in targets] + + tgt_ids = torch.cat(targets_labels).to('npu') + tgt_bbox = torch.cat(targets_boxes).to('npu') + + # tgt_ids = torch.cat([v["labels"] for v in targets]) + # tgt_bbox = torch.cat([v["boxes"] for v in targets]) + + + # Compute the classification cost. Contrary to the loss, we don't use the NLL, + # but approximate it in 1 - proba[target class]. + # The 1 is a constant that doesn't change the matching, it can be ommitted. + cost_class = -out_prob[:, tgt_ids] + + # Compute the L1 cost between boxes + cost_bbox = torch.cdist(out_bbox, tgt_bbox, p=1) + + # Compute the giou cost betwen boxes + + cost_giou = -generalized_box_iou(box_cxcywh_to_xyxy(out_bbox), box_cxcywh_to_xyxy(tgt_bbox)) + + # Final cost matrix + C = self.cost_bbox * cost_bbox + self.cost_class * cost_class + self.cost_giou * cost_giou + C = C.view(bs, num_queries, -1).cpu() + + sizes = [len(v["boxes"]) for v in targets] + indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(sizes, -1))] + return [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] + + +def build_matcher(args): + return HungarianMatcher(cost_class=args.set_cost_class, cost_bbox=args.set_cost_bbox, cost_giou=args.set_cost_giou) diff --git a/PyTorch/contrib/cv/detection/DETR/models/position_encoding.py b/PyTorch/contrib/cv/detection/DETR/models/position_encoding.py new file mode 100644 index 0000000000..6afc9190d4 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/models/position_encoding.py @@ -0,0 +1,91 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Various positional encodings for the transformer. +""" +import math +import torch +from torch import nn + +from util.misc import NestedTensor + + +class PositionEmbeddingSine(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None): + super().__init__() + self.num_pos_feats = num_pos_feats + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError("normalize should be True if scale is passed") + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, tensor_list: NestedTensor): + x = tensor_list.tensors + mask = tensor_list.mask + assert mask is not None + # not_mask = ~mask + not_mask = (~mask).float() + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack((pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3) + pos_y = torch.stack((pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos + + +class PositionEmbeddingLearned(nn.Module): + """ + Absolute pos embedding, learned. + """ + def __init__(self, num_pos_feats=256): + super().__init__() + self.row_embed = nn.Embedding(50, num_pos_feats) + self.col_embed = nn.Embedding(50, num_pos_feats) + self.reset_parameters() + + def reset_parameters(self): + nn.init.uniform_(self.row_embed.weight) + nn.init.uniform_(self.col_embed.weight) + + def forward(self, tensor_list: NestedTensor): + x = tensor_list.tensors + h, w = x.shape[-2:] + i = torch.arange(w, device=x.device) + j = torch.arange(h, device=x.device) + x_emb = self.col_embed(i) + y_emb = self.row_embed(j) + + pos = torch.cat([ + x_emb.unsqueeze(0).repeat(h, 1, 1), + y_emb.unsqueeze(1).repeat(1, w, 1), + ], dim=-1).permute(2, 0, 1).unsqueeze(0).repeat(x.shape[0], 1, 1, 1) + return pos + + +def build_position_encoding(args): + N_steps = args.hidden_dim // 2 + if args.position_embedding in ('v2', 'sine'): + # TODO find a better way of exposing other arguments + position_embedding = PositionEmbeddingSine(N_steps, normalize=True) + elif args.position_embedding in ('v3', 'learned'): + position_embedding = PositionEmbeddingLearned(N_steps) + else: + raise ValueError(f"not supported {args.position_embedding}") + + return position_embedding diff --git a/PyTorch/contrib/cv/detection/DETR/models/segmentation.py b/PyTorch/contrib/cv/detection/DETR/models/segmentation.py new file mode 100644 index 0000000000..01faa88518 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/models/segmentation.py @@ -0,0 +1,363 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +This file provides the definition of the convolutional heads used to predict masks, as well as the losses +""" +import io +from collections import defaultdict +from typing import List, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from PIL import Image + +import util.box_ops as box_ops +from util.misc import NestedTensor, interpolate, nested_tensor_from_tensor_list + +try: + from panopticapi.utils import id2rgb, rgb2id +except ImportError: + pass + + +class DETRsegm(nn.Module): + def __init__(self, detr, freeze_detr=False): + super().__init__() + self.detr = detr + + if freeze_detr: + for p in self.parameters(): + p.requires_grad_(False) + + hidden_dim, nheads = detr.transformer.d_model, detr.transformer.nhead + self.bbox_attention = MHAttentionMap(hidden_dim, hidden_dim, nheads, dropout=0.0) + self.mask_head = MaskHeadSmallConv(hidden_dim + nheads, [1024, 512, 256], hidden_dim) + + def forward(self, samples: NestedTensor): + if isinstance(samples, (list, torch.Tensor)): + samples = nested_tensor_from_tensor_list(samples) + features, pos = self.detr.backbone(samples) + + bs = features[-1].tensors.shape[0] + + src, mask = features[-1].decompose() + assert mask is not None + src_proj = self.detr.input_proj(src) + hs, memory = self.detr.transformer(src_proj, mask, self.detr.query_embed.weight, pos[-1]) + + outputs_class = self.detr.class_embed(hs) + outputs_coord = self.detr.bbox_embed(hs).sigmoid() + out = {"pred_logits": outputs_class[-1], "pred_boxes": outputs_coord[-1]} + if self.detr.aux_loss: + out['aux_outputs'] = self.detr._set_aux_loss(outputs_class, outputs_coord) + + # FIXME h_boxes takes the last one computed, keep this in mind + bbox_mask = self.bbox_attention(hs[-1], memory, mask=mask) + + seg_masks = self.mask_head(src_proj, bbox_mask, [features[2].tensors, features[1].tensors, features[0].tensors]) + outputs_seg_masks = seg_masks.view(bs, self.detr.num_queries, seg_masks.shape[-2], seg_masks.shape[-1]) + + out["pred_masks"] = outputs_seg_masks + return out + + +def _expand(tensor, length: int): + return tensor.unsqueeze(1).repeat(1, int(length), 1, 1, 1).flatten(0, 1) + + +class MaskHeadSmallConv(nn.Module): + """ + Simple convolutional head, using group norm. + Upsampling is done using a FPN approach + """ + + def __init__(self, dim, fpn_dims, context_dim): + super().__init__() + + inter_dims = [dim, context_dim // 2, context_dim // 4, context_dim // 8, context_dim // 16, context_dim // 64] + self.lay1 = torch.nn.Conv2d(dim, dim, 3, padding=1) + self.gn1 = torch.nn.GroupNorm(8, dim) + self.lay2 = torch.nn.Conv2d(dim, inter_dims[1], 3, padding=1) + self.gn2 = torch.nn.GroupNorm(8, inter_dims[1]) + self.lay3 = torch.nn.Conv2d(inter_dims[1], inter_dims[2], 3, padding=1) + self.gn3 = torch.nn.GroupNorm(8, inter_dims[2]) + self.lay4 = torch.nn.Conv2d(inter_dims[2], inter_dims[3], 3, padding=1) + self.gn4 = torch.nn.GroupNorm(8, inter_dims[3]) + self.lay5 = torch.nn.Conv2d(inter_dims[3], inter_dims[4], 3, padding=1) + self.gn5 = torch.nn.GroupNorm(8, inter_dims[4]) + self.out_lay = torch.nn.Conv2d(inter_dims[4], 1, 3, padding=1) + + self.dim = dim + + self.adapter1 = torch.nn.Conv2d(fpn_dims[0], inter_dims[1], 1) + self.adapter2 = torch.nn.Conv2d(fpn_dims[1], inter_dims[2], 1) + self.adapter3 = torch.nn.Conv2d(fpn_dims[2], inter_dims[3], 1) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_uniform_(m.weight, a=1) + nn.init.constant_(m.bias, 0) + + def forward(self, x: Tensor, bbox_mask: Tensor, fpns: List[Tensor]): + x = torch.cat([_expand(x, bbox_mask.shape[1]), bbox_mask.flatten(0, 1)], 1) + + x = self.lay1(x) + x = self.gn1(x) + x = F.relu(x) + x = self.lay2(x) + x = self.gn2(x) + x = F.relu(x) + + cur_fpn = self.adapter1(fpns[0]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = _expand(cur_fpn, x.size(0) // cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay3(x) + x = self.gn3(x) + x = F.relu(x) + + cur_fpn = self.adapter2(fpns[1]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = _expand(cur_fpn, x.size(0) // cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay4(x) + x = self.gn4(x) + x = F.relu(x) + + cur_fpn = self.adapter3(fpns[2]) + if cur_fpn.size(0) != x.size(0): + cur_fpn = _expand(cur_fpn, x.size(0) // cur_fpn.size(0)) + x = cur_fpn + F.interpolate(x, size=cur_fpn.shape[-2:], mode="nearest") + x = self.lay5(x) + x = self.gn5(x) + x = F.relu(x) + + x = self.out_lay(x) + return x + + +class MHAttentionMap(nn.Module): + """This is a 2D attention module, which only returns the attention softmax (no multiplication by value)""" + + def __init__(self, query_dim, hidden_dim, num_heads, dropout=0.0, bias=True): + super().__init__() + self.num_heads = num_heads + self.hidden_dim = hidden_dim + self.dropout = nn.Dropout(dropout) + + self.q_linear = nn.Linear(query_dim, hidden_dim, bias=bias) + self.k_linear = nn.Linear(query_dim, hidden_dim, bias=bias) + + nn.init.zeros_(self.k_linear.bias) + nn.init.zeros_(self.q_linear.bias) + nn.init.xavier_uniform_(self.k_linear.weight) + nn.init.xavier_uniform_(self.q_linear.weight) + self.normalize_fact = float(hidden_dim / self.num_heads) ** -0.5 + + def forward(self, q, k, mask: Optional[Tensor] = None): + q = self.q_linear(q) + k = F.conv2d(k, self.k_linear.weight.unsqueeze(-1).unsqueeze(-1), self.k_linear.bias) + qh = q.view(q.shape[0], q.shape[1], self.num_heads, self.hidden_dim // self.num_heads) + kh = k.view(k.shape[0], self.num_heads, self.hidden_dim // self.num_heads, k.shape[-2], k.shape[-1]) + weights = torch.einsum("bqnc,bnchw->bqnhw", qh * self.normalize_fact, kh) + + if mask is not None: + weights.masked_fill_(mask.unsqueeze(1).unsqueeze(1), float("-inf")) + weights = F.softmax(weights.flatten(2), dim=-1).view(weights.size()) + weights = self.dropout(weights) + return weights + + +def dice_loss(inputs, targets, num_boxes): + """ + Compute the DICE loss, similar to generalized IOU for masks + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + """ + inputs = inputs.sigmoid() + inputs = inputs.flatten(1) + numerator = 2 * (inputs * targets).sum(1) + denominator = inputs.sum(-1) + targets.sum(-1) + loss = 1 - (numerator + 1) / (denominator + 1) + return loss.sum() / num_boxes + + +def sigmoid_focal_loss(inputs, targets, num_boxes, alpha: float = 0.25, gamma: float = 2): + """ + Loss used in RetinaNet for dense detection: https://arxiv.org/abs/1708.02002. + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + alpha: (optional) Weighting factor in range (0,1) to balance + positive vs negative examples. Default = -1 (no weighting). + gamma: Exponent of the modulating factor (1 - p_t) to + balance easy vs hard examples. + Returns: + Loss tensor + """ + prob = inputs.sigmoid() + ce_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduction="none") + p_t = prob * targets + (1 - prob) * (1 - targets) + loss = ce_loss * ((1 - p_t) ** gamma) + + if alpha >= 0: + alpha_t = alpha * targets + (1 - alpha) * (1 - targets) + loss = alpha_t * loss + + return loss.mean(1).sum() / num_boxes + + +class PostProcessSegm(nn.Module): + def __init__(self, threshold=0.5): + super().__init__() + self.threshold = threshold + + @torch.no_grad() + def forward(self, results, outputs, orig_target_sizes, max_target_sizes): + assert len(orig_target_sizes) == len(max_target_sizes) + max_h, max_w = max_target_sizes.max(0)[0].tolist() + outputs_masks = outputs["pred_masks"].squeeze(2) + outputs_masks = F.interpolate(outputs_masks, size=(max_h, max_w), mode="bilinear", align_corners=False) + outputs_masks = (outputs_masks.sigmoid() > self.threshold).cpu() + + for i, (cur_mask, t, tt) in enumerate(zip(outputs_masks, max_target_sizes, orig_target_sizes)): + img_h, img_w = t[0], t[1] + results[i]["masks"] = cur_mask[:, :img_h, :img_w].unsqueeze(1) + results[i]["masks"] = F.interpolate( + results[i]["masks"].float(), size=tuple(tt.tolist()), mode="nearest" + ).byte() + + return results + + +class PostProcessPanoptic(nn.Module): + """This class converts the output of the model to the final panoptic result, in the format expected by the + coco panoptic API """ + + def __init__(self, is_thing_map, threshold=0.85): + """ + Parameters: + is_thing_map: This is a whose keys are the class ids, and the values a boolean indicating whether + the class is a thing (True) or a stuff (False) class + threshold: confidence threshold: segments with confidence lower than this will be deleted + """ + super().__init__() + self.threshold = threshold + self.is_thing_map = is_thing_map + + def forward(self, outputs, processed_sizes, target_sizes=None): + """ This function computes the panoptic prediction from the model's predictions. + Parameters: + outputs: This is a dict coming directly from the model. See the model doc for the content. + processed_sizes: This is a list of tuples (or torch tensors) of sizes of the images that were passed to the + model, ie the size after data augmentation but before batching. + target_sizes: This is a list of tuples (or torch tensors) corresponding to the requested final size + of each prediction. If left to None, it will default to the processed_sizes + """ + if target_sizes is None: + target_sizes = processed_sizes + assert len(processed_sizes) == len(target_sizes) + out_logits, raw_masks, raw_boxes = outputs["pred_logits"], outputs["pred_masks"], outputs["pred_boxes"] + assert len(out_logits) == len(raw_masks) == len(target_sizes) + preds = [] + + def to_tuple(tup): + if isinstance(tup, tuple): + return tup + return tuple(tup.cpu().tolist()) + + for cur_logits, cur_masks, cur_boxes, size, target_size in zip( + out_logits, raw_masks, raw_boxes, processed_sizes, target_sizes + ): + # we filter empty queries and detection below threshold + scores, labels = cur_logits.softmax(-1).max(-1) + keep = labels.ne(outputs["pred_logits"].shape[-1] - 1) & (scores > self.threshold) + cur_scores, cur_classes = cur_logits.softmax(-1).max(-1) + cur_scores = cur_scores[keep] + cur_classes = cur_classes[keep] + cur_masks = cur_masks[keep] + cur_masks = interpolate(cur_masks[:, None], to_tuple(size), mode="bilinear").squeeze(1) + cur_boxes = box_ops.box_cxcywh_to_xyxy(cur_boxes[keep]) + + h, w = cur_masks.shape[-2:] + assert len(cur_boxes) == len(cur_classes) + + # It may be that we have several predicted masks for the same stuff class. + # In the following, we track the list of masks ids for each stuff class (they are merged later on) + cur_masks = cur_masks.flatten(1) + stuff_equiv_classes = defaultdict(lambda: []) + for k, label in enumerate(cur_classes): + if not self.is_thing_map[label.item()]: + stuff_equiv_classes[label.item()].append(k) + + def get_ids_area(masks, scores, dedup=False): + # This helper function creates the final panoptic segmentation image + # It also returns the area of the masks that appears on the image + + m_id = masks.transpose(0, 1).softmax(-1) + + if m_id.shape[-1] == 0: + # We didn't detect any mask :( + m_id = torch.zeros((h, w), dtype=torch.long, device=m_id.device) + else: + m_id = m_id.argmax(-1).view(h, w) + + if dedup: + # Merge the masks corresponding to the same stuff class + for equiv in stuff_equiv_classes.values(): + if len(equiv) > 1: + for eq_id in equiv: + m_id.masked_fill_(m_id.eq(eq_id), equiv[0]) + + final_h, final_w = to_tuple(target_size) + + seg_img = Image.fromarray(id2rgb(m_id.view(h, w).cpu().numpy())) + seg_img = seg_img.resize(size=(final_w, final_h), resample=Image.NEAREST) + + np_seg_img = ( + torch.ByteTensor(torch.ByteStorage.from_buffer(seg_img.tobytes())).view(final_h, final_w, 3).numpy() + ) + m_id = torch.from_numpy(rgb2id(np_seg_img)) + + area = [] + for i in range(len(scores)): + area.append(m_id.eq(i).sum().item()) + return area, seg_img + + area, seg_img = get_ids_area(cur_masks, cur_scores, dedup=True) + if cur_classes.numel() > 0: + # We know filter empty masks as long as we find some + while True: + filtered_small = torch.as_tensor( + [area[i] <= 4 for i, c in enumerate(cur_classes)], dtype=torch.bool, device=keep.device + ) + if filtered_small.any().item(): + cur_scores = cur_scores[~filtered_small] + cur_classes = cur_classes[~filtered_small] + cur_masks = cur_masks[~filtered_small] + area, seg_img = get_ids_area(cur_masks, cur_scores) + else: + break + + else: + cur_classes = torch.ones(1, dtype=torch.long, device=cur_classes.device) + + segments_info = [] + for i, a in enumerate(area): + cat = cur_classes[i].item() + segments_info.append({"id": i, "isthing": self.is_thing_map[cat], "category_id": cat, "area": a}) + del cur_classes + + with io.BytesIO() as out: + seg_img.save(out, format="PNG") + predictions = {"png_string": out.getvalue(), "segments_info": segments_info} + preds.append(predictions) + return preds diff --git a/PyTorch/contrib/cv/detection/DETR/models/transformer.py b/PyTorch/contrib/cv/detection/DETR/models/transformer.py new file mode 100644 index 0000000000..dcd536750a --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/models/transformer.py @@ -0,0 +1,297 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +DETR Transformer class. + +Copy-paste from torch.nn.Transformer with modifications: + * positional encodings are passed in MHattention + * extra LN at the end of encoder is removed + * decoder returns a stack of activations from all decoding layers +""" +import copy +from typing import Optional, List + +import torch +import torch.nn.functional as F +from torch import nn, Tensor + + +class Transformer(nn.Module): + + def __init__(self, d_model=512, nhead=8, num_encoder_layers=6, + num_decoder_layers=6, dim_feedforward=2048, dropout=0.1, + activation="relu", normalize_before=False, + return_intermediate_dec=False): + super().__init__() + + encoder_layer = TransformerEncoderLayer(d_model, nhead, dim_feedforward, + dropout, activation, normalize_before) + encoder_norm = nn.LayerNorm(d_model) if normalize_before else None + self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm) + + decoder_layer = TransformerDecoderLayer(d_model, nhead, dim_feedforward, + dropout, activation, normalize_before) + decoder_norm = nn.LayerNorm(d_model) + self.decoder = TransformerDecoder(decoder_layer, num_decoder_layers, decoder_norm, + return_intermediate=return_intermediate_dec) + + self._reset_parameters() + + self.d_model = d_model + self.nhead = nhead + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def forward(self, src, mask, query_embed, pos_embed): + # flatten NxCxHxW to HWxNxC + bs, c, h, w = src.shape + src = src.flatten(2).permute(2, 0, 1) + pos_embed = pos_embed.flatten(2).permute(2, 0, 1) + query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1) + mask = mask.flatten(1) + + tgt = torch.zeros_like(query_embed) + memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed) + hs = self.decoder(tgt, memory, memory_key_padding_mask=mask, + pos=pos_embed, query_pos=query_embed) + return hs.transpose(1, 2), memory.permute(1, 2, 0).view(bs, c, h, w) + + +class TransformerEncoder(nn.Module): + + def __init__(self, encoder_layer, num_layers, norm=None): + super().__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + def forward(self, src, + mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + output = src + + for layer in self.layers: + output = layer(output, src_mask=mask, + src_key_padding_mask=src_key_padding_mask, pos=pos) + + if self.norm is not None: + output = self.norm(output) + + return output + + +class TransformerDecoder(nn.Module): + + def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False): + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + self.return_intermediate = return_intermediate + + def forward(self, tgt, memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + output = tgt + + intermediate = [] + + for layer in self.layers: + output = layer(output, memory, tgt_mask=tgt_mask, + memory_mask=memory_mask, + tgt_key_padding_mask=tgt_key_padding_mask, + memory_key_padding_mask=memory_key_padding_mask, + pos=pos, query_pos=query_pos) + if self.return_intermediate: + intermediate.append(self.norm(output)) + + if self.norm is not None: + output = self.norm(output) + if self.return_intermediate: + intermediate.pop() + intermediate.append(output) + + if self.return_intermediate: + return torch.stack(intermediate) + + return output.unsqueeze(0) + + +class TransformerEncoderLayer(nn.Module): + + def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, + activation="relu", normalize_before=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + q = k = self.with_pos_embed(src, pos) + src2 = self.self_attn(q, k, value=src, attn_mask=src_mask, + key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src = self.norm1(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) + src = src + self.dropout2(src2) + src = self.norm2(src) + return src + + def forward_pre(self, src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + src2 = self.norm1(src) + q = k = self.with_pos_embed(src2, pos) + src2 = self.self_attn(q, k, value=src2, attn_mask=src_mask, + key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src2 = self.norm2(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src2)))) + src = src + self.dropout2(src2) + return src + + def forward(self, src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(src, src_mask, src_key_padding_mask, pos) + return self.forward_post(src, src_mask, src_key_padding_mask, pos) + + +class TransformerDecoderLayer(nn.Module): + + def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, + activation="relu", normalize_before=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, tgt, memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + q = k = self.with_pos_embed(tgt, query_pos) + tgt2 = self.self_attn(q, k, value=tgt, attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout3(tgt2) + tgt = self.norm3(tgt) + return tgt + + def forward_pre(self, tgt, memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.norm1(tgt) + q = k = self.with_pos_embed(tgt2, query_pos) + tgt2 = self.self_attn(q, k, value=tgt2, attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout1(tgt2) + tgt2 = self.norm2(tgt) + tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt2, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout2(tgt2) + tgt2 = self.norm3(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2)))) + tgt = tgt + self.dropout3(tgt2) + return tgt + + def forward(self, tgt, memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(tgt, memory, tgt_mask, memory_mask, + tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos) + return self.forward_post(tgt, memory, tgt_mask, memory_mask, + tgt_key_padding_mask, memory_key_padding_mask, pos, query_pos) + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def build_transformer(args): + return Transformer( + d_model=args.hidden_dim, + dropout=args.dropout, + nhead=args.nheads, + dim_feedforward=args.dim_feedforward, + num_encoder_layers=args.enc_layers, + num_decoder_layers=args.dec_layers, + normalize_before=args.pre_norm, + return_intermediate_dec=True, + ) + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == "relu": + return F.relu + if activation == "gelu": + return F.gelu + if activation == "glu": + return F.glu + raise RuntimeError(F"activation should be relu/gelu, not {activation}.") -- Gitee From 5149a6da1c93945011167c3623f6e3e5b00346ff Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:05:42 +0000 Subject: [PATCH 45/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/cv/detection/DETR/models/.keep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PyTorch/contrib/cv/detection/DETR/models/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 PyTorch/contrib/cv/detection/DETR/models/.keep diff --git a/PyTorch/contrib/cv/detection/DETR/models/.keep b/PyTorch/contrib/cv/detection/DETR/models/.keep deleted file mode 100644 index e69de29bb2..0000000000 -- Gitee From a7e62044f5c7e84074906ad55466046e3cc13b16 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:05:55 +0000 Subject: [PATCH 46/59] my first commit --- .../models/__pycache__/__init__.cpython-36.pyc | Bin 0 -> 275 bytes .../models/__pycache__/__init__.cpython-37.pyc | Bin 0 -> 271 bytes .../models/__pycache__/backbone.cpython-36.pyc | Bin 0 -> 4639 bytes .../models/__pycache__/backbone.cpython-37.pyc | Bin 0 -> 4609 bytes .../DETR/models/__pycache__/detr.cpython-36.pyc | Bin 0 -> 15537 bytes .../DETR/models/__pycache__/detr.cpython-37.pyc | Bin 0 -> 15827 bytes .../models/__pycache__/matcher.cpython-36.pyc | Bin 0 -> 4383 bytes .../models/__pycache__/matcher.cpython-37.pyc | Bin 0 -> 4459 bytes .../position_encoding.cpython-36.pyc | Bin 0 -> 3528 bytes .../position_encoding.cpython-37.pyc | Bin 0 -> 3499 bytes .../__pycache__/segmentation.cpython-36.pyc | Bin 0 -> 13199 bytes .../__pycache__/segmentation.cpython-37.pyc | Bin 0 -> 13099 bytes .../__pycache__/transformer.cpython-36.pyc | Bin 0 -> 8981 bytes .../__pycache__/transformer.cpython-37.pyc | Bin 0 -> 8983 bytes 14 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/position_encoding.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/position_encoding.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/segmentation.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/segmentation.cpython-37.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/transformer.cpython-36.pyc create mode 100644 PyTorch/contrib/cv/detection/DETR/models/__pycache__/transformer.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..412367e3f4bbf044d602c58174d344e91abcd382 GIT binary patch literal 275 zcmYjKy9&ZU5Zui}L{0B6r11({5kYM%EY!;8h+fcxJmiu<1UqfT@9>w>TKNlB?p`4- z%gFibfWfr z@$lw^DWPZ+m2iP8k^`3^8kWf9MdnaE!FY6jggEADEb2P(kX4o>H$Yb&KyeD2{?K+_^#hHm4r(N@_n+wwb%7 literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a9664cfcc4178ecfaef6baa75383eab6cf2f20a GIT binary patch literal 271 zcmYjKy9&ZU5Zui}f{)%mkXU#>AR^kQv9md%cf^Cl$R$Dq8_~+f@9>w>TKNlB?j8sZ z?Ci}AGrJfJx74-ct6J0es literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de63f498e3348a44805c347917b7e681251135f8 GIT binary patch literal 4639 zcmai2OLH5?5#HH10q`kGG*vDas#1vqmkf%MoG6asa%5RfQZ|#=ik-lAt(TYqx!__K zoLNXDYH(tEHPwRt5r!iRVteSa{t_{{Z>w}HX2Ie7ONjC>uoh@d)XYm@ZKel*X zIQaUdsngkptie|yTNRb$G9+t!9g=l%87r<}zQH#!-^7X?NM7Pw{L*8qv&)^2tk(AU zQtfWsJvq!X(H!J_lnT|ZJ;C;@7Sr`#OD?3~_mXaLfVrJz=y!gZ6+#XKPvSznAD;{P zo+uz`+1jO*+I>G!1!j!{p$fqth)m^DuO})R#7PznW!@D^Jz>?4>>Q4glt;r{B}I~F zk;u9^PqL#fDKWpC*+AXGKv*3H>)V`loG+~&e88Q@VaGLNmwOm{+~)yy>3?atRwuwt zL%n%l=3|lFjf?K_K`sY3`1re86W0Yv^U*Oj+$@eo^N`jxRS_51P104(IOFC)-0LN< z)-aawKw#f^f>(OUnc(f3`QSVG@ci{*jLkQDG9NUHTy~EiCJNhY4-(aF=R?(eloZF! zm~+9K0@Fj$%kdd`s$_A^e1#U|qXH@=P$bUIn>qB7YG;P7Z5;!M})vxl*4 z1=>@ip^&l;opcyQNtP5*q}Qc5f>(r$4oAJ7klHgQ)-GVIv^N$qR|H2T(w=4n+Lb8k zrm<2{^jGWOdwa+EKvy zXho}9Ouj^#?NWD@Ix^w+&{PARWrr+ezFk80e9=}Gh$X9fXz!q@0v!-(^%)T^ z8k@Qk8xq?{+?m87)EbF5lemz$M&i#T9=9hx?C+NTj@9?3*0wbX;0><~wyl@QYm7!^ zxI;b(4kRgZ^v~mO|NOUq{o}1C|G_1{57Dw~`I5G)0|e%A3^1?K<8|uRNbX5>Di=Ot zXPQ0I>`>eCA=*GYAO}PNoHTpWm~&xM9e_3-#ge~{Zz)28APii)5j1KBtz8Hf7x|JdbsOEB;F0KD;R#>BUtd@BPE_CM1R~__*NlVaAG@6u zL-IB9hi=TX+1*%)maD^g80$3z46@H+-XPd^L82MhRBuaBjARCaq1+awl-5*tt`+G{ zV-v$A28A^R+K3y^?v#vJ@}Bki_0Q~yN9bu->P^l{?R?c#;Emb+=Inm!zGl}o z+t=)dW;e0>YT3Sg6DrG}Qg;U(=x}wm-D;n$pyCjsIlVz%M?PMpH@zms=_mnwqaz7+ zk(4RG-8ny(kYixY^qE{Cbi+$7!Q(QzTvcZBI$Xh;|;f9A85I4X}$XydVu_n&h7p2K;@Yw_S z%$*>w^&NN)_B~lqw+i=Dudo84GX%v5B^k`6#}~A$ckT`oWn)enDdUOlb#YguIWL z{2__hKpxO!5wI0IQ%ut<6^qynn174;V32FLkGG$4!JI(q@;ZHKpSl~=QG{s!15-)L z-_gW@?iF*8ZiYD|RdLIopX0&s#T$4;a8-n|(7xJ`BzZAcT*H$&S5Pr1h%NBP?HOAj z(!dtWBGA*}d>i8*d??g`C{X9k%3o8EA$6igT$Y6GmRniQ0NY?Xdq2=cyPbv z=$EA!rg2vwpUZ@ch_pmjuQbp#q;bP*g6mp_qPAi#9adG~{_Xkb=B>m67!xW*TDJTG zO3D}=C^I-h@-GUBU|QCx+ONtWiS<=+v7`vl1=SPJ*hPLuGD8A}6`srf#4aDBRYjCb zdB1@SHQt=NWEt4uv#`2}_p}2O7)(`g`X3l*7E!_I24=?K=ZJ8nkEmiS`n>SY`k9Su(TVK6u=82BwSJ$y@RHHkFLZi zbr)1fM0f-uM03yH1Q{ZofehQT1coB1^eIJ^D30(BNRQ?~G?G^z$;+g#SBU0Hw`ap1 z&VgY!Qw_B|`2l`Q`|)rnGA_vx@_mwcyf`0{%$`gy$B74vn#`avgv5isi!#7%m_() zln#^uZ53B0!bqVi@A}$}<&ly{Bw6s2S&Db*H291a^;&h-p8vurUYr>;MAG%a^Awdb zgS>TBnxLz9^E5@}557~9a93rUONJ}jS=R22GV?S}EAp>h5d4PLchPlrFK|n{qLy|v zDw0%H<%ADp0P~WpE6L1yg`S$@a^*TAqR!?rOlJQkFm{uFpx3C>G}*$KOxHd_*&?zc yfo8^Nw5*Fm77WFbZmo-p%Gz>+7r0fm5;cUOUb#-HPYvaEP-CK$P literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d72cf5d1aa2f98e62ec7462b03e9f0bf9f53edc3 GIT binary patch literal 4609 zcmai2OK%*<5uTpc4tJMNQKl&v!2$uItRb&NNlp~UF&x>J^RQ_kRvs9~7!S95$yv^0 z)jcbbi{T-m0tW&74}2hx`2qP8`3Ze>E;{BGLz~gqNuAI!cQ^BFi&>c;Hl|O4 zFu)TxbuusXjLb`yvS!#cV?S+W?XaCKhs$Q(NLRAea5Y;C*Dw$GQo5dPgd5EI(Be(r zdTQ~OaE9LS%Bd4>Le}QXkS&W^aut#lz6!~zxQZ38V7|uJF<-}uEl6JF8~n;sE8OPJ zr&f1!a;14c?jPH0lD^w}?Ly@aO>eWO=S)An2NEUse)H7E9$kyICNqICX zR8l5I9*MkP@Fd^wlM=JbnGV!l41^Uj_`v2YbiT9(@C0|B2BB-lF846@xX&BdrT?Yn zT44h_4fOg0SxiKJKQ8+RyM@ed^U1f(4z3H5&XWUdxKkd8&K|Ans4_0Go20Kgan8+y zcrZv{tx+uFOkm%5f>#E~k>I_i`QUrS==hCMjLmljvdB7RA^Qi96NT;dvPAWJ#YlCY zB;`RT=3MZOz;sUx3VcRBQnI{ZzCsI%aS4?YC=%z#odSAEa+63EREhJ;UpDfd6e<_x zXFK<9+_`(>&09BiZtZ+-z6ZPWabI-uIERmE<9&N-R=WFWWM&njE3J}M)~S7Fp99R5 zbp~*k?wMDyb9N5M8;K2xe@WuJAaO6g#Vt6(+tvOPk?bD;D0@k)bX$(|91f2%5$C!! zn>~(Yx1l{X9tkO1&`AeTl;lYnMS4|=eRxI4Xm30i2&p||V(kLPN_!I_3q^2LA{}T( zpk0chei|zkMSr#aef{P^k%^l}Nn9MkC;K@eStmgiRP4}~uT-oGesaOmnA??8xs_M>RpXrK zgY0GR>MQv;iwquxib@Fu1@Ag?4Iiy7b2Unfm`4P*h!86xjzH`#$ycyMHy0x9kE8%p z^-5aw;}j+@V;+~W_EjkI)+sYiNWK}}yduXaeS6M^rj1e89v2*vt zhQz)kaUgLnNnA+WOA?PeCqC@&R{oZC=AB!c)=2~2@T$h9^(uLd(WnZx$S3ZuBt@S6 z2iN{T55D>H-~RQFce=i${pkiG@E`_imoby8)UA-*lj@P2JB1x-_C&KiZOg}KnRdVe zfIGZ0ebd-)ZbaPxKRk&ge*@oAcX=Xle6HfbJ%N(?+H>D`YG6$1T5{pvGU#dUTigc&33EBw;rF9B;5eXi> zRWYK&_pQ%+zqe03f{VPzS>+uzm!QR)Ujew02gq5GZvwvsqBQE=p5E_F?{^<)c1yDz&2DRU2YauV?Z~&F zu>3J~Z=(b2El)RF@3PydDT7!|uaVD@cIR14uSoG|oB+Jhz66;_s(C=s1wWSLt8QS} zL=u(BA5uqB`6G1NrH|>Q`N!maNN4DnL)gHIG*cVsEY<{>z6MeSe8CATY;yIYPv?w^ zeY8qbLqsV^e@+GDt2q@s#so>ps@M=I>cmd06X)oQ$|Nv&>>)hno*;bKt!zFdM zbpOaoZ)oP*)>-4+s=Oi9YSy3ZZHwDq*=I{-bJ#jZeYXudDBDB_r-r28wjN!BU#W(2 zcb|O=9)OyzN|1U?U9&7dfJVAeD~`ZUN|7n;X0bZdD?e3ANUC-|AUf6Tfo>T*j*1lI zt-S#_uml+pjc6OWjRK2&QKYhg;mj95Be8v?w1YxHKEOq>&?SE`)Mfs2>4s@?6gLE^LA*oJV{QL|G zhA7^|BSNYofCcuowj{~RN#X{c%t(UDJwY^qKW_bCKf8W#6o0%AshQh?qw+vJ&XEN`*oQkv8FB_ z7MuWnpnBmsYsk+?W}@EEzzf-(=;YUEb&=#!UT-5IP1a|QSVVI0Jecm_J?&uQ21zxD z{s(kIEdO-&Qvu*>n?FhNO8_E}(xqNUHv^D4AD~dc`2axT4)O|}7TnM?dBsO8``Z6S zf$CoNXJ4Q@0hR8Cx9W^Q#TpgDhN;}`5n3nXV7hM-zk`HAIyj5B462O)}?%fBB=5yCsinZ@D5n6 z@=uY~Du0L!d=BPBzWNUFTICLG*u&X1bY`lZt|vdjZ)rasjYQ5RIYR!DBpxr1M`8va zQdJTl*+A>`r3`mnPQ&jd~IVA~qb&k1WxT2Tmy}>v) zPvf+v{N@G0C$xSKUATRLTY5FM^rCT@q^hnTd>{jumtx(d% z9{-KqB!S!1kS0eMlj-KCs8d8;_ow4q(j literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce70cc4869f118856944389e0a555174d5bf678e GIT binary patch literal 15537 zcmc&*+m9Ssd9SLjzW0pB9uh?C`EG$(1?GxACfB%qS{HKxm**Jd|NBCnDp3yKo)3dr} z!<4^PC)cqXd7S6EW-r&snY?dz3%z2aDChZZsaI~4dzD6|S8Y^#wMMPC&{*g#HWqQe z;1#>adP|KZIWKjO_m&&Wa$fG9=$&kw?44?y>YZ+!?wx6z>8&(Y^&NMz^ zcvY`<$M9-?uJgb+*LV;$3*I7X7L%HXP;<;%Ld{ZA^Dt_Td&{U zZW#4b2ftK#?!z}f>GTGk-*w!+=d@H1`6?LnomSTkLqA-r+{1Dh>vc1>`hAqejlo`X zuoKn`ac$iXBj3B}_rrmTAMVRlGm^{ZrW*7n=iMNT;*#5H4VBy4j~Cm1)Z87YuGb8M z13#_?eY|vM&~+m}ekcf=9@;nE+iuWpVo=SXA4GxMMN{vdiNRnLiyOlLk8ilG?Tta- zN6mt+>A6vB%UAV$T;kM1zuk*ZTpxtdbv0=DVR+r`4|bxU6(7q~-e9Y;7qt9lcMyj0 zau~FGgTQNU4q9$k>hOl+`VEnC`{Cw5^?cRh(oBA&AO6B;QTPVv#%Nfc*~od7-!e8q zH=cc`+_3$;mw(+PJ>0Py1;5xSb;@4JE8j83W&+t@JB3kUl=I9p#+dcRsD$f+XL-3XTbI$Ig69fyw~DLU$m}ej1=!(k z4lNc(Cdy-@Jjx|6kI59=XD!2Uo=^8t&ql~@t>PrV>#lNp7{3ZHr**tF+2qSkG*GQA zQY+^_*pwxi+{I02!*_a+mfKJIb%MU*I$_XnV`0%(8u(M9ni1uVngLF)~f67gLARw zAnxgArjt+=(C$65**%gnII_C!tG?eADY9Na9_Pa$7$Pn;MPxLaaW%8RaW%7banWlE z`6&>l5f@$v`o62;s%%oz$JTjq?&B|9k1HFLcet&Rw}5ixDvt2GC?aEIMrH(f z+_c7)T8ZqDIW`X=0Xlh}y<#0&hmb8^;jGaqj164dhxt+7v$o6Ohoa&a*s_G9JhI0o zWk#pcsY3n~&l+Rf%bhh23#h9>-k66)oI%oz&CcT3I4q5dUg@kcg77?Rc=qeoVHx+2 zsqdg%8Wl$+NT{-xzhaEakU^E7~xhGsDnSuFYz8e?phw?x zE(HBncPMP%A3UL@V4Z94xZUo28Iml;%%2-v#ix%fV}`G>m=}ccw8G*%?p&(RSzJAA zFtg85lM4C(POViqQjJ9%C=3@`NO*LZt5`G4JH(n`S>3mgta|JhT5PGBd)OL%w!i9Z z`9XUt0(k{qv{k1JM~BXBw>!iODfKcVf8;gT5Es4!j}ILBzUq0lJm)+n|1K^A5ytumPD*dO<~fR;15)8<91mImtQ&ed-{3 z??HQ8gI&sR5E*cd4i@?=P4ow!wAxoTlFH;wL#zQf~ET1 z{e*V03Ab5E`;dfYG$_g{s%vG31W3x0M;r2kU5Op ze#?(-(WUdE3B?7idMj#of+J933DnqDKVJMGpjL71KCIM03B;&(vgJEa#D!garVBwMFpDsxHh zigUQD9%olltjdcMdCZHDC}kX5GE3&7wPe=J^Hv4$v0ZN2c~h`VR7>EP**r&S6;=L&1TqlU)!mv-hTV5P5 zBnnE)?{@J}I<~s0-p$9j%jyJ*Q#isBijrxYi>DSJmf997(jPa5zp#TMYg36u1(W1X zsbPDkyu4SqV>a@Bp;H_i9@GM{rl1&j6x^(1Ub>h@e~$U6!yWi!3tcCaR(M(cp9dUpp{~sbW)t zz+&FX!0M$s)$61`XKUDpXbzy4!}t!`ePS@InrPRDtZOUEO(#-Az_ms!n(Q1x!g3V* zycESR6?RX_D?~NE9RP4dXzjRx7$@3Z_L9z@(4Akz;C-qF5FXBj>63c$d?{XEmlw=f zncOZO;U`gKtW4U26DyMzqAhkOv_PB}BACF?*kCHd;xt9u%fp(qp|?RR^ekGMaE9ct zEqV7C?Z~SN@<-!J>aUo}-TRr5869Dctp(M^Zx1N;fit}ziil5K%MdCeoDaa}M1^sZ zrn6RM;&-?_u})p0Do}8;vo6#)D2Rf3$$mzzf+Ga*g!KfmK$L?0(_JI>Y7q^^zZG59 zs#-?Rb3@{gjFF5=HB~MqNQhq1B+)J~6YO`YD(H@n1BZ5Bok=kf%CMfc-l$uPB{mvpHuYb&;9%ilVur#nuI!#l>Wo;%a7PRrxe4>M zsD&^4QzUam&`>v7e2T>8}6=FRO+NyBeVLrLD z#su*}7xI3=vbwFF!&9*t#bz@$KQ;w8jcgym%?O8vdgEYqW@kxYP2Jl(z+b;d1)*-m zc`w@E@#BK$--cIGlXy`cZu$2-v=O~B^;b~As~=PE$BlTUNd|&W)`TylH;hCtZwB68 z0@x5=>Z5FEZw|X%_)KVDDdCSi9I@|vulg+Sn%Clmmf~=ML6A=SaasH))GdX`73r~F z6LyG;2?E8%WIN*`O%DKIT&CdFx^51ZVWYyKX%jKdVOWi&WZOjA_+fLc&UN7F6_$fmGAeaSV`BuU8$$y< z1gnk8V4}*X0yZfh{EKr{YZOBAEwRpEnnwGM&#e^?Acdy85e~XgJao0F6qLZJt1*FX z2uWnsp*=X!THge&`pMB4bjGcUm46X6cZ&b3lq>&uaQz;w}ZhD_(V`t8!X7Z60;Nh6INBXSvN!x z+g{+d70s9UMBDHCO8W*0T}|{9FY~0e*V^CRYDRY%pQ&LIQe4_|VDV#Vq7OgUmtKGds5U^2=b&GBirCD%iR_f|viW96=)l`@`|# z*_(8}lsiSQrX4X916~PwCVEfWl-Z_J@+#C^(N>?KH^c;rV0-i&RWKGt2y#(%a^_Xx z@vn-Wxp46Px0eERP$9}j4wta*EYnk0k(w!s{ES>L{r;R>zXd1ol<^zdA1EEAEJ?Ve zDk6I7Nt_5=)(}|{u$(|;oR`Nm8)~T3Y&c6&ifZYf<9G}9laA)G-@1AQPa=qgXdc64 z*WmH8C30`m(-1F!Q`&tPj|j%8P?%Nt5PGcUME0DVa-Hw}6mHHeXC^u;0r05KN{vCi zkFU4j^DOa7z_z#vuQxQ6*uEY3yXrV^ETKpgQZ)x_;ot-}LgR#ziZ@a96#mo;cn1%s zOe$4z9>#Ov&4pdfq0;O-crZMV!mw-LO7f-lUE(bFAg)iNmUot{dAurpr13TGI1f=k zDX$_xZsN{_Cvt4kc?piNq#icj9Vkyp9inU_MB$)N+EdpiJ4>}L2Ny@p9d?8VfwY>W zCX}RA9hwNuZm{hqW{)~R7wTyil=DXfmr&W?qJBbUzcM46?q`_Hqq&q1D3CdcBYZIN z&1MFzN1hXP@vVpWche!lnVP~vT=t<#L-0qOuK2sDsKsR+!J`TFRa_sPG_mEc*(3EP z1Lnp_^Wa@S)5QP3w;i_bylq#%i+Rj!xL|q?Tyu2p6e~Z(@fI6ChpT^q4bM{P%!Ug& zQo)!v#DAC$F}vaF_fVhRYW4f<;A?Yw`7-MM9>?2k@|?-0J{$@!vf{6?o99r#qr!s` zaAexRgfw3#E}EkvJUeTV4d18vO?W=P1)mPQJh%fa1&_mfn%_hQ&Q}=5u-7pG+DmW- zOsxR5j~aCc#lgegz}X)RC)VDs3zu@nF?moB*!$IbUNJXF?XeiM_&N(}PxX>E@@=Xm zb0uqG8}amZ*eHqO(sOr?Y(iX`IQ9M!{e{#>3}`X@R{_HuaeqzPrU9NRnsfCi3&Lli z^J|HSydz{XQToAaBc{m+IS7Y#1QQ*}R#>Yj zCHbf_Hk60^Rpc7LY&Q*@FO2dLG7TP(^u4=Q=NPgM@-ScudynFMOV1gvK8XD)bdKX{ znfrs?!F$H?{H@nsGTX?`3^)zwiR0VwoKQl3RQ?KF2@(=n03yIYm##}iMhDHxG;aOq zCF9l)UNS}nP|c!y1IwsG8h~7y6`5d$c3O+$XsQm~&#yiSJ|rG$dZ8r)5QbKWU@`SA z?Htms^fXH#8>_m)rwxz|0y7{OOJoevK-DcYOq??FG|e~4S7<%$?Wkvd zJQx_{ucdEYt5h#t->nyn7nB)P^MTO>)t8bzBZyd>5*H6ZIWCiv@9RZ;E zsI>^Lq}oLRE7jkNVw>5taX|-R^qk7PTV;l;3O3-%^WqA;a7{60{WuRNXm3Xqr3=17 zv`++BcRXk%>YMCeKZbXN^KvLQ9(Fk~zzC zuj1~%;t1)TGK$N&T5j1q4{uVp-XtabPT;4jDtO4mj;tlfh97sQ_H8g@j$W&kIZr1!&{gw7GBbayA%Gl_l9 z2EHlp6R!)cRT}?-Oi9QH=k)!}=I1^2YBtD3NSe*3Vs302Y8AU69dHI-lGAH&q5EWh zfl@jik&Pq~E@UbTFscCl^)xw4UUgx^2W|LpM?|$dGa#sGPa(Yvi5EW{Y)0_z1ApM+r+iRfK*4Mm zWM+CI@ew|Y@C|}E6zdC9H-9{F_X*)ODip{`%&Ww6`YK}^!IKU3XY7~?yy)xzmR`HL zrT^^0L4lkgzq#3p;v!aP*p0%Yi$Gaua}iGC#3(OYCn1Jw=DA$OT7*bGXPveVvR^<< z7G@ox?0FJd%;E2GJnT91HtvaD_dmrMax!#+5wmv`Yg#zcAD4^2@B)ggIGxGdL5u+T zIeE#up;}r{q?m(Wc0P^lYH%KtkHIkAD0o7W!}+~K%ilpf5_u`emOy?5z!R?VYuDE6 zsl$`x$Ae=Ci8q;4l;nelwmGsrL>3^b#5mC6JwjXqck*Wt*%;;8@(F}CA(*J*fu><@ z3Gt`o&UcXHiToi4KUMO?vjTD-TZtaU=goC!I8?mUgD4W!BR2b5`;aA`uJa$G$%P4h zLy39>8R=LmBd$0fh(1*#q&Mb8XtBjFxj@@Vs}tP|GZ zV^?+D7NTVaJ1hQ@?ai$T$WDhy{C+2gNc+ryx2u4d>p?9$(D`1DXVHA`GC&I^keEcP z02)aZM-6mNL{OotXwahLYi#!MSqje%&095?F`|{~-GJ>%rAYl13l3E?u#Gs-Ffs3A zb5E4II4_gVkaeo@yqnV?-5BuTTrdx+oJ1l;?!zmhy9}afaiOM>`9>zym06Y(X=2Hb2sz|c;%C9q;%_Mu6oRWZ5Qh$Q$MO-tf z1L^fRtI1hW{RGBoY453M3H8Ul(h7QlY3qCm$snEO#2j6OIa=(T&}O=rq2I^e6=A?4 zh=8;%e6i*oZ|B6QEx}MmT-;l3o9|~B;iPwBRA%y5bPDsKB4Kn+pw7xc{ zgTMDfT({pNzyJDGMgpTPVB^-H>&2zX_S7GUYt)`XU0pT(1^%3_%XS9`j5 zda7i`coAPO9^jK3ohBBl8Y&x$@+pvbxR@u`SY&`v;{1{~*I1-0KYNWe)gKhf{Wy;g z2ircfP<`aTh~!n@s zAb!L--J-@Z=5A;5+_WWGPBSLsMWl2@NtWtBg^e1%yo83@Z#O3nyon)95gF%&WS-kQ9T-y++4$c@GVU514uWD$$WT}5fkM(zG zn~)kv=g2>pAofzZDBvtmWI;iAax1WwFP_ondQ$LPcA>QeC&S#@|Kjb literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4739f63cb5dd95eccfaf35b4aadc85493664cc9f GIT binary patch literal 15827 zcmc&*+jHF3c?Yn--npcx8)exJsyH^cmTAjLnp9OBC$g0|wG|~+;&yF!3ogJVsJ+k! zSn<+jGh-@=(;21dr1v;2nRYt8Bz@>}X4<6FX*+%COI|Q}YM+$O^bhbu+TZUxSS(0U zax!hFE8!40I5;?$@B4n=4IiJGsT%mTp85^%yU!WMzw=`9vvKk)j_5y8_(sd{P2cL7 zEmQtl{eoqfM&E8ZChrt_X1~xX$hqAs_DijjoIAa8ztXDotF3Ck)~faEt$Kf^HPdgj z8vWVUEZP+PQg5z*taVJz%f0#j@z!xUuk=pzPqt3>Pqj|mk(4_zl!F(wc`+GwaWxW-hIH4{DD2^Qf6mYh3@h zf8v(W`WfGP*=U~JesA@;=dXX(?GOE+=XwL*?WiygR5%>Cot_s(L9|r8gM~4!HqFEu z3{aMqhnwx;dekf>^{YV?2mbY75DiuG@IbEGv0S!S)UZD}?}bsEJklNx2Al17X;t<* zossf7TS=oE#O;lt>iO*`+zygjIKa@>hdnP2l83^m?W5PWchd`dZ4A8~4#GI}dT8q3 zF)?I}wXr-3@%XaWxv@MP1gM$OHGMDctOly-BxNo@6m*cYxS|YHiH(XZ{IGUMPqEDoET7 z{OR-VM&PQTGwk;R$dL{kYn{FNZ^hTS)DHq-AYJ^@^SQGX_Pi^5XouQB3U9wjY zxr_Ykf_?)J8!C6PRy}VEoQpLFaZfihn}n)?cJGkQ?vRwhk+mCv8U#I&B3GNo5+@pg zA(C=iL`J)v)N%`))N)&wl>D}kp8{bTN%6&S5O^x7$tJY}Y@MGJUV8CbQeCE0)D@NI zSzcT3@_89Nc``PwGMdtKI-(ijrzLSRcjhbv0Z|$`dM0%9CY+l(nOvx1t`! z0MuEILPndCiF$z5kFmPC8v1@PfK=>b&Xu;Nv3C0}jCbDqTq!%Z-K_et=- zdQv79$vH?=ok1t3aYQFj7}cs-lz;Q)oY^o>;Cy?A0*J_kIXN-N@FPt%Uinv>coH2JwID>@QH}@L*#!h)$ zVykhPrEkA&?Nspetol>5nj4qKWk{xq?_4s*70RB`4Uo+D_pU>HLR4;e%6Eg!b+9xD z(9Po<2z@mKId+DF2ml0`x^hqufyke5)0J7$V0aN?hJfR1u8Gc*;iFY=J#b%@J3^c8 zru%*zpTzOX>lr-@Bj&s8d(n-H7}Rn&Ak*;jUft=g&U37>2Ydu;U2q}w*FBJm@2-cN zAi?T^SG^E}iy_AboHwv8b~nO!Rb=SOu-6-Iu#=z{fXm}U5?CgFZe0aFIJ_Ii(K$#? ztf$awxDs|efgF9$T?_}E-bmPeFnmHwzb4n-^?JRdWko5>=0V zvBQ?Cy@{>SXItmp)gbJy#vrfIk5`+N-RRK0>Gei;U6b`6+=)DK+tqwORw?SmUGmZ;`8Vz!DTON)uEW94ee%A}1aL-XR4o5Mh zCb#O-P}7D3uLqVI^tMvk#U^a51^_EU3z;MZ6t1EN#pfnA5%>>UhsNZ&R8O%c@3xe} z4tN>Z4oIA5D$XI^7`oo_5Hq?FY-tL)8@8tNx$|H6T~(r}T1hR(HAz)2gy@o5Znl7~ zY6$d=VsFq15?i!tN3@-!sFiR<4NkBEiYmbvTNosbj{!~<*XenshDu;Wy^k%=p-74w zLV4<8*0`)GrJAiqmYH@0O*})Ss#;{jB{nRt1Rj;1C@J7gNg-05#HK__tko!)Nw-Nu zQSOqQloW7Roo824JRvVmWiM-@1r)~YoLL3{&6#!coLR-$JfLXK+^!24rOi79HpP#B zsnA}=5j}!}R@^OtA>Z%|U$g)S*5qv0dJft&gui{Y!+qe79dHd_#t{)*yT;gfeIB@E z?pV-R@$lQ$Z&`2Hc+6~C>M1l)7g%t}%|c==J(ZYVVlE$=+B?-t+%Im)GcTcOM9c)J zKB_@AzrstGdNe4aXK*rkL_Nvw&7`m$t~U#M2R?=>^>G#lS5f@}>OP4h5@W-`TjuY6 zOVhm4tWyaZK%IsXRBcn$fy#{YdK>zt*Gun|+HHT>X}3j1Q0!AZ$Kr!1lEPXTC&kf# z8nwJQnMqZTPSES&p=@kTQ$5bdxY3GA#z`De8AaK&&Bn>bgHqc;MfT%%@E5J2$XiQd zE5X>fRc_h-N#F5{x6GCk6!%K|hF|i_Vkf~~@~g0RE82E$Rkg+3s%bm9RoB*WYbLdd z8)8?@Vrx$&^_PSA^28YSPJuqOGA&>R-tR;HKnRfYMN1`c$s&ssxsGb6bTlPh0Mt56 zVk*y+5wMuobHI6_NyRzs&s`l2Abdk;+%Uew?tmx@dnMioAkDgplF^OT2!O0Hi6%RT zkTMztJ1<4G%f#8!@(NLUZ-#&r5lib{C}xMYg#EPhCv@jeV(z zH&s&HwCO>6G4&{1oYb^x3WvUA16_ zO5ceuYGo{^$$1fxM#e}+r6MX96Rbn8Xp-s=m3GEf??F|41<-6(`JV> zH~o|gdhUgQSj>(K^ezjdhjVcPvZCGbv2>W)giE{Bwln=c8Be9y?24a97F9nu9lIEO zA|I1_&|i!(PFEKv!s;nQc}yUY*L=gJSpnFwPds5F^ZB|s6!CKP}Y`5 zD=RQOOIrA{KSh*K*I9gy#phX&^e3WDWS_`8T34^&h~`lkm5Fsl8*2`*wtX@$ZZaF| zdd6LKJe2|4U*f_y$L5VhUQLf z?5IEUtMM$nKP6gQyOv)&V=NeBxGTZIbW+W1?_Q?%oi_m>7ZF(NZ-GW+6iK18J^~KP3 z{@e%DGVhvKl9`U;z=2PYTw6&+ye8B(g#i@30L{9vN>WPEDJi9!oRnyI034GFg|gOk z3$P1YRgO;^h)DrsYt5ybDUv9N+QU}kV2yZJHRolThAewh%dITc2dz)qb+c?$O&2bW z2F@06*wzE)_QJh3r9(2|kA5YdU&aw}ty8uGjIF^95u1Q8VY#vBm4Gn&wgsnz16FkC zyjZh+2d;?Xf^nk&Mx@K497AInm-ov1#uzZT4`=F5#V?I3V6E!73g#(q|G>ShwGCl} zj@V`}Oyey#;Qor=j{?>J@L7L%)t=w`rVB| zKA7OPm6Crc1_lv1sIuQK?36=;$Rq4vpM_`6uUs;KF4kTpuI|-vh^J5fmiEnkYu{Wj z$SAodYN$UKkX#a3$XeTA61M>o02?fVJ(z~<)2UGKW-iNfT zbw};^p<5`YQqiLQU%*O7CaqfN4eUf&jSDq)nG zt-vTsEpdFC-ic%eoYEb@;6!LnMZ&hij8JklJF@5Wl$>(*be<8@mO=GH2W4FjLxAj>^eA-JP8k{MKiEmco5|&)bh@pbrhdUA6dk0V#f$B ze*k^Pz~a>D34i6TNf#%0#FBd8@hvD+X&vHjW5ng4Wab>8?2v}0kdTWLA)$q-VZ@H` zAi!6X(}aq&4n(t|-3xC76G8v$=t4cig3|tw&=Ly!9_l9)_VWki(!DH}qv$Q;0Sei@h+Qu#AGv{PZ^I>p8X=b zc@719FFfE)T|>?dv}^NiVxl=N!4tF;+dk;~8whiJv#@KzD}*}?cZ>`0wmK`w#rdiZ zc3=Xuq2MK$ng?nJHR>%C+YkFgcWXGBSb!TIT+lg3=5|pa@1spe#e8PW;>#?4mj#7% zv#bq&n<~kXlC^jP5%zA>DvQ$6_tp<>LQjShKE)zfKGfHJ*fV z&U;A1a$u=Aadp>FKJL>$2J_xDa6U73;>NCVRx}n^IkU(^aA4CEHy_1*%{^zl{vh_N zxOWV1pXdHyckrHldH#*JUNQVKy?S3W)uqk1;^RQ=GEHARxA~g&73MT>)MY*D&E>Su zUlP5ja>G(TdgF&;L6!U}u71q02OJ6X+3{_7PAI_{SH24Of`m|JfCy#SPm^`2%IKg~ znZ_GGe8qUPLyreO2Jmw;rmW}vm z++rv;1h2n-YJyunoJNYiT=VN`^ie&HMtPBzm^akTNu6dW`koPavAwAn-15GU#-%@hfaGBld^(R|(j*&vVug0V!#AZ=8Q(E2`F<~x;3w465A z)ieJ#8J%rxKlU!8n>>S_Jv^rUFywo>hWg<^uMP#83`EfZdnKC*1q0FdIF86gA&V_q z4wM8evTwX)e8c#rb=t_5JJVl)2=qFsKW4!N$0S9~5u6uy{~l`J!I7?Y^F-nxKd>L^ zhyhJUtwn6`?AK7hObs^U#AfzwQq(~kJtxYsq{duV6)wZQ=OZrXRTsSLQ$IL z(mBg?2~I!25iu#iD9sn@g?WVQi!ePG3O33I3s7tEm(EuULcDC1aed01x3*6lwb4!U zI{en3LT8HTr(R_75{ok|1b9T@EDeTjIu{Sa5Opj?4Ax7)0jg^(GF=Rt_BbnK0-wPd z4@rJ2I~elu>dCoA`DFQ_auNTkig!CWvL8w4SsX$>`OtJ=?>PuUiz^t(x^M%79h{_g zZJJ)Z{TVGub-P;M`LOW7Tni}cK9}GKFpf9uxBIF&H+L) zBab$?^F>SSZX_p5?p_g*COWVsrZl!-g~JyBi3JNkuB^c?2m2dG7nyqS@4@npD|@iM zVUN^t-x&itu~Kr+ufPj1gW8$#48rzkEm?ds~{b zYYmL81kVBei1@pLdEl9{yseD49UC7T&!J`>ZRg=Jgx|ogAvPvaD|jXfq7xh+AIDDB zy}u#SUDF^Z;yed$ncTh?x}{u&AgZ@9LAps~_F!0^=#u((XcyDXAUK$YvT#o43Y_=>MU4PAJ)g*ilE~X~3d1Udhd)aWlUFS+ z2cX0N{!rxWda;Kbt1hz?q>pJh@94K7{Qzkt;dcIo5rj_<2?hFyy7(ug7&GsQ`2`6OnPm_79g)J&z!Gy?cuP}Kx_AIr&tCCug!@@82- zkby)`?fGCZA&nzM!<88E1ppm_3=~A_ODLFzgQQSj2lO1IiFM#XVH8^|^2A>CdMim~ zG>DLv9Bc>L=R@CS^Tad~kxZP9LX@p4p&^aPm3dCLX<{R^v#kD_Z>A|B)&n4}-)*nz zKO1nBAcrYvuXN(XiaO#O83nD`thOy6*D6Py^vuCo02hlmJMpbZpieg$R7px$_E9g6 z4zDZ)JF-KJXb~qyW!^dgOsSg>6slGO_(If~hkK=tVBZ7U)nc8pw)3BQOxFJ@BKt=H z4q--pn}vW#h97z)f*#-DyEzEb6qA`0Ywkh|wvut?(c(ZN6JjGsS8^l`iCTA2y$8>1 zf6)Cr(#XN;%(e!H_Tumf$%5yH9i3nuAy}m8AbAAoCeRHK47hUTYBTfYlFCasNGK7f zn9IybzwGFmW7|h81u_!0&mJsE$sIsE2X~y)1~PRDUF7FX@(7r>vOk1q=%tU8G)sp!mn1`>W2*TM-* z)n~=`*uY$!=nyha86wq_$2ml5WCkd%2+G^8=LdSU6X#hp-@FJ+hJ7y<-+ADuljsXYNqPbn;BvN5@4_*=FOxWZ;x)w=#and)q#j0yE&uC}{4KjLt6D_d< zu;5|;G<<6S%H;P0sDY*ed~jfZn217PmBh~p%bxZ<@?QAYjzPJwdiex1%SwLnwgY_z z`7UZLuIMicU|S*R4*kSIq<~KqVTT)Qb-(g91Na7X5?IvCN5fO!LfODI3Y=!aqK)>_XbbCovzyOYojkxPJmOVhr>&c#D0Uj2Ctd z$<4ud(Z^aH)x1OX@{Xj4%<#Iso_;BJhvp9Ww~0T~uTq+3tG1#DHf?@$l6v@c&c>T;xy$0K zEPfM3MgkNgB)$Jj`|MXpgA_eQWBZJJw3ibhnejWtH}ws)6&IPgQq@^-p(Xuxx_5f24ctf?_?UAW z-xletxk!;iTMhZtv&bor-BUYErt&?!hcX)q}#Vm^EOk&fEsA*2-@+j|}MS&0(ziFLrQEQe7 z?76f*Z75f=1k$8|ES5M;+8wH>RmUf$P>BcK_QXFqvGXbE6GuqqL)?d}xQ6awR%z>6 zux(-E>VI@s7E=v`Sb%U3u5AcZ7iSFpu!j1H4g@h+i!ZJ$MMq8ZoHlu>fwYcGp8wKd zu(UE7NEqAer8t(WxPhjLJ&MDg4oX*ey3`M&j?UXAG%GTeBxs4-K*z|ctdrFfauCYq zE+=(T@9InRP?E<}_RHLPl4qSY+QkSH=Ts~ps1B%PEmVY`rf{tVe9 hqTT=>0al_8UG5R2Y{7$I!KZL({tL!a^NaJd{|zv%2V(#L literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0adfb0bcbde625e011cf77144d73aa22397f8604 GIT binary patch literal 4383 zcmb7IO^@5g86J}QUTM8?HjXza+JSTE3fR@!Xj-7b#*OQwP0_?~Z36|=1_Z?!NmM9O znHhOMAYCdOIr-3g{y_f{uRZxMDUjnjt0qCCI%1OBx$?Z z&0c!yho|A$$`53Izxb!c4AcD|Y>gFr<=@(rm7zS?2z*)Y;P zj3Mw61r4S7N&()J=0wWGf@1QXqIuf4TM3S*>bpjRAhDS*lPel%MlKN?HBtz5Tf0SV`)iNg@;ms)F1!xDB15BnQ z4{(oD4C?}!ArvVj?(n~m2Bd4C)j1+u7hh++%f~E#Lf#yk?9Ol~ScE^hR8nCmrmo>=4e2aZnxAPu+pqLKG z6(|hoqF)!chg*M+!LS7zISb~nnKNU)J99C5GY?~Fh9LRQ)5kM7iv9KSUsD$D!ub~h z2<}h{P9ViYD<;HU`n^=S-N<&20~L(*r&eTI)3uWLQ*J#6fYQrK#({|tB_}c(49(IP ztZos^kB4c(rBd(*Snz`1GS03tAt>I7$Jh4?x=Yig(>y#u0P>~^u?k!0cjQaBzY||N z$m0}o^Cre`RZ@`yoWCE$u?3WTGBJoVe)mzkl9lyzB2`xN{V0iy@2eV4t2zx04B7Sr zt)(JBza%`{rL3f7+}}8cyi@mc=kopku%9|d!*ncQQ6^S#Y1(pkzGl6TT|12-kEhqyId6h11PJvL&( z72ZpCkNwk~dvoij*tMx^;f`Ffy>RAboUN>O=N03PN}?)iFWAief-M}JsoFgx^tHX( z4Bjo=m+q|E2Zt>-uj5pCR+-gASu|dG^M
  • a4PGFxEiF#hI-c8`WlwU54NKtiH=W zq*-IOMYDNx)*Njirb^<*!W%Ug*b!SRuJe{?(#o`OZhO`eI}5}Y$vEE@EqiX8R*1Iu zf|m$k{#dGhs>W;a2LFG1z@<}z%A{8w8E{Ith@im0c@h&uNWGZHNUgLjkt|VmR3tY- z*T(N1%4vR_4kDxPfT&C~A`%WPqA~wRkLrT2QH>6Gf-3g?R1%$hVPje9Z}~}NhBksX zqC2O0DU#QCA{yn-Vfo1#qaqO_pKiqT3OnLdr22KN;s~Hnd=_v+MR2Rvnk~PhgDWcSjq)b*Ba)D zG*gvq_~HY;Gz>Z0niUQ%k)KYeKZE)9qeMt_M^yf4GHU0>x@wEV;zYDW6fZPg*PPM4qEgNY%cA&63{e8hV#I!VBs-D*8Igwk;w*v;M~+6Sa!a&hTLxiw(5M z&g8cR2;{e`C7}3>kE3LtjCCns*Bx)AShls60v^4luELfhRmCi8*oW2~LMl>p(GhAu zp5oS25(Y?;-!|tCQZ?IS|8~DGA;mdaC3i1{UY?=fbvt)6=eb?w!+Y5Osc}q}r+)kX zeWFLiEA0H>N>xFkJPDNO9LH%G#QHua%NKvPWM!*RA(*-amg++qNYa&T>Nb{tz+DIh zm7K{XUR2a93UtzT4)h?>0 zGk3wxouQg8+2vX}ihyi;mRaURnG?QAJ1cw+`6X^-d;9F}lH2~0+X!Yh4Aq&v0&lEn zw5z_mR2aIJg*lr@G__@us&JUq1MM4IU9xhNnBRR;$fii2%o)8&x@UM)VDb8YxYWnk zutWO>x}sq)ky*8mMiYdys!sc69Gt3O5kU_fW>yXbTtpFciX1lYnQZrD8>iCqNS@&6 zWSZ3_!UbPIMuPApj!gT8+9pEPqTpuLB7`%yr?aiK;Of?J#Gk0Nm)RLLs@~vNwQql$lP%U@)JCH+vgn@l<%e&?5c)QMS<)*h&+Vy_v+;q2{ zkDT+{*ToB9CS$IKc7r;}Bp6HI&lZq_P#%238JTGNXNYCDSBK1D&UDj2AL)HJD8O787#yLaJplv*x2L`o^} zYVs|37H)j6yrORu_Yt(Y-2ApnmQZ(FOME$9d_Hws7i(*d)snh&xMvCVT(6|HVrH%0 zG>U~^G`)W(fhjhaqbN1Cx76?PV?hAFFq)io(us-2Frtk%MRA%nrzVO!Fu0#ibnbSf pSysvay0 literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f51dca35d0b794935a0b0a263c79cc040c87a4a GIT binary patch literal 4459 zcmbVQ%a7Z}8RwAHb62wBY!W*yf`JRPt73O;13?RH z%*gwIbZFN`fC4_`-ox&({}Qh~`CrJT{e44frPX@Vv|MmFGaP=;-}lgd(rPswTz~!f z-@|X;a-4tBVsZJHJix7=Vh~Qx5lpxV>oGjN#2tG*kI|l=u(99s?Yxpy#?@YRTth++;k;#*?X$+>9h2hb9`u z=@6UR@GuozXUX&8U6d&$^(0FL9r*0o!zX*q*O1V8+-3PYNu0`11$sITLapOrI+m%y z3pY)>ZhpO=9R|^1baXHZOcop-9UkRx4`nJ8JRu)iHyRjmId(`U&bjf zjkaSsz-3DE5cfF4uqu!lL6K78J^lyMfOHMD+~Eh3i;Rc-?ykvp?=GEQD$$ql{R*vu z*TE=G3JEh}3p0S@G~Kn3k%Dg*W|;z0(9~cj)I817-9j7P-LoRUhS{C2mp2Jvfwg_! zu=74JP)>&g1quVY=;`A2aO(#cjC10gvNMM{Gd6PqcjjXBW*$a=h9LRM)AwcoihX*; z(^Q0e!v2N;0v!DC08%`%VnW;n?xiaHM!tIzs&K47vLe%(uBF5uaf=)nls;B62~CVB zIgs&iWENPky2WrmUS=tmN&ye>;5olVoLy!@ruacTKD%4eUAQir7U2m3kT+9^Rro^x zK)z)6AHC|B^O|T<@-A0(Vh5OQ-H3s0i;mkLTw_lxAXH8KNo3Fh2rm30wta`#QHc!}z zi!-g6v)`C)ZaH{1XB%72do3?1r>JOOPbJYQ z7B;S>{*oWWW@MvzEz%3B7ovKFCn8h+FDyS?VN@n%96ro;}Zz=}LQ-riQ;RCdWd%-W z+~M%b(h`<(X{jq6^HiFtO4qRXkS`oVU|aFR&IR(*3H54l-*%h|iB5@H0h;W03uj&S z#cp{bURV?##8E+o41uhP9s@{PDfC31qh?6ey=0pOz5OfbUF;6esTWq!S5dYt65-6^ zk3%ME8=;-yqbw0?)*?R>Zc8H&ZkJ1#;$uFJ(`|Crg@9cJ-cqrAVIOYrbrTOo3a$yNeuTXTxJwChFSWX{$(y+oIcuZNoEfT$u+quG zLQMS_uVEPvj{I*-|FmR!&Wg(KxjTrKWwnK3%jPcFxib{u1v{$=G$hk@J*#2P;+#cq zQn7ahH-d+C{+$K4?FF}efWSEWu5ey~H&!&-)xa$w!JunJn4^hA4_q;sibi=O)Pb=D zD6hn+`NfB&Y>IpSj_I19o z5_P>gjsz2x^>aI;ZnoDhPX|Rr^jeFx#a^#wh9-!G?$v2RU#zusIeP>_kR)|pD^uB; zU7&su8Mh#&{s4pX&b!{btmC&?+wHJUb zE$(?D05q@BCzKBqHIqa@)1+SY-5YJYcMg5ZNaV?)pooHZk!Sjf5$@)j+=tgI*^(t&UZwg)n8RZ(k9hEi=MXE2NH^tf^A8gz9>>Q;(fS?fmi>D}TYN9fYu1Cb04 zFfu-K#mZM~kBdNrkK)XezKFi&)-W#%zG5EBLKwGdUi)-z)7{|%DafMF%UX{AlBBqH z4@#ZluSkcDlBsTsS`S?aU|8d>AtL`Dynh}7W5o9>M$kgVX-gumeqIm0enB( z_%37jcKSZXbkWnn?c&y-g0yVP27Jmm>v+d4Pq4#S-RfMy4r#Etj4Jpg(_J-^X}Oo`eo$YlrKI{p-1?uKAwUB*aX|S50iw|8i>-rjxUaB*JGEgLc!Ib7AP~-D zXX1Ar=I(MHbZii+uZ1@WJHPVG;H(O_i-dcLohO~iWp<8%|E$iLX13>H_2@F2#7o=eZ)O%Q z-_t55KLh2_Is%|M*{j`@_Hg{@2?* zM_s_&1VTP6`%;nU8G#+-T~Q9tsIJex!KL0L?l}|AQcF{9=tHx92Q{Y%S!NaE?^R9F znxH=#>e0~nEr9gEgqC~r#*=C>K>;A%m$OQ`2;H`vP-kt?KQyi`YvcACO`jG7UYJ2| z2nU9vR`ymERfu{G#PAb!1Ht_vhXywAen_Hvw?Btk&10B*VXeB zfu#yS?(Os9CF784W&Lv4YWwK>xCWmVzr z()=D*1R9FGP!vZhdQD44p2yc$EU%C0@>+Jb_rZFmvn95(xx%B)HfzV3FVX4H;m~{^ zd0A*XTtw~HTzt;?U&%t%QjD+L?s4aVZYo5My7SW=teQ#9TMXI;TLRXiuaMjN2FR1% z(mDh5kd9BOyLivce856L{l|6>XXlK*?1d`AW44W?*TlU=%?n?7{)Sf6CDukeI@oq8WgL{Tal9&KKb969D@F;DmsX zILmLy4zwit5_}JhAx82>I~EkNKNg255bAXhFfQS7_TMG^t*E!a07xCssp)MJyh_A= zM|*5{UG+#P9@YAbk(6U;0ws@d=1|wc?lnQ!qb^DTa<_ekTn z!bRo(Q{0*?Bv{(Ckdz?fJdDrt>wKKPG=K{%;_3(Zx)-Z$JQN*j)%S@|t;~vq(&7o= zsGBrhnzBfL3p2Oyif)_vkqej!B2N)#ncGyslwLM3et`j^iu3*@Ho+MYF^LZY^>&M+ z2;zdsO`Q6wpa^Z!fe#H`mr2r&|8-O>$5 zeF6?L#fF1k&vMPyc|2;%YHL{Pz73BFKC!LiDYNL~rwp*BP^3)slUbX&O|h`cdiG4Y M>a4n(*EZk$A4xG#p#T5? literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/position_encoding.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/position_encoding.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7aa0717efd318b39a44233205b2c25f204707e3a GIT binary patch literal 3499 zcma)9UymF|5$~S=GxsN#vlH7fwkDA!yiT|awn2c96(@?3P_hb&#ly(bXuREbH+!=) zm+o2Hx0-zbJBozlPe71%C0>wt;DxWihv-*$xd)z+cmRIYySKKnjhMZu>8`1%uIj4# zRo}-O8wtbpt82e0b~YIMD@{%>4}*_z>w6%AWlV74R6OJM>0}O`Zsm;J%;ogXtGtn) z`JDZX310+Hm&>aH1tA60u00uq+nNm#nuoxsm*`P-QdL=~1K0wruL6 zN@YE0L|O0abgxlqdmz(R6}84>B-KuG0<&1Z$Bk3hpzD39TPg0!S~qGy?@ljrgf9Id zh-7epk@1-;*1lnTTm&L~5@(+DMf5GVhIv`=4f9wQ!nk$w))z~gJ{{ecf-L&7tmWj7 zNs8<4L8(*x73ru^GSzKSi$aO?fm9l9q|KhS{M4;fp6d93M2=Hkj>@WldySQAYH8Ju zwG`?7W6QhUwz8%@vFlV-dfbfDT0-EcIFjmvbYIp|6;(L_3wjM$kgQL!)ej@5Va z5qv+{`YvO4`U4+hy6EZPc5&-3Kw9>g&G?*g*71(zPUnESoeS!9o?wTuy4kscA4ckC z>kH>G|24P#NZk=`8w|s#ez-;+hF%~~r*5GdNQk|tg)V30=DSbnNh zql`By+5_VpmC#13^nBxKS?wuGd9_AloyZm~$xWQ+g9=9E`5)LnKioZNMsoL2Su~HZ z@%y_%wrY3O2wCaf(^%%G3A%H1tYT<+6Sv+5VLai1{cZ6Gzl2XNzcPt-P91~+@QG44 zuS(qxXdmbz3rY1Sxb@#SLtut%>VWbo0z)Cu=UWG1aNl4vcW$?T=n3BX!$3GsovGh> zn7gZa(6M2tz7^gy?EKCbDDAHu6f99Y-g7UoERf1MdmdJgF0yI7vTgZhVd3gMt+LU@ zFW9w^48e21c1d!!IcKkw0S2sO-k7sJM*xzZI*U|KdX72=rU`_6 zP!6Ob`4d7qeB_F9bV5blJ7dB{R%t2+eP}jsqs$b6$)ZU7y{ai%6AZ>9Jsugq1$G{o z&~k5Fe^wwSD1g2Da#1`NpxaguN~C=Mn3Nz>pVf$#@%HF!7;!tma z82(6I#xslRrp?K!EIiIFE@t7Q#lX1oNM}C$$lJ`Px_`8I92*xib%vB#a+WlP=m`&j zy5GS>UjbpZ_|~0-Z{Zhkk6-0iowxaUC*>)>hu>vBx%BFiTVT+w(v&%YWM)$&c4=b{Rr>4k>~NvHOm`dy1G`)>_=d|(99Cs%v|A7Gh1}v%$I0xXlh9ACteoXrWR3xne#6> z|CuaQEyd)TZ5X%j>!w2Fs3AY!w5o;Fva6s|uoYGjW|4X1w!RMXtev#Z3~i(1b80Bw z3p2lCvw&KTZ5U4fgud*BD#By2jilGatLAHB)jV9LNjzw(<#0<$E!+GEc{-9Ux-Tlk z<{ZKO={~Pg#%iW5)U*!a^(KCkcVD+FE1(`wq0vQ8^7IqjHs@M4_@%x_8viR?RQ@01 z)?^{U(zb=9geQ)N@mYS2Ptw;0@Dz)<`T@S~#cCH1Mdw)ceIisViz1=4cm_D?22EF{ zoTk5pnVWb;x6S;>1$%1&)V|Kb zevbbQj?X)E!}d|XW$?!?AFukMP4mH zYTTC9&ZyJ_8wwMAZrj9jX3@vb8DJwsQ8Cd^7cJ!$?Ymvpvq#5uXWiYtx_#+CcXv^_ literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/segmentation.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/segmentation.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..64ceb75c03bb4c1cfaca8528f33cfbadbe2d833e GIT binary patch literal 13199 zcmeHOS&UrSS-xBC)zw?sz1ZX2nM@KFc-mVMf;G&N@p#5zrsIqqO(-%Hx9i^8x;CGC z%e|y33baS#5ixNP+rtP62!gOmhzA6SzylIb2nmA11wtf{B8wM5;w2Bj_x-1;y4-e# zLA+9}I(6<@|9j5A|9@Yco6CB4KjD4-X~Xy@W9E}b{W*NYGp1qqM%nO9-)fs>6Tf!b z>eyx5WSi4=I*D?klPo9Y*=ncCDH)q?Wy%?;Wm}n6uAE0b(N1;d%5yR@*)DYE%k$$| z3$4ZSB1WXzOP%HNa;I1>c8-;gnZ{lbt?Bmh&PsWuvszv?jn|EC(J_Lx?X3EWY4God zjIQ+=<0Zq-_}SZrpA9mt^#i+nA~@;i-ZA~WKX=@!0pH<%E9{Ehonj$-Fw;45iUAI@0mTIqitJfZ|$!oif!1F_Q5C*;* z^)NN?o7Kqec;R+<#`QvXCuq0%%eC6QFbu-%ecst-$&7PY+#9qbPAOTj^GY*};`Ar` zdT9(u-Uzy3Pob8$+VSc^$&T}zK^O(T?uc{EZWO3~ukA%ae5xy5l}IL4YO2?n)Z17Y zJ0zR_1y$dQlPE4U{VD;Oe4KG5JcCML0Bel0<)_{80cJW&-5xpz zHrjK(0kBe=SL)bTJ~+XYuX_C+t_@%tO~W#b`+&tL<>N#+=m#oJS1P>eN+ovuAX0I* z?b&PB@WhP9sRU}DOzbb5Z!juTyCLtJ?E<>w<@0zV6{VcENEY~h#UPqwSdjm@C{ip%&eKT9J62+tR-{F++Unr@5bbac%Grx>9>Qhij$NN zr>5|I0pD-~M;qY|BQvsEcFj7l)N14mal0ejzP{N5^Rw2-!o6F5dd+C1T4~=}!%gEk zgXheFf&RoOIZXP-b|$rnkK6_ExVO#3{{T zLQqb_>j!bh530RRzZV8^;d!t)MYMWRsUC=^*#(>xu+)|+62{!kL7?_3L1!!Qp922o}5s8$KzI#+gXHG`cv*Bx{!+@l$U zr3H0@-A}MPT?@Qu01}REKpI;#>j+KcDLr>v=K=CRjP zer%&R&IqtP@^S`5RguFlFHR;-o0LmHq<9>)I0fFZ<0=0GcnF_FVVDbM+FY>kXXf$e zT6wfAp>4@7S|7x7+T35565@Dz$-GiZk#lvysld6i6|$m1oArDJ=is$2$#&4KM~x}z6Z^>Qh4lG0KzV2y zAU5b8kwvnHMu5MAM9T*IHV-Iu?M?N5jMWJ5#F;@i#KnQ#K|Te2+*WKuyy2)I-GFhF zUmEe$tDNRe3w@%MV7Kpe{pZmf3QscCb=0f)PCtARe!`EUm!BX*|$#uUy-7y*^0At2Sh8bpkHWmUn048~%lPM3~yXxo&{4(@~C}!mA&SOixEjkBrWYM@!2PV*7y^*_j#W zkr}z^476Yd*yhY0`I#B$kr{K-8EC-_FxHtFg_#-Xks0&T862CM8NDD%$KtqO1PoT> zd!|HXSeo{rZTvpEXL;Il2)FJj?tYwNK-8CyLOnS?Ohh>61M8lt?xD8QS{)`@Yk(tr z*D?)hSx3tWKXZWRd*;o1uR~7S&TN zW78|0RP!9b@oED_Y<0VFl1jw|AMD)QJ0E9WA`Wf>1!KG3Juj^nSiKzLh>W5kV zuv9<7>PO-fI9UaPn8aADu;Lm@gptzDSbF2*U%dO}4FGIpp;#S7bBvjd1LgKm@R zQbm#S;tU9eA}4~fwbk^(Qc`p3COXUMT3)~trH*_kjZ7c(@s z;t~8Hz99u3bQtSN^Eg=Ox>+<&&3xCbq_vLLbqo3pwVU-+nL7Np;SC>-jJyQ(`nk{vq2wKmyEi`_=&nTK{g_>K(9IkodTL+`kSuI2yamfdCx923_h+5(hNqyY`A9P!*>8&Oc(` z1?6qRpblqRyg$qLk2N6taP6t0ENTRCS;#z~3VxQ#1iC z5Wyq^^j5449R>-af} zcyfQ(vuQsAZ^5B}&#M)zx9^Kb07BYxFI;w~=9{az+!=SL(X2MyUbnsHssNg0R~z|< z1H)b|>Ug{E>2A;Mz|3#<+f5(?bYpjG&%Ncf2SMrE6Uc8=`C9z&m#4x*34Z~_jO9%0 z7Mum89XYK8>|NT@Nz_xIZwt1ud6#3-54B{_l5OQ6urjSYZS3iDdghsSO{P_t_VYPC zexB#PE$ak0fp0HE#j7lc@N(eFP&WZ5(GZ_+Jv*sdJuqK=s0kY!}b(dQkb*5z&&UwR=z<3skwhHmI{w z^Wa|50;ZhN<_Cm`24X}=xVJ1h!8ex;f^@{uFX0;=MnKbh>i=n6Mzx|a5?ft3r z;JJJVv!+rbg$I6Lz&HG76gW@V$RY>KmfR~9wN?U33A`UZtW}J#TS?fGP@;Lq+D4XN zsw229d}-Zg!0R2_&a_3xqaNs$R@l|@ODj16l_dvxL7$4znkyQxs%==?Hmz;-`@zrP7T;nQGJ2^dnn>0b?ytk`Z2apsZg6Nh=_{( zOtV|LUqLrfj*A8G0kc=Elilm0iBHSnBs2RWzM5~Ej#Zc!2KRugj>`#Jeg+j>Ae?n0{?cYMM}`m5 zADTswKc9;RoHnfE`hkj;4>kl<0#*Q3g%#1CSBw*|9Y+c1yVe1a?gaJRg!d)WFvPZb z0S<#f6o_B+^3@k!6zAv}H-xs>_7r`o^`ILlp#40KoONo?6<>um!Z>B}xdF%Q*li2W3>PrA31ES_!LB&C4rfH3 zst3`-yX-WhhP($XyFhP4kFSPl!@Uk~%GjB_)$9V2Dby1XTycLiYZK|v_O^od3?S`* z&N?hOUh|;^PM?#}^f7mXIz-GZn56Uwx-f{{^V3c^KH$opbe8n1X(RkJ?){MPO1Aob zHc(rEn`F>|TQaF%2;Luk4w|8BpA>mJE(>o;r*^jom0unlx@?Y+2HZu>Y9P*>-^B7^ z9)*!jgG82~auiMVGkE?s8x7+xpB0S@rhmd~_ZwcEsC%7`SEYR9CyOuZg1Y;tj8QiM z4kaiHr-KE5U2>Qp`oh>vtgufgpgck>F~TE_%%XRiejPGd8OEmJe%9Lx&z}Fjb06C{cj4T*3;L4&AKju-TdOCGw*!uT3B6L&P^Fs$1b9^^KWFxZz&bF&_5^+^5cqDyRhp zwi?qP0TQgf`2Ki;mPG#DX5ag zOJkpKrPix@ZQ&z-jCLKmSbz`JT!kMsjWTViUqnw8-|2^fj-QbFz|;enocjMH%+*R9 zh=qI``b-&p@cgA9zEbi0wO$xqQ@v^shSw288MCQxJ^O$wl-@_1dr9XsVS03xt`l z?J#6BO6`6z%0@Yibg&O*BDlB9fXrv@QD&5t&cLm(JGp?B`Uhzi3T1Bg&V5l6mjx8?!Mbo@=lFd{Av{x#?w zhAOe3q>IyAO>JRZ!9|Fkx5MT+=o|HJrP@=_Jwn0!I3YfLH9$XZXF9gD5ZiErtBagx z!_I?ux+me2)Yy$Jda5?!MVRtVW4qcP#F=0$FN4_LZ}zq3dYj#eXaIjQoML-XP%3Dx zGX>m$`51~bj!~a@v8S5#ik8C_dLY8`0w8#>1#s_H8i@1bXQw=kWyssaX0x28SAb8k z8O2tFuurp7uQc?}&Jn5;U%@{DLTw0HMfd~g6Iilbi0#v+W1h43SB@g%M5Q&=FJo#c zeZ&gaXmudKrEPs3=9%HvP}?9WsD~kI0NDWuNXQcK5bz4m zP^?HVjnfFsxeqk}sRQr_pjkL@wO2(R!DQa^*RHzXz9^r6d3LI?letWp41-@YOa^KX zTm)n_(8YoKmiO#L=cc5&HVtYY3IQ}X!%EbkpSI)mFS&=FkDUghdhT|x2USvOZC#&3 z(`SAIv`fB@_%i4y+3NM$VETAX=y+Oi?KA=mQAdRpVHtn8!g$dTvKcE<;GMpq;2&`z ziaj@s2DO@d`ij>M(IjJM*PC+BOAN%+kh=jnjb+9>^^!XdcOx5vs`|{Bc5f$uvC#m? z*p>ar*4Uzc)`o`WPdC{tIfv|82_ffaj|@zN2BP{S7VLgB@BR<;h#6+25Mnp?Pd)NhxvY1EQ=?cq0Z@ieox(<(ni^c;*Fo?y+?!OmS_=g@k=V&SKe(pke)okKB7 zYQ~Qew!DM(ZL3$F%2-L?u=J!Gsswe37-AVSVrkIAf4GHZUc zfT%d)U}x|?i=!pLJwIG( zEgT?O0(1L!tjSYrakxmY8G<@}1a;iE%(tw!?6;h^619Y+Fl9!|!)3h5Jkyu%+JMFw z6^9G-nfOcFIAL{p_Y2W-t0*jL*)QHsPj&~q%g8oBE`fgxw#>2NvDON5X#fRgUBYP{ zn+H5{Ub@B;)3uD=^T5*kb**cJ@Ec6W!kVjo3gsH0&?_(C&X4at9To22{qff4@YXA% zRouf0#4Yw4tqqUkO;_X`)__}U!&M|)L6N|lTQ_U38=qf@)>|hIj4wSgS|1i$Cx`1y z#$xzuxZZjU!uhzr_Kr0=fip5cWxQs5#()B`I0VKZ>*qGaHTA5=y)OXCH5aN4x}t@G zYQEDvAYEDEabFCzkZ7f6y+`GICPDT7R zhp4dOUah(9pcZk(P1rMdPn-~W`#9rnvel96cwm=J^(6SM))_y5T9qm!Kl~*tzK(B5 z-y)a_T%t&EDh$yW(P;~xm)1*x&QSIpiHWM;!C*zFhx&CCu(E8Zdq*^_I34vWq6IEs zBP}hU={x5BQ;(7sV-;)T={BvQ@R^OtY;CsvitMAP+=}8`MP>Toi^xk*m#U)*P-mcV z(W)OGM=ef2-S#?LzW2;&!i5&&Jwh~@cbFVVVMd!|8Q<_3EQb>VLqJH=g2$1rVTChe$7HHfj+oxzzXf|66(lkfB;FeAK^}wuq(vT9w)W>r+$qS?y%bmZYthyoFTjPl=AizU3G%{NRu3N`g=_BA!Wy@%@?14 z?ZxXi;_Sy^^xTl-)i}pGJ*VJs%z{KJ>lSoCW?M!*8b`>9lU7+v<+xcw95#Th-w(RJ zI3AFN5Cxr5Ui~TuBt!5U%uM+3ZS>1dr-IC~1T#BtmGOh8a!T}em`&r#4nhJ*>w;xe z&UX7qJ(GIuKpFPb?_tqWA~qZGvS_FkSxP5XF;Q=ZFDkkyVsmU(J?PL4e z=1WXY151Q)1v<#~VbFw;;_$hN4S*zH44B50j@B4)VpiQLjm{Uj{cp1nn)oVe$gn{M z44bIvYy18;*q&mVZ?7N5ORQ9Pt9v_*3fS@P-tL|bDDfE7Z?O+Z!4WLc>Gj5DleFHVt2!mFP|#;|uQh|{=m9aTQ5t&!iw!1DYN z85ieg&=OQ62|ki#TFxkMXPm((Rq^eY6BMjrIZJnePDd?gCa!}}S7o4fjeMQJB_)Cp z#?E#S^z}?lddTR(Dq_IlMSenFUB;?!qOQI{fD4GnK@p*f#QbPQADdvBDzmoB;tC3# zAqyRiVH=H=NsWi}4=DP{)Zeop>8XEULAnzU17&#lBX}}0X^7S$Vsh|dSm{L4JY^RV zt1BY$l7CN14)k#+3sPL7YtcIX#1h&p%z6R@s(6ySa=nENN* z1>r$8e+U4m9;A*sMm|SCB|qY2)?Q)ZP6|vs%+dSHI7w(A!n| zfDRn~E9#(|Uc0S>51}GL!f&a7R&_dRnj2)JxWIK5Yb;h+FoYx#U2y=5i%3r8@DO#42m!?g$}n7P&+{`gMFpQ9|IkmkfBlrA{3AU~J{Bs^;v0TURTNKYD4yzR zf!a{`drOU_-g0AERo+ngHAC@_ zZx+Sxs90ZBepu;ipH^O0ynUwu zg_b#Q9xd}}%X_`5w{S~o*xq~m$Na~<#e15!cb$xVM6_E9uzxq2+{k5-lgw zmd8Eqno>WtzgfKA>4tWz8~FB63~qEiKeVHcZ+m{L+wVr*LEj#&APoJmc%QCrt*#~|EbeXx5lyLU ziScSTj1ub;L%B2tO;!D?G5B+xUa#K&2$>Ep2^c)URM9NKNtRCR%V>MPBE!M>O z-IZ7w>j%n<%3GC@9_wSnQx8;C**f(GMs1iIIWF^>GBQSHWW@&lOakYQfi((Ik)D?n z&mf@MdTiX((O&WtfR$gn+{V8A!3pL)F&GYTZ2(*8Dw?9)2P{gxoS5PE&=-m2ICRsF zlNg>KiKIAmh1>ISzF|@k{?>L^_@UDluID7BNVxs5H4r^a%zI96;Q2vf_6Zx3%4@GY z7vU1<8eemVNo9R~u*rM4Ff-H;sp6a zQ;7+^fN!{pBaLu(ks4_mdP}>hiRH+M)tl-Fx4vPHaia(7o7zajeQTb2T-nHPSe|wq zH!PpWg#!is<|r5EJY};GUXO~Omvc+8d7zH+yiPgJ$2qEP%xqL*omM(9FwYtlxPLab zFt*6`IjWE650p_UF4D-jM5U+S(?(@pqbmM{M=r$0xOAY!WzTp@K}#*Rczgk{n0NHi z4Dj_BcAf#OIf5O(`Scg>u#s=;2l}WISH_FLOoDDHuHgMF>`A{2(Df3FUW+(mGUJ4Pk2}M(U5Sq@lyo!oI2Z^o(b1a}7+5;W zMge5w@S!&5Clj+KVG|%+Jce452Upl}h4+3ugpZ?8)H&5s=QRAOW&GJ%87)<`RrQ+o z0X$pk{#-_C>GZmKxt=GU)CUg&PbxaZWVQpUmmQpg8^|7}VUfo^;uRD)Mfp-q2p-2d zgs!fN_tGE&Qj}59`G&;xEwxhS7GPB_@cZqklMy%J7p=|2O>YANfHnbgf_@QcMA(q^ z_&YGiIvBHhK=M>y6LpN0C~YK#?S6=h10#dv@rSsrM2FzQQ9*72<47_|#1pU4G&?Kg zi8}u7(CvHAp*du(ql#;&H}TCr^dj^LKZYU`Uur|o1enz@d=q~;-vsR&_w-bdLBv6j zStKz8@xuM}qK(hBF0ncLVQGIB(!Seo+ifw}9@>4#fK{IGsUdSl>VhWWft=ETu6_CH zn(YojDsHpGW1ACjNkr25vY7l+^@x}RTU$}U+%ztAE)6ct6{VdU8N_5az+Pk3%XxXv zVzFpa>p17>V6_^=VLlsW5VM*ZX=Nk1hesFE(cBV4AU`mpI5mTq8LeN+W}pQ#z!<0Y zC{N8m56_s%W}pQ#z&fX9RHkO2hiA-YGiYpnYV;hd6Z1TWL`Z>Gr1#7pkTOusdeD}> zkL+2cg^&?#UC{;h2 z3TH7S|fbh|x3>}>bDqzE~JC@(31 zU`XUdP@~qnZdlJrPTfUk!)gs#cL^IM42%5Th&36A5^PP?Fj0FGMrC7$GZxg^bIY?5BXHP&!OYLp*aSz95i6=m&#-V%{G5M6~1 zWQ@z6b^wBwzni+a2Q~{5PutV}6~IBGqx{AU2*3brx~IHN!~tFSj=rQYER}5E*+=X< zm+DQA+!v_6_&w}ApY2?I#Lf%YnV7-1+xhPIuyZZj`Pd_NUQByc;`dWxN)T=(`M&Q$ zkzi{q>_T1{q&PDP*Ow#}>PUlP z8h!c)ny7&`PRVCHEq>dlqPli;Ewmf`~H7JB~MKLe*eSMf4+>v%NyK zSE;Zk#e}pUqXv?-FX0=K473bGE1%#tI-Be>k&_RJb+5>R9mo#L)bL6q*oo)EKDE38 ztp7u?UKUGOZ{K4JAEMc{FTQAJ=9n#c+gW?3(`|O_K|k2Dg%5qQFOBxYf#IMP_1slZy;k7Ao3JPrXB9tcO|(Xeo|MurUf7WwM#GXKU&i_6qtdSwB5n>GLe@yP=&> zpu%YjQ0$yJ7Fjx66JiT+5*p(3jjz_{4@FT@AT^x59^w>w#gCv!@-Ir`TFlT>?$s{Q zQDSX5L0a2PYogBs_I)u39o{9G-F2bgiz9NxYy|`8i51eX+yJELv|M;eq%>(1q^$vo zA%PeXB(yg#IYBd<8W`6IHNS{&xQK%7Ts5bb$;yRNF8}tI9yHrCQ9;M?LCnfTL>>=B z$}iv>{u2t)iBndij`=!CD;wkTpeJ*QgVr~4Fb$zGk8!oqWiNEacUuzL99_h2q%bdNw&%t z%J{{qPtJ!SyWJCSV?a_SQw(M?oOOXCZc$S{rA^6>nb{IHK%*x!F;TY?byM1d+Eypg zHakgP8mQqRbCQ@sT#JY_B%78n<+S<%bycmwNIS{gbARy&o=g~{CSuIEURT%ZM|q_9 z40_(Fk^WGg|D0%tm$*ho7W)vwOZIa% zle;agK=QP@V!aC|o7l726Cq73nlkxZheI`Ww}J!1b&G9$*jKi{%MPl;8Idd6e)RAz zJKdHJuUxTkCbpo$<`+y{adIEyY&W&zsA_W3>-4CV!34C(4VZn6G zhZZ<KN#eTIYJfi19h7|due_f%ZFtYO3?y! zRH0SWRPnQTPDS6JJtIk9OH4NycHG2lyS<*=)@SqvccEX{`AX-vCZ0q(DIAH%ZuEvO7^U3@@EK=Zf9#jn#o@ zpL5s4a~Hnv{Kr?%Up#;QqP(B~$MlR&Z2Nn{bXtdT>5_eBFo}-XZ?K1v+%2RTfckN9 z@XEJ+YfEcSU z*)Q!5;pqXDr_mwSCPB%EV4b;OKVc7@h#)!&NeUnsR|pLZ$q^blUQYZBIvRPtB|OM|Up7D`JKBz>FsW*?Gq^a)87$Xt85%(2Ijsj+#$rtdANvkmmY zrI#lYFPXhM2&1cF(DcLb8p0nboBGBx5BNCAt*Fx;B=)BibW+DPR*NCl^=UOY>OeYl z27y=Cq>*-jnUB^y|DG1+w*bf7Up@j*rf6m?vV!qsH86|(cT^Bj(Qm3KjhpJG7S%}(uaXZ=A`dp&Jzb&4mc{)rvxSaV_OemH zCYc|IZrhP^*df4eMs-d3O_m_?$<3yWP#TZlT))8xe${m^S4>%gi z`+O%u|Ci{L2o53v1RD%NEsrMgv*^i49Nv4+Vlsz_XEVP*j^A7ARPec&EKI$^Rku$9 zqPhQBHuj}1MFs~VSJIr9@tY}b4UG+wf>baB`Vr7aq=A=!6M(MpEQuBFB||g#k&@%0$ zAVd?7on9~Fo|h=X(n7ig*@$k>(okzlb-`dI)C z&C51fs~d;xS}-A(r;iLwglt3c2UJk^qj~p#qK7RoC6DO1x_|N!*p_kdwaiCw64OZ2 zrE3qrj*BM?Yts_t7=dpva*f$JSS{GO&FmanFRXgx>~IMSUnf?Jw1-hnGJcfiP1QjA zrY7FRlbpTYXG{L`Y8#B4Lt4ta#$9c<#V%B^ez&X?p5xy+Xx#K% zK*C0WVuKJ6Y71-LHsVso_6r^b=#jz!CsvWho`iFIl4#leL1} z^QI>E>uXmD!mm;O6xLk!ER@INxpd_@-1*|}Q&HtM-XCv$9&f!gTE;yrLD*uy(eZc@ zZ@R?ia2%L*JYM#e?!nE7H`lhjHsfaTlS7qR4ZN?zXzgu z(ObEvjZWZ<)K4m}E1y=NKFr4_XoaoMLs*lRb<+JDP+an$)^?wDP|z%U3MAXjxZ|P8 zZc~C1XxJwYtRH+ZoHpkVt@m%k{p>UlKh60lkBYV>UgRNLry0SQ``|7x>wU=+c~gXG zo0*jjCIhuPI^3Fg9#mb@9N}(ap*_7FX$~w%NdAW~P00O$^orDr5rPx8;243ZpNN|- zC2`S#K#+io!GQr=UuKTs95U8L`#$txcp;4>^t7z)wh%V6JvVY~cl$80$6nu#z!4$I zeON$IlX&xkFEy=({NLUPY8F^DOdbkWP6Q_6?W7dqQa_urDX?x zE20(GV9nq?aYDq|(-gVMR!6SmfnCr@iEjA z>#4x)t$Xg%XE5(S(4u^oU`^&7ru325hkV})_=ZnoIr7|)x8^{DzmW_Mr0{LPs|dLP zUM((B%=|XEeoI5mL|@J`zz_9)6&~q4`(iBWKj36A@zcA{-d4#V0Vn$iavPA`N1=&L zLo{|z#|Bzg;Fr`PFZ6MlQo&?2pFG)H!#9+z#v96Q{Wc^kxL^*(&&;?I&0=i22k}En z0=r3`Q7g7Ks+2+o*k(2s5Hf|i6Y1j`AU%fUEsHqz0r~GONKoV(9O#@tR=}9$=y+V< zRdn#*;_hm+0$7m=!#wyDnM>mn3}XrNPhwZ_(Nxydz%h%l)uJpJgdgcHrLq1#)Q|^b zdeI*8u5zA#10J{3cu6eK96L1yxq0iSk^<)+4x#9YcWH?WRIsPq2Uqy40 zgYORp1v}T1oYecouhN9u)UEkjf{e|iK%CMQ!rc?3xiH!zrE$AA+@rJ|BI+c+_R@2& zzjW<-QhXlP&UH>SO-fWJZz-4?Wh^4?bREhcWkyB=vWk$$#;v@TDA8sndujmNaOn3v z_BbG~Ao6?lviKDmkPE?TFw^wl)EG95o`YO5ld>>xH1LC`MxGUR*h^_;2VnptYQZFG z6#GLYlW{#Upar|)_poT)Ow>-Yz^bXkOUYCzO0Jt?iQ*3l05!Fzn2O49AY;qWp|zgC zt;x(TqpDT36|JhlIgiY)qIyazYe-4ZA4j_ZE9;cHf8o1g0@LdwgdqM3ATV}hOpF*9 zQ9M&0sh4fg-&-}{x;oNsLvsYxDVyq-v@L^Jg1AmA*eH-RtgW8s14z^+dj4k8!x4{f z6`vwOM$-D}>-1)1rR0?TbKe!*hjW3e+`$WIG z_A;fLfg8fA0^%2kuw=qW(D0dw@{fev)_^kMPDzA)k~a7)Dj4g34K-xNAZvx14ydrG z_zh~$Q_}5V7$#M!GoYDj~feNO~%`Fta3{ z8)Pa;n$q3=USdE{h2oEChSnV)!Aa3s)**!Y8Gn@w*tx?bS%+L@w|&DTsTnqkz(xejw$5s8&Ver45GHDp0l;90Dw`zO8wN`o%G z0su(%=a2e4K10xQcEc-FdzA`%QefhHss0!hzmMYI@Rcz;Vhbg119AKu^`Ajef4>X| z{wwOBj6o2{xI!ohA>oqR8-fUA4h0ds#6u_&Gr*e3RvPMS;g$0@}@?4*$cjtdb nFmUronT3CtZXGGKNT99Z&TF{SoL;S0jOxo(+p1PiR+s(Z2@dG7D-=@|~mA=mQSYwy@uI6K0Iw6s9JpePP3X&qZjC%bZ-OuT8grkmuD z^QGz@i6WW^vfN}HAOeyAIoMg>e6X)R7&d}ja}SVHU2_No{{tV4JWo~ke2J87c!%if z>iX*HdY|`w-nWWxuC0aflds0_?;6Jc8Vf%M`7hzheu5-1x<+Cq*1+tVxZ4A3Xm{O?;ZI^LAQpyGjNA~*B^%6&@|pM5;yTaFcL5IkLu@kw}BEr2~ZN0C2J_D zB_T?}vZRTUdeT5iqbylR$y(AxNwX~3NY;~$4~*_+vYE7SZzWsFHtt)A^{&x*>GYT3 zt=sqBY2Op^DBB;4VJh0aL7ZhfVff|o8_78^h6=ZEd# zIO*^AdvV?$kFq}u+xT5=PsUlF1@WMrj(TV@I>_3wNZS}POOtkg)PDObah|6mwkkWN zC%K5*clX+H-p&tG^i100eLVoC*3a4^9gZKSN!dP0F)}Jdnoq?D4!laHhc9KT2hP(Tr;TT;H>%A`P3yb&EungmAfMK3` zOblfI?|a|n?`MB39gz;EcY0y@x+d;EE*n?&K9bxxH;&A6!!&ZMFbk`23$O5t;M_Y4 zi$Gk>?IWi!&&`pQ7-#j|EyBVs0IkA^yxd>u(WqLVt>r<{D4b%g07#!WXk9Ceqmb`G zA0w*cO-9s641A8Y*D>XmUi+v&%2VJc>Bo86S(9!youonr5i0t56v-q4X7mFy7)Qx3ut8&!t{vh4f2!$jwNkS(-s*MTA+_LlM9D`#e561j zXYLA;tGF^MnqfB0hI#s#r7u_oNY7cBBBPX#7SGBVX3++&LX65dYS1?Cq)5_^2&J!D|Y#aqe4a#dl zDec64;-1xu`cW;h@m8w{@m}N->)bvHXC}ber*-%Co>C5iD=9iiVAZxW(Tk4L$54fY zCOI0$*|D@?tUI>6GK6szkDsX**U(wk9!yhIvS}UaeHc$t>F*De370;UeK^fiS(}bB zG)zxZ>4|ib#<_GAbI9;C72_-#^p8`q#VLJ4jjHg6K~w2Kn2$OQEkhdXqE>t3@LPquQ8|Ann|_zYTXG9PFSsbU^RVWQUPWOk4h?R_Gd_DcnlOC?Zi=d ztSut(vV7uxFCjD(FJaidJ2MG#>NAMGfrl4E6lN73Q3X_g4pea${k5oO_!Da= z>$qBIud#lms#IIGW@??dtbd6sQ;uAT&lQB;(vHj=p`Hw2&a`c=mquBc!fh6>ns0HP z6Csktn@D<}D!2KfbT?V@%L zjsrn&D8Sna@S>_ITMM*yl%vRA=mD1zDk0ui>=vGz?;1tz$Ol&;2Nc38yo*_hdrZE; zWcfoBaRI1GTe$GjizgF=1WI*CS`+aHEOwRCFMbdCl0L*O4tJZ$^I!4jXwq#g`mW*~ z_L_lyu5A&@Gs@iX9gF%Fn3nnrp?TZXJlyFYd8;~D$+yx$Ywr%oS^a)vJ){Es1!qP( z06h#eJ1A{r-I6xW*x{d1GtNk>2)YlZ>gh0e6#0I9d90qA}?fIzSkM~N)v+v+xQNICG#zC~rBp(N! zrHc|<(UONJzi`eO!vv}ZtyN78Paz{E)YKR9jfEa-3;AZgPAiYr8{h*Y+MLBg;Z<8D;!Q74FI!gN+Ej6N1dU{ZSeV=?~QXn)Gk!_(2U<6<-&d9Lps) zUemYhGAJXBYqQ*S*;4d6k36FNk$4qdMH@+{NJg+}b}&?ckHW&r9f zI8P1GO#>u03(qp5p4@U5aHAkm&B=PkEF?0b zPtEoO8JbYc(mA&rgTVkzoid@sAYAe%ZeqP`oY9##U=ICr^NNAcfW{P9Q#mjX5Ng2n zS57}oEdRp)kmqVHlq2*htwZ%^M|<{~HF#T1X6cLt&%|^Y{t^FCgW$ z%Y?Bwz>5}MxmkM2wFnT6VHqv838I{|nFweExa?5cQUq#a1a8{m@-xmti&6x-Pi@wx zT6l5DmFY7%Vp6WLTGW+==2Bj|(9c<8F=WSMCIcobD6h{}Nu)nOy zdf_UIbhgN}sk{`r>Dpsy5TGGNtghXIa|c?<1zvcDc0yy?XfMW0Mo22}=mPYte2Vfh ze~d0VDtA15!s)9on0Chzf5lgInb5JZZYhH<7P;uP@{`7V?{UtEBQR zMdjNoRQ}Qmm5aZ|7b#mRo5PCK6^o1aSflJH{X}hBzs($JTN_;(1kK!UYEw*$J7exQ zMaq^R2UY(k2FhMRqNsY6tS=($(nJ0D30v&*qpmSowquHx{u|{NQS{PuRw?=gQNvRB zfwuTWhA;S_9y)C08*v0Ho|(eMT3{j9reQMkB9V^f?PJu}j9+ z{ax&}S9AkNmyAz}3UH}Cb(WTNqx&EjoArSgr23F}99r3nh{u(#+8dKL$bY3pXYE4f z7Zk;e>M!J{oJ}$$ziu!!<1FbQPF-~n6g&J6hRx`B82+5E$XU(;cdhscOK6y}ADoOA zi5d&Yg{F|T&_l$#q_9v$y^BJJgM}Iu=PvmO>`y!c!SiXC&~I~%h;&lk0rV$sAyCd*KOpf7bl=F+A-FO~MME9gfYY$okh&892xb6lmI z7d?|zNiTUR7t0x5O0}WEJ1?ctzl#i(52jWLrSwXwO^owk*;`pGztC%`X`lBNM*UUx z4OWa9od^rQpx)*2J>`nwjA7B6x#;uY%}n^l2TZg#Q*8{a-alryhD`n+xU#EAlv}ga za^`yn*IO6o1^e|>RD}%h4lWy)BAgtWqIMH-BJlyjZ0f)*gJ@SfK1=6A?6}2_3@Q&G z%lDdRCI|QqI{b_E0RE;z7EUdk9?qh{lxaF)j;FIbUzBDa2VS1uXV(~f9V7iByfMHD zZf`uC+}xx{I7!bqeRW~mdFRGp+=~a#lOe~?7^{FHE;pWDJ44D5b6OMl|;QM&Z`TvGMJe>)y69PhXHrz4#8 zk4No;{=*bC_#Xg)qb;RR(mB8}y0i}l)6>`HnPvZd zoo(qyQ8Mn~sE2z9(#s!DI8|a)P>b}2=GN?gEvj#e2PHZe3jUGp8MR4IV^#q{?ZK;k zaTSy(h1FZx6pnT>=hId>s*Mb8Xid6F3#(3l9ZY@$AJH&3?A9BtPqsR(8?A2yPO$d> D35=zU literal 0 HcmV?d00001 diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/transformer.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR/models/__pycache__/transformer.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8e00b3332583fa7bb3f462d40de05091169ef664 GIT binary patch literal 8983 zcmb_iTaP5kRgSz?R#tU&cXjWM*Is+eurgbtrN=WXanmpuJTq%xwhg;u4273;O+|Kh zb=Peov*yyu7V^%*8pKF|v*Q#A69H(iDA&Gni_ZD4J)0>FzkeucvnbI`X3AF$Cbm{6Q@&0Nks-I z35t?6lvHJilCUUgprj`2D5)1E>nK^14U{yBk`1{oH$E1fO}QzXxHsjN+{S%N+8>Da zE64vA-n@19{nlO89VL5XHH=lOH|QqGP8fcDJbC={lWvm6tvxj!w$icc9qf!oJBy7U z_S1vba4h?K{a!cik4MR`gf09owZYyqAVyEw8t>%;U~2uOrQ+fEK`e{*GRDZLP;oj{BaGil(r)jF zeY(B0{~(`IKL74u&@$~Ylh&a7I95sc3=$N(ZCeN5oaB?$?wx*;qQoAJw0}1qCHQQ% zDu2}?lk6Z93rJmbY}xok>{&99)sL$kM~1S7HFjlPuHo)|B3#k&vEJ*twy=25ge}Cg z0K+`-tHL~qPBsng3G~`e1s$wPsFKp21uoLW@UEfWqwx4 zf+zk-m<8%`>YTcnb!LrhDNbssmxY;=0a}@e{Is&tqh7W?SxbYgp1IjtR+ZLM7p<$A zI1Tw8^bt`FZxT^m3Ve>e+qU#pzV=anl*Yi1?03_+y{5fnI*FAIB2@I#DAJb}zWf#e z5vw~Cf}%R$yHfx7v8LtQZt!g!0L+{}e8kE9#yk_$ zMSyKQ(B9}E?#gyUyGcCQQ=h|Z)N7nlb2J@B#e1V1FL_xpYFV}}`@?82jwSTxVOL2E zt;Un_G}Yl8gL-3xZ|e^F$8mHoM$1@VI%nY`jIGy}UaITmBS+9{OK;cp@{yI0Mm36z z{zlP1iXUCRaxfmoS044d<44f3{VOs~)fMyE$rUJCsq#CM$7&s)yp1dQJQCrCc3_28 z(`uM|XyM5YY~MQGUXadCDcqjLPWS=Pz`77$$7K}c(OW=|I1@L;hp(SlnMK4(>&R7M zY9AH|wZ#4K+c2aCQO`J3+tcBKtQ@Rx4ilGP((8}95Bjl+rlUS>R@Al?XRWP?`Z?V5 z0d*Yq=CzX?cpCX}N zr#7v-bj}=np8H&xn>nWrYOPaOI+_ZFYMexa{!y&9IHgz3s0M!+G}bPJ`LJEjW$0j{+BLJQ z$nD$h3XR+db_GT*jA&J+1Lo^dbbs0% z?+F=%&{SJ?+JVK@`m=-L}+MIJQAALEFTY@J#`|2mCd*0#C7KiIqy!CO1$s zW2K5=B_PGHQV_eb_nKHtrTZzc1<%9l_=cq_z)w~2rMo>UGJm|2c6cS6-757wAteCS zNP$(Po>69Nx9uzPmic&Hi}qoHdxP;PPPAVb)R0ED$h*ddxdsR7XV`C-$v2Ud%VN~q ztPSne*Kj2ikAMv87Oc}37UXy#mvj7`gLS7>%@EXC8!|n z*7k`HX7%uSPUeymPrUo}Zwu6d`j4vWmtf(n%!3kpnU}MoqXX(2Q9OSgtkioLJO7?a z{*}MVg5_ChAJj1cdw6D9=FrG7LG;|lm2hE&>)QP829O-u$Dcpv>q{_07|!nQZaYxl zKtJ`HOn!?A7g8B}K@1sms^4bLtTmHz@y)su8l13M_rPvch)DyOB|I9bu-We*nc*=o zcyy#|@K{|$;zfDsed&VPXn;9T7oC*h;(1*ET$OTaD$(j|KU+JWIAOKp%} z0Ba|!o`K^)&>IHujsd(XYl_wit=%&^i0p+P$U!;OM~2We!pr317gz|(k7Zum0z6F+T{z7QpHZ>1-`bYk%4wmvQb8)xkBm#CR(W7B9Otv_OxT>oU{(>#iCJHFg{UMFP@e8H&3 z>r~|S1@#TNj`~2pazXuu+(doV@J$Frys9=Zt>y41C)ZvD75wpD>Ui>fJS^(>p<|*M z+GdiEo7$%yN*uUJjCJhOO6Hz1b_q-kTAP|GokB>B7V?dBJ@cNh^#=Gr zM4Pi1NPP)2YBv{52q{xH->KvepOd)=5d1= zt}MQ$HaV6@ZoHb`uIZqNFs{yW*L2g+>pbF!_D1RrbX6@R?SgM+(T(~dt2pn4M&}x_ z@1adXmrVfbO*l_=&`ljF{!ntfxOY%c?j6Q;bE{_MY)kvhAMfNFW1i@dWr$FZ0#(mw zC=khBVkBVg8eIfw8`s{)J^>=}+}bTGscvu+g8pglAO1ClO!xv}9e?F{M$9L-+y&gI zNK|vOo-qrFjE2?B_7O6SXE00m%ytEX0h&5vLK%a@C4cHk`!#VwXI{V@R?e(T0-*to zDX?a8U>+b;!1WivCQ9?aqEE2Ur&{zW45y>L2i?IG-Zl}G^Tft;v|u8rQ0?=BzQd%) zq|4;HOy*MmBR)zd_mIFQajlhcN|u*HTlckfgb-QvuC3TeT`$ZVC6N9A10+o(!rL$y zX}~S!hPqg;^YPEVbd^;+TV&c)#u5hIbnWRNs8_K-h}CoV;LL?q63GkC&`xMf3+>gI z$p}g59i4-ol}|A~=0BlJ9+kU3KH>O{mrT3usz2bXI!x%{+|K=30`J z=lm{nFK~3pNq>whq0MCQ)!1lyq}Nt`o*L$&dd+^c7yKpvf#j#5+H=X%e-Zg!(BlPk zd_j-r37Tg5$0umY_QeFfLfIZ@)(60H%C6ukC@7mGjU5u0L6SK(jZSMciO7?t%RFhi z%r}rPd#^9#Hx}}nX|tg6EkotoD^&i<3YDur#1|P`svESi!n@&yl%rjx|uJXg7NeIUN-L!NEeI`iUKIbso6_sX+bx69|U8wcft!YeMmem zt?YTk<7vi{L&UrE5keF7E(#qE7FwD(cgaWKek}cj zVhoczPw07npHXko-#B#U{d0i}SSJ%nQ|zeYJL|JPFBY|$V$swCCd*KOpf7Yk=F+A- zFO>FAD(HtCY$okh&892xzqkrHFM1}cl3wys&X+U1lyXCZcW$E?FXb1IFz`W)!gU3iBW%?eS;NaMkm6CFPQK0RD9!#;f!I?n>p|E;LS|<#`{ci zZ>HQBSiOJ7ZVj3IUvVXukr=mTtLe`74z4xN&nx!pnWzdG-sw6zxD4T>5QW)Iz=_1i z2(vTmiFH#v0ou)u&(ir2J8rQfgUWr#^25f7#R0xA&aA((@5A3T$ik_G+rt?&m@cB%Gu#j^9|=cHX%@827q^?=_|$1A<+l!%4W z!vqPqK7xEg^K!`OvBG1Vr@$>7+PjDkOpI<~G#*3df%+GiBeUmD_m`1FOCL8K9`vP* zM-k7+*h~B7Kr=BNJ0T7a%l-9G#$~<}OjH~dXNGQ9?LT*JvUzXXAIK=3pAv5KbDMaB z!0wm3^r!v=rAwdBB{k}L1DT9Rezt0)SOlu4wDI&ho^V=)CkIcL@#K6UQk;He;&1Wa z5P#=9u|BdSPt7aGZ@+KO_gmf3!m)m9Z#u#m|9I5e?>~r9gR_5yqb;LP+TF+Txpwvk z)8jYinBU5e&=9lsA+S|GkMRMH3Q4jYJw4XknaH_^A5a{^x&QzG literal 0 HcmV?d00001 -- Gitee From 55b13a1573809a5304c6e4bf17460fbecb7e2aca Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:06:36 +0000 Subject: [PATCH 47/59] my first commit --- .../contrib/cv/detection/DETR/coco_eval.py | 81 ++++++ PyTorch/contrib/cv/detection/DETR/engine.py | 153 ++++++++++ PyTorch/contrib/cv/detection/DETR/hubconf.py | 168 +++++++++++ .../cv/detection/DETR/run_with_submitit.py | 111 ++++++++ PyTorch/contrib/cv/detection/DETR/validate.py | 261 ++++++++++++++++++ 5 files changed, 774 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/coco_eval.py create mode 100644 PyTorch/contrib/cv/detection/DETR/engine.py create mode 100644 PyTorch/contrib/cv/detection/DETR/hubconf.py create mode 100644 PyTorch/contrib/cv/detection/DETR/run_with_submitit.py create mode 100644 PyTorch/contrib/cv/detection/DETR/validate.py diff --git a/PyTorch/contrib/cv/detection/DETR/coco_eval.py b/PyTorch/contrib/cv/detection/DETR/coco_eval.py new file mode 100644 index 0000000000..b7bbe6bd8e --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/coco_eval.py @@ -0,0 +1,81 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Train and eval functions used in main.py +""" + +import os +import torch +import util.misc as utils +from datasets.coco_eval import CocoEvaluator +import onnxruntime + + +@torch.no_grad() +def evaluate(model, criterion, postprocessors, data_loader, base_ds, device, output_dir): + model.eval() + criterion.eval() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('class_error', utils.SmoothedValue(window_size=1, fmt='{value:.2f}')) + header = 'Test:' + + iou_types = tuple(k for k in ('segm', 'bbox') if k in postprocessors.keys()) + coco_evaluator = CocoEvaluator(base_ds, iou_types) + # coco_evaluator.coco_eval[iou_types[0]].params.iouThrs = [0, 0.1, 0.5, 0.75] + + + # ort_session = onnxruntime.InferenceSession('model_file/detr_640.onnx') + for samples, targets in metric_logger.log_every(data_loader, 10, header): + samples = samples.to(device) + targets = [{k: v.to(device) for k, v in t.items()} for t in targets] + print(samples.tensors.shape) + + # onnx_input=torch.randn(1,3,750,800).numpy() + + # ort_inputs = {ort_session.get_inputs()[0].name:samples.tensors.cpu().numpy()} + # # print('inputs',ort_inputs) + # ort_outs = ort_session.run(None, ort_inputs) + # out={'pred_logits':torch.from_numpy(ort_outs[0]).cuda(), + # 'pred_boxes':torch.from_numpy(ort_outs[1]).cuda()} + # outputs=out + # loss_dict=criterion(out,targets) + + outputs = model(samples) + + loss_dict = criterion(outputs, targets) + + weight_dict = criterion.weight_dict + + # reduce losses over all GPUs for logging purposes + loss_dict_reduced = utils.reduce_dict(loss_dict) + loss_dict_reduced_scaled = {k: v * weight_dict[k] + for k, v in loss_dict_reduced.items() if k in weight_dict} + loss_dict_reduced_unscaled = {f'{k}_unscaled': v + for k, v in loss_dict_reduced.items()} + metric_logger.update(loss=sum(loss_dict_reduced_scaled.values()), + **loss_dict_reduced_scaled, + **loss_dict_reduced_unscaled) + metric_logger.update(class_error=loss_dict_reduced['class_error']) + + orig_target_sizes = torch.stack([t["orig_size"] for t in targets], dim=0) + results = postprocessors['bbox'](outputs, orig_target_sizes) + print(len(results[0]['scores']),results[0]['scores']) + print(results[0]['boxes']) + print(results[0]['labels']) + print(postprocessors.keys()) + res = {target['image_id'].item(): output for target, output in zip(targets, results)} + coco_evaluator.update(res) + + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + coco_evaluator.synchronize_between_processes() + + # accumulate predictions from all images + coco_evaluator.accumulate() + coco_evaluator.summarize() + stats = {k: meter.global_avg for k, meter in metric_logger.meters.items()} + stats['coco_eval_bbox'] = coco_evaluator.coco_eval['bbox'].stats.tolist() + + return stats, coco_evaluator \ No newline at end of file diff --git a/PyTorch/contrib/cv/detection/DETR/engine.py b/PyTorch/contrib/cv/detection/DETR/engine.py new file mode 100644 index 0000000000..a1a78c8342 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/engine.py @@ -0,0 +1,153 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Train and eval functions used in main.py +""" +import math +import os +import sys +from typing import Iterable + +import torch +from apex import amp +import util.misc as utils +from datasets.coco_eval import CocoEvaluator +from datasets.panoptic_eval import PanopticEvaluator +import time + +def train_one_epoch(model: torch.nn.Module, criterion: torch.nn.Module, + data_loader: Iterable, optimizer: torch.optim.Optimizer, + device: torch.device, batch_size: int, epoch: int, max_norm: float = 0): + + model.train() + criterion.train() + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}')) + metric_logger.add_meter('class_error', utils.SmoothedValue(window_size=1, fmt='{value:.2f}')) + header = 'Epoch: [{}]'.format(epoch) + print_freq = 1 + + for samples, targets in metric_logger.log_every(data_loader, batch_size, print_freq, header): + optimizer.zero_grad() + samples = samples.to(device) + targets = [{k: v.to(device) for k, v in t.items()} for t in targets] + outputs = model(samples) + + loss_dict = criterion(outputs, targets) + weight_dict = criterion.weight_dict + losses = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) + + # reduce losses over all GPUs for logging purposes + loss_dict_reduced = utils.reduce_dict(loss_dict) + loss_dict_reduced_unscaled = {f'{k}_unscaled': v + for k, v in loss_dict_reduced.items()} + loss_dict_reduced_scaled = {k: v * weight_dict[k] + for k, v in loss_dict_reduced.items() if k in weight_dict} + losses_reduced_scaled = sum(loss_dict_reduced_scaled.values()) + + loss_value = losses_reduced_scaled.item() + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value)) + print(loss_dict_reduced) + sys.exit(1) + + with amp.scale_loss(losses, optimizer) as scaled_loss: + scaled_loss.backward() + # losses.backward() + if max_norm > 0: + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) + optimizer.step() + metric_logger.update(loss=loss_value, **loss_dict_reduced_scaled, **loss_dict_reduced_unscaled) + metric_logger.update(class_error=loss_dict_reduced['class_error']) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + return {k: meter.global_avg for k, meter in metric_logger.meters.items()} + + +@torch.no_grad() +def evaluate(model, criterion, postprocessors, data_loader, base_ds, device, output_dir): + model.eval() + criterion.eval() + + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('class_error', utils.SmoothedValue(window_size=1, fmt='{value:.2f}')) + header = 'Test:' + + iou_types = tuple(k for k in ('segm', 'bbox') if k in postprocessors.keys()) + coco_evaluator = CocoEvaluator(base_ds, iou_types) + # coco_evaluator.coco_eval[iou_types[0]].params.iouThrs = [0, 0.1, 0.5, 0.75] + + panoptic_evaluator = None + if 'panoptic' in postprocessors.keys(): + panoptic_evaluator = PanopticEvaluator( + data_loader.dataset.ann_file, + data_loader.dataset.ann_folder, + output_dir=os.path.join(output_dir, "panoptic_eval"), + ) + # ort_session = onnxruntime.InferenceSession('model_file/detr_640.onnx') + for samples, targets in metric_logger.log_every(data_loader, 10, header): + samples = samples.to(device) + targets = [{k: v.to(device) for k, v in t.items()} for t in targets] + outputs = model(samples) + loss_dict = criterion(outputs, targets) + weight_dict = criterion.weight_dict + + # reduce losses over all GPUs for logging purposes + loss_dict_reduced = utils.reduce_dict(loss_dict) + loss_dict_reduced_scaled = {k: v * weight_dict[k] + for k, v in loss_dict_reduced.items() if k in weight_dict} + loss_dict_reduced_unscaled = {f'{k}_unscaled': v + for k, v in loss_dict_reduced.items()} + metric_logger.update(loss=sum(loss_dict_reduced_scaled.values()), + **loss_dict_reduced_scaled, + **loss_dict_reduced_unscaled) + metric_logger.update(class_error=loss_dict_reduced['class_error']) + + orig_target_sizes = torch.stack([t["orig_size"] for t in targets], dim=0) + results = postprocessors['bbox'](outputs, orig_target_sizes) + if 'segm' in postprocessors.keys(): + target_sizes = torch.stack([t["size"] for t in targets], dim=0) + results = postprocessors['segm'](results, outputs, orig_target_sizes, target_sizes) + res = {target['image_id'].item(): output for target, output in zip(targets, results)} + if coco_evaluator is not None: + coco_evaluator.update(res) + + if panoptic_evaluator is not None: + res_pano = postprocessors["panoptic"](outputs, target_sizes, orig_target_sizes) + for i, target in enumerate(targets): + image_id = target["image_id"].item() + file_name = f"{image_id:012d}.png" + res_pano[i]["image_id"] = image_id + res_pano[i]["file_name"] = file_name + + panoptic_evaluator.update(res_pano) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + if coco_evaluator is not None: + coco_evaluator.synchronize_between_processes() + if panoptic_evaluator is not None: + panoptic_evaluator.synchronize_between_processes() + + # accumulate predictions from all images + if coco_evaluator is not None: + coco_evaluator.accumulate() + coco_evaluator.summarize() + panoptic_res = None + if panoptic_evaluator is not None: + panoptic_res = panoptic_evaluator.summarize() + stats = {k: meter.global_avg for k, meter in metric_logger.meters.items()} + if coco_evaluator is not None: + if 'bbox' in postprocessors.keys(): + stats['coco_eval_bbox'] = coco_evaluator.coco_eval['bbox'].stats.tolist() + if 'segm' in postprocessors.keys(): + stats['coco_eval_masks'] = coco_evaluator.coco_eval['segm'].stats.tolist() + if panoptic_res is not None: + stats['PQ_all'] = panoptic_res["All"] + stats['PQ_th'] = panoptic_res["Things"] + stats['PQ_st'] = panoptic_res["Stuff"] + return stats, coco_evaluator diff --git a/PyTorch/contrib/cv/detection/DETR/hubconf.py b/PyTorch/contrib/cv/detection/DETR/hubconf.py new file mode 100644 index 0000000000..328c3306d0 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/hubconf.py @@ -0,0 +1,168 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import torch + +from models.backbone import Backbone, Joiner +from models.detr import DETR, PostProcess +from models.position_encoding import PositionEmbeddingSine +from models.segmentation import DETRsegm, PostProcessPanoptic +from models.transformer import Transformer + +dependencies = ["torch", "torchvision"] + + +def _make_detr(backbone_name: str, dilation=False, num_classes=91, mask=False): + hidden_dim = 256 + backbone = Backbone(backbone_name, train_backbone=True, return_interm_layers=mask, dilation=dilation) + pos_enc = PositionEmbeddingSine(hidden_dim // 2, normalize=True) + backbone_with_pos_enc = Joiner(backbone, pos_enc) + backbone_with_pos_enc.num_channels = backbone.num_channels + transformer = Transformer(d_model=hidden_dim, return_intermediate_dec=True) + detr = DETR(backbone_with_pos_enc, transformer, num_classes=num_classes, num_queries=100) + if mask: + return DETRsegm(detr) + return detr + + +def detr_resnet50(pretrained=False, num_classes=91, return_postprocessor=False): + """ + DETR R50 with 6 encoder and 6 decoder layers. + + Achieves 42/62.4 AP/AP50 on COCO val5k. + """ + model = _make_detr("resnet50", dilation=False, num_classes=num_classes) + if pretrained: + checkpoint = torch.hub.load_state_dict_from_url( + url="https://dl.fbaipublicfiles.com/detr/detr-r50-e632da11.pth", map_location="cpu", check_hash=True + ) + model.load_state_dict(checkpoint["model"]) + if return_postprocessor: + return model, PostProcess() + return model + + +def detr_resnet50_dc5(pretrained=False, num_classes=91, return_postprocessor=False): + """ + DETR-DC5 R50 with 6 encoder and 6 decoder layers. + + The last block of ResNet-50 has dilation to increase + output resolution. + Achieves 43.3/63.1 AP/AP50 on COCO val5k. + """ + model = _make_detr("resnet50", dilation=True, num_classes=num_classes) + if pretrained: + checkpoint = torch.hub.load_state_dict_from_url( + url="https://dl.fbaipublicfiles.com/detr/detr-r50-dc5-f0fb7ef5.pth", map_location="cpu", check_hash=True + ) + model.load_state_dict(checkpoint["model"]) + if return_postprocessor: + return model, PostProcess() + return model + + +def detr_resnet101(pretrained=False, num_classes=91, return_postprocessor=False): + """ + DETR-DC5 R101 with 6 encoder and 6 decoder layers. + + Achieves 43.5/63.8 AP/AP50 on COCO val5k. + """ + model = _make_detr("resnet101", dilation=False, num_classes=num_classes) + if pretrained: + checkpoint = torch.hub.load_state_dict_from_url( + url="https://dl.fbaipublicfiles.com/detr/detr-r101-2c7b67e5.pth", map_location="cpu", check_hash=True + ) + model.load_state_dict(checkpoint["model"]) + if return_postprocessor: + return model, PostProcess() + return model + + +def detr_resnet101_dc5(pretrained=False, num_classes=91, return_postprocessor=False): + """ + DETR-DC5 R101 with 6 encoder and 6 decoder layers. + + The last block of ResNet-101 has dilation to increase + output resolution. + Achieves 44.9/64.7 AP/AP50 on COCO val5k. + """ + model = _make_detr("resnet101", dilation=True, num_classes=num_classes) + if pretrained: + checkpoint = torch.hub.load_state_dict_from_url( + url="https://dl.fbaipublicfiles.com/detr/detr-r101-dc5-a2e86def.pth", map_location="cpu", check_hash=True + ) + model.load_state_dict(checkpoint["model"]) + if return_postprocessor: + return model, PostProcess() + return model + + +def detr_resnet50_panoptic( + pretrained=False, num_classes=250, threshold=0.85, return_postprocessor=False +): + """ + DETR R50 with 6 encoder and 6 decoder layers. + Achieves 43.4 PQ on COCO val5k. + + threshold is the minimum confidence required for keeping segments in the prediction + """ + model = _make_detr("resnet50", dilation=False, num_classes=num_classes, mask=True) + is_thing_map = {i: i <= 90 for i in range(250)} + if pretrained: + checkpoint = torch.hub.load_state_dict_from_url( + url="https://dl.fbaipublicfiles.com/detr/detr-r50-panoptic-00ce5173.pth", + map_location="cpu", + check_hash=True, + ) + model.load_state_dict(checkpoint["model"]) + if return_postprocessor: + return model, PostProcessPanoptic(is_thing_map, threshold=threshold) + return model + + +def detr_resnet50_dc5_panoptic( + pretrained=False, num_classes=250, threshold=0.85, return_postprocessor=False +): + """ + DETR-DC5 R50 with 6 encoder and 6 decoder layers. + + The last block of ResNet-50 has dilation to increase + output resolution. + Achieves 44.6 on COCO val5k. + + threshold is the minimum confidence required for keeping segments in the prediction + """ + model = _make_detr("resnet50", dilation=True, num_classes=num_classes, mask=True) + is_thing_map = {i: i <= 90 for i in range(250)} + if pretrained: + checkpoint = torch.hub.load_state_dict_from_url( + url="https://dl.fbaipublicfiles.com/detr/detr-r50-dc5-panoptic-da08f1b1.pth", + map_location="cpu", + check_hash=True, + ) + model.load_state_dict(checkpoint["model"]) + if return_postprocessor: + return model, PostProcessPanoptic(is_thing_map, threshold=threshold) + return model + + +def detr_resnet101_panoptic( + pretrained=False, num_classes=250, threshold=0.85, return_postprocessor=False +): + """ + DETR-DC5 R101 with 6 encoder and 6 decoder layers. + + Achieves 45.1 PQ on COCO val5k. + + threshold is the minimum confidence required for keeping segments in the prediction + """ + model = _make_detr("resnet101", dilation=False, num_classes=num_classes, mask=True) + is_thing_map = {i: i <= 90 for i in range(250)} + if pretrained: + checkpoint = torch.hub.load_state_dict_from_url( + url="https://dl.fbaipublicfiles.com/detr/detr-r101-panoptic-40021d53.pth", + map_location="cpu", + check_hash=True, + ) + model.load_state_dict(checkpoint["model"]) + if return_postprocessor: + return model, PostProcessPanoptic(is_thing_map, threshold=threshold) + return model diff --git a/PyTorch/contrib/cv/detection/DETR/run_with_submitit.py b/PyTorch/contrib/cv/detection/DETR/run_with_submitit.py new file mode 100644 index 0000000000..b6780def01 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/run_with_submitit.py @@ -0,0 +1,111 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +A script to run multinode training with submitit. +""" +import argparse +import os +import uuid +from pathlib import Path + +import main as detection +import submitit + + +def parse_args(): + detection_parser = detection.get_args_parser() + parser = argparse.ArgumentParser("Submitit for detection", parents=[detection_parser]) + parser.add_argument("--ngpus", default=8, type=int, help="Number of gpus to request on each node") + parser.add_argument("--nodes", default=4, type=int, help="Number of nodes to request") + parser.add_argument("--timeout", default=60, type=int, help="Duration of the job") + parser.add_argument("--job_dir", default="", type=str, help="Job dir. Leave empty for automatic.") + return parser.parse_args() + + +def get_shared_folder() -> Path: + user = os.getenv("USER") + if Path("/checkpoint/").is_dir(): + p = Path(f"/checkpoint/{user}/experiments") + p.mkdir(exist_ok=True) + return p + raise RuntimeError("No shared folder available") + + +def get_init_file(): + # Init file must not exist, but it's parent dir must exist. + os.makedirs(str(get_shared_folder()), exist_ok=True) + init_file = get_shared_folder() / f"{uuid.uuid4().hex}_init" + if init_file.exists(): + os.remove(str(init_file)) + return init_file + + +class Trainer(object): + def __init__(self, args): + self.args = args + + def __call__(self): + import main as detection + + self._setup_gpu_args() + detection.main(self.args) + + def checkpoint(self): + import os + import submitit + from pathlib import Path + + self.args.dist_url = get_init_file().as_uri() + checkpoint_file = os.path.join(self.args.output_dir, "checkpoint.pth") + if os.path.exists(checkpoint_file): + self.args.resume = checkpoint_file + print("Requeuing ", self.args) + empty_trainer = type(self)(self.args) + return submitit.helpers.DelayedSubmission(empty_trainer) + + def _setup_gpu_args(self): + import submitit + from pathlib import Path + + job_env = submitit.JobEnvironment() + self.args.output_dir = Path(str(self.args.output_dir).replace("%j", str(job_env.job_id))) + self.args.gpu = job_env.local_rank + self.args.rank = job_env.global_rank + self.args.world_size = job_env.num_tasks + print(f"Process group: {job_env.num_tasks} tasks, rank: {job_env.global_rank}") + + +def main(): + args = parse_args() + if args.job_dir == "": + args.job_dir = get_shared_folder() / "%j" + + # Note that the folder will depend on the job_id, to easily track experiments + executor = submitit.AutoExecutor(folder=args.job_dir, slurm_max_num_timeout=30) + + # cluster setup is defined by environment variables + num_gpus_per_node = args.ngpus + nodes = args.nodes + timeout_min = args.timeout + + executor.update_parameters( + mem_gb=40 * num_gpus_per_node, + gpus_per_node=num_gpus_per_node, + tasks_per_node=num_gpus_per_node, # one task per GPU + cpus_per_task=10, + nodes=nodes, + timeout_min=timeout_min, # max is 60 * 72 + ) + + executor.update_parameters(name="detr") + + args.dist_url = get_init_file().as_uri() + args.output_dir = args.job_dir + + trainer = Trainer(args) + job = executor.submit(trainer) + + print("Submitted job_id:", job.job_id) + + +if __name__ == "__main__": + main() diff --git a/PyTorch/contrib/cv/detection/DETR/validate.py b/PyTorch/contrib/cv/detection/DETR/validate.py new file mode 100644 index 0000000000..a64e1b2400 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/validate.py @@ -0,0 +1,261 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import argparse +import datetime +import os +import random +import time +from pathlib import Path +import numpy as np +import torch +from torch.utils.data import DataLoader, DistributedSampler +from datasets.coco_eval import CocoEvaluator +import util.misc as utils +from datasets import build_dataset, get_coco_api_from_dataset +from models import build_model +from util import box_ops + +def get_args_parser(): + parser = argparse.ArgumentParser('Set transformer detector', add_help=False) + parser.add_argument('--lr', default=1e-4, type=float) + parser.add_argument('--lr_backbone', default=1e-5, type=float) + parser.add_argument('--batch_size', default=1, type=int) + parser.add_argument('--weight_decay', default=1e-4, type=float) + parser.add_argument('--epochs', default=300, type=int) + parser.add_argument('--lr_drop', default=200, type=int) + parser.add_argument('--clip_max_norm', default=0.1, type=float, + help='gradient clipping max norm') + + # Model parameters + parser.add_argument('--frozen_weights', type=str, default=None, + help="Path to the pretrained model. If set, only the mask head will be trained") + # * Backbone + parser.add_argument('--backbone', default='resnet50', type=str, + help="Name of the convolutional backbone to use") + parser.add_argument('--dilation', action='store_true', + help="If true, we replace stride with dilation in the last convolutional block (DC5)") + parser.add_argument('--position_embedding', default='sine', type=str, choices=('sine', 'learned'), + help="Type of positional embedding to use on top of the image features") + + # * Transformer + parser.add_argument('--enc_layers', default=6, type=int, + help="Number of encoding layers in the transformer") + parser.add_argument('--dec_layers', default=6, type=int, + help="Number of decoding layers in the transformer") + parser.add_argument('--dim_feedforward', default=2048, type=int, + help="Intermediate size of the feedforward layers in the transformer blocks") + parser.add_argument('--hidden_dim', default=256, type=int, + help="Size of the embeddings (dimension of the transformer)") + parser.add_argument('--dropout', default=0.1, type=float, + help="Dropout applied in the transformer") + parser.add_argument('--nheads', default=8, type=int, + help="Number of attention heads inside the transformer's attentions") + parser.add_argument('--num_queries', default=100, type=int, + help="Number of query slots") + parser.add_argument('--pre_norm', action='store_true') + + # * Segmentation + parser.add_argument('--masks', action='store_true', + help="Train segmentation head if the flag is provided") + + # Loss + parser.add_argument('--no_aux_loss', dest='aux_loss', action='store_false', + help="Disables auxiliary decoding losses (loss at each layer)") + # * Matcher + parser.add_argument('--set_cost_class', default=1, type=float, + help="Class coefficient in the matching cost") + parser.add_argument('--set_cost_bbox', default=5, type=float, + help="L1 box coefficient in the matching cost") + parser.add_argument('--set_cost_giou', default=2, type=float, + help="giou box coefficient in the matching cost") + # * Loss coefficients + parser.add_argument('--mask_loss_coef', default=1, type=float) + parser.add_argument('--dice_loss_coef', default=1, type=float) + parser.add_argument('--bbox_loss_coef', default=5, type=float) + parser.add_argument('--giou_loss_coef', default=2, type=float) + parser.add_argument('--eos_coef', default=0.1, type=float, + help="Relative classification weight of the no-object class") + + # dataset parameters + parser.add_argument('--dataset_file', default='coco') + parser.add_argument('--coco_path', type=str,default='/home/xu/SJH/datasets/coco') + parser.add_argument('--coco_panoptic_path', type=str) + parser.add_argument('--remove_difficult', action='store_true') + + parser.add_argument('--output_dir', default='output', + help='path where to save, empty for no saving') + parser.add_argument('--device', default='cuda', + help='device to use for training / testing') + parser.add_argument('--seed', default=42, type=int) + parser.add_argument('--resume', default='model_file/detr.pth', help='resume from checkpoint') + parser.add_argument('--start_epoch', default=0, type=int, metavar='N', + help='start epoch') + parser.add_argument('--eval', action='store_true',default=True,) + parser.add_argument('--num_workers', default=2, type=int) + + # distributed training parameters + parser.add_argument('--world_size', default=1, type=int, + help='number of distributed processes') + parser.add_argument('--dist_url', default='env://', help='url used to set up distributed training') + return parser + + +@torch.no_grad() +def evaluate(model, criterion, postprocessors, data_loader, base_ds, device, output_dir): + model.eval() + criterion.eval() + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter('class_error', utils.SmoothedValue(window_size=1, fmt='{value:.2f}')) + header = 'Test:' + iou_types = tuple(k for k in ('segm', 'bbox') if k in postprocessors.keys()) + coco_evaluator = CocoEvaluator(base_ds, iou_types) + + # ort_session = onnxruntime.InferenceSession('model_file/detr_640.onnx') + for file,(samples, targets) in zip(os.listdir('/home/xu/xiaoxiong/effdet/coco_data/val2017'),metric_logger.log_every(data_loader, 10, header)): + samples = samples.to(device) + targets = [{k: v.to(device) for k, v in t.items()} for t in targets] + print(samples.tensors.shape) + print(os.path.join('/home/xu/xiaoxiong/DETR/detr_bin','{}.bin'.format(file.split('.')[0]))) + samples.tensors.cpu().numpy().tofile(os.path.join('/home/xu/xiaoxiong/DETR/detr_bin','{}.bin'.format(file.split('.')[0]))) + + # ort_inputs = {ort_session.get_inputs()[0].name:samples.tensors.cpu().numpy()} + # # print('inputs',ort_inputs) + # ort_outs = ort_session.run(None, ort_inputs) + # out={'pred_logits':torch.from_numpy(ort_outs[0]).cuda(), + # 'pred_boxes':torch.from_numpy(ort_outs[1]).cuda()} + # outputs=out + # loss_dict=criterion(out,targets) + + outputs = model(samples) + loss_dict = criterion(outputs, targets) + weight_dict = criterion.weight_dict + + # reduce losses over all GPUs for logging purposes + loss_dict_reduced = utils.reduce_dict(loss_dict) + loss_dict_reduced_scaled = {k: v * weight_dict[k] + for k, v in loss_dict_reduced.items() if k in weight_dict} + loss_dict_reduced_unscaled = {f'{k}_unscaled': v + for k, v in loss_dict_reduced.items()} + metric_logger.update(loss=sum(loss_dict_reduced_scaled.values()), + **loss_dict_reduced_scaled, + **loss_dict_reduced_unscaled) + metric_logger.update(class_error=loss_dict_reduced['class_error']) + orig_target_sizes = torch.stack([t["orig_size"] for t in targets], dim=0) + + results = postprocessors['bbox'](outputs, orig_target_sizes) + + # print(len(results[0]['scores']),results[0]['scores']) + # print(results[0]['boxes']) + # print(results[0]['labels']) + # print(postprocessors.keys()) + + res = {target['image_id'].item(): output for target, output in zip(targets, results)} + + # if 2592 in res.keys(): + # print(orig_target_sizes) + # print(res[2592]) + # for i,value in enumerate(res[2592]['scores']): + # if value>0.5: + # print(i,value) + # print(res[2592]['boxes'][i]) + # p + coco_evaluator.update(res) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + coco_evaluator.synchronize_between_processes() + + # accumulate predictions from all images + coco_evaluator.accumulate() + coco_evaluator.summarize() + stats = {k: meter.global_avg for k, meter in metric_logger.meters.items()} + stats['coco_eval_bbox'] = coco_evaluator.coco_eval['bbox'].stats.tolist() + + return stats, coco_evaluator + +def main(args): + utils.init_distributed_mode(args) + print("git:\n {}\n".format(utils.get_sha())) + + if args.frozen_weights is not None: + assert args.masks, "Frozen training is meant for segmentation only" + print(args) + + device = torch.device(args.device) + + # fix the seed for reproducibility + seed = args.seed + utils.get_rank() + torch.manual_seed(seed) + np.random.seed(seed) + random.seed(seed) + + model, criterion, postprocessors = build_model(args) + model.to(device) + + model_without_ddp = model + print(args.distributed) + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) + print('number of params:', n_parameters) + + param_dicts = [ + {"params": [p for n, p in model_without_ddp.named_parameters() if "backbone" not in n and p.requires_grad]}, + { + "params": [p for n, p in model_without_ddp.named_parameters() if "backbone" in n and p.requires_grad], + "lr": args.lr_backbone, + }, + ] + optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, + weight_decay=args.weight_decay) + lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.lr_drop) + + + dataset_val = build_dataset(image_set='val', args=args) + + if args.distributed: + sampler_val = DistributedSampler(dataset_val, shuffle=False) + else: + sampler_val = torch.utils.data.SequentialSampler(dataset_val) + + data_loader_val = DataLoader(dataset_val, args.batch_size, sampler=sampler_val, + drop_last=False, collate_fn=utils.collate_fn, num_workers=args.num_workers) + + + base_ds = get_coco_api_from_dataset(dataset_val) + + if args.frozen_weights is not None: + checkpoint = torch.load(args.frozen_weights, map_location='cpu') + model_without_ddp.detr.load_state_dict(checkpoint['model']) + + output_dir = Path(args.output_dir) + if args.resume: + checkpoint = torch.load(args.resume, map_location='cpu') + model_without_ddp.load_state_dict(checkpoint['model']) + if not args.eval and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: + optimizer.load_state_dict(checkpoint['optimizer']) + lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) + args.start_epoch = checkpoint['epoch'] + 1 + + start_time = time.time() + if args.eval: + print('start validate') + test_stats, coco_evaluator = evaluate(model, criterion, postprocessors, + data_loader_val, base_ds, device, args.output_dir) + if args.output_dir: + utils.save_on_master(coco_evaluator.coco_eval["bbox"].eval, output_dir / "eval.pth") + return + + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print('Training time {}'.format(total_time_str)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser('DETR training and evaluation script', parents=[get_args_parser()]) + args = parser.parse_args() + if args.output_dir: + Path(args.output_dir).mkdir(parents=True, exist_ok=True) + main(args) -- Gitee From de4a79063965b23b5fb772c4074c959e81e2a4f6 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:07:53 +0000 Subject: [PATCH 48/59] add PyTorch/contrib/cv/detection/DETR/LICENSE. --- PyTorch/contrib/cv/detection/DETR/LICENSE | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/LICENSE diff --git a/PyTorch/contrib/cv/detection/DETR/LICENSE b/PyTorch/contrib/cv/detection/DETR/LICENSE new file mode 100644 index 0000000000..cac29a5170 --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/LICENSE @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright (c) 2017, +All rights reserved. +Copyright 2021 Huawei Technologies Co., Ltd + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file -- Gitee From dc6568b79948c3c0e69b52bad1f1d8d327ed9564 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:08:16 +0000 Subject: [PATCH 49/59] update PyTorch/contrib/cv/detection/DETR/LICENSE. --- PyTorch/contrib/cv/detection/DETR/LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/cv/detection/DETR/LICENSE b/PyTorch/contrib/cv/detection/DETR/LICENSE index cac29a5170..5170571580 100644 --- a/PyTorch/contrib/cv/detection/DETR/LICENSE +++ b/PyTorch/contrib/cv/detection/DETR/LICENSE @@ -2,7 +2,7 @@ BSD 3-Clause License Copyright (c) 2017, All rights reserved. -Copyright 2021 Huawei Technologies Co., Ltd +Copyright 2022 Huawei Technologies Co., Ltd Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -- Gitee From bd90011e704ecb516963e2a7cfb3fa7b166c7675 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:08:53 +0000 Subject: [PATCH 50/59] my first commit --- PyTorch/contrib/cv/detection/DETR/requirements.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 PyTorch/contrib/cv/detection/DETR/requirements.txt diff --git a/PyTorch/contrib/cv/detection/DETR/requirements.txt b/PyTorch/contrib/cv/detection/DETR/requirements.txt new file mode 100644 index 0000000000..bb8f7823ba --- /dev/null +++ b/PyTorch/contrib/cv/detection/DETR/requirements.txt @@ -0,0 +1,9 @@ +cython +git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI&egg=pycocotools +submitit +torch>=1.5.0 +torchvision>=0.6.0 +git+https://github.com/cocodataset/panopticapi.git#egg=panopticapi +scipy +onnx +onnxruntime -- Gitee From 1f8e7a8dda74200b41255eb487c1cc5e452942a8 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:17:54 +0000 Subject: [PATCH 51/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/cv/detection/DETR/d2/README.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../contrib/cv/detection/DETR/d2/README.md | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/README.md diff --git a/PyTorch/contrib/cv/detection/DETR/d2/README.md b/PyTorch/contrib/cv/detection/DETR/d2/README.md deleted file mode 100644 index 7f1d753190..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/README.md +++ /dev/null @@ -1,39 +0,0 @@ -Detectron2 wrapper for DETR -======= - -We provide a Detectron2 wrapper for DETR, thus providing a way to better integrate it in the existing detection ecosystem. It can be used for example to easily leverage datasets or backbones provided in Detectron2. - -This wrapper currently supports only box detection, and is intended to be as close as possible to the original implementation, and we checked that it indeed match the results. Some notable facts and caveats: -- The data augmentation matches DETR's original data augmentation. This required patching the RandomCrop augmentation from Detectron2, so you'll need a version from the master branch from June 24th 2020 or more recent. -- To match DETR's original backbone initialization, we use the weights of a ResNet50 trained on imagenet using torchvision. This network uses a different pixel mean and std than most of the backbones available in Detectron2 by default, so extra care must be taken when switching to another one. Note that no other torchvision models are available in Detectron2 as of now, though it may change in the future. -- The gradient clipping mode is "full_model", which is not the default in Detectron2. - -# Usage - -To install Detectron2, please follow the [official installation instructions](https://github.com/facebookresearch/detectron2/blob/master/INSTALL.md). - -## Evaluating a model - -For convenience, we provide a conversion script to convert models trained by the main DETR training loop into the format of this wrapper. To download and convert the main Resnet50 model, simply do: - -``` -python converter.py --source_model https://dl.fbaipublicfiles.com/detr/detr-r50-e632da11.pth --output_model converted_model.pth -``` - -You can then evaluate it using: -``` -python train_net.py --eval-only --config configs/detr_256_6_6_torchvision.yaml MODEL.WEIGHTS "converted_model.pth" -``` - - -## Training - -To train DETR on a single node with 8 gpus, simply use: -``` -python train_net.py --config configs/detr_256_6_6_torchvision.yaml --num-gpus 8 -``` - -To fine-tune DETR for instance segmentation on a single node with 8 gpus, simply use: -``` -python train_net.py --config configs/detr_segm_256_6_6_torchvision.yaml --num-gpus 8 MODEL.DETR.FROZEN_WEIGHTS -``` -- Gitee From d3a1edad99d59335dfc62fb04eed79c99c858c4c Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:20:03 +0000 Subject: [PATCH 52/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/cv/detection/DETR/d2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../d2/configs/detr_256_6_6_torchvision.yaml | 45 --- .../detr_segm_256_6_6_torchvision.yaml | 46 --- .../contrib/cv/detection/DETR/d2/converter.py | 69 ----- .../cv/detection/DETR/d2/detr/__init__.py | 4 - .../cv/detection/DETR/d2/detr/config.py | 34 --- .../detection/DETR/d2/detr/dataset_mapper.py | 122 -------- .../contrib/cv/detection/DETR/d2/detr/detr.py | 261 ------------------ .../contrib/cv/detection/DETR/d2/train_net.py | 145 ---------- 8 files changed, 726 deletions(-) delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/configs/detr_256_6_6_torchvision.yaml delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/configs/detr_segm_256_6_6_torchvision.yaml delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/converter.py delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/detr/__init__.py delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/detr/config.py delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/detr/dataset_mapper.py delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/detr/detr.py delete mode 100644 PyTorch/contrib/cv/detection/DETR/d2/train_net.py diff --git a/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_256_6_6_torchvision.yaml b/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_256_6_6_torchvision.yaml deleted file mode 100644 index 25d641845f..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_256_6_6_torchvision.yaml +++ /dev/null @@ -1,45 +0,0 @@ -MODEL: - META_ARCHITECTURE: "Detr" - WEIGHTS: "detectron2://ImageNetPretrained/torchvision/R-50.pkl" - PIXEL_MEAN: [123.675, 116.280, 103.530] - PIXEL_STD: [58.395, 57.120, 57.375] - MASK_ON: False - RESNETS: - DEPTH: 50 - STRIDE_IN_1X1: False - OUT_FEATURES: ["res2", "res3", "res4", "res5"] - DETR: - GIOU_WEIGHT: 2.0 - L1_WEIGHT: 5.0 - NUM_OBJECT_QUERIES: 100 -DATASETS: - TRAIN: ("coco_2017_train",) - TEST: ("coco_2017_val",) -SOLVER: - IMS_PER_BATCH: 64 - BASE_LR: 0.0001 - STEPS: (369600,) - MAX_ITER: 554400 - WARMUP_FACTOR: 1.0 - WARMUP_ITERS: 10 - WEIGHT_DECAY: 0.0001 - OPTIMIZER: "ADAMW" - BACKBONE_MULTIPLIER: 0.1 - CLIP_GRADIENTS: - ENABLED: True - CLIP_TYPE: "full_model" - CLIP_VALUE: 0.01 - NORM_TYPE: 2.0 -INPUT: - MIN_SIZE_TRAIN: (480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800) - CROP: - ENABLED: True - TYPE: "absolute_range" - SIZE: (384, 600) - FORMAT: "RGB" -TEST: - EVAL_PERIOD: 4000 -DATALOADER: - FILTER_EMPTY_ANNOTATIONS: False - NUM_WORKERS: 4 -VERSION: 2 diff --git a/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_segm_256_6_6_torchvision.yaml b/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_segm_256_6_6_torchvision.yaml deleted file mode 100644 index ade490e6d5..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/configs/detr_segm_256_6_6_torchvision.yaml +++ /dev/null @@ -1,46 +0,0 @@ -MODEL: - META_ARCHITECTURE: "Detr" -# WEIGHTS: "detectron2://ImageNetPretrained/torchvision/R-50.pkl" - PIXEL_MEAN: [123.675, 116.280, 103.530] - PIXEL_STD: [58.395, 57.120, 57.375] - MASK_ON: True - RESNETS: - DEPTH: 50 - STRIDE_IN_1X1: False - OUT_FEATURES: ["res2", "res3", "res4", "res5"] - DETR: - GIOU_WEIGHT: 2.0 - L1_WEIGHT: 5.0 - NUM_OBJECT_QUERIES: 100 - FROZEN_WEIGHTS: '' -DATASETS: - TRAIN: ("coco_2017_train",) - TEST: ("coco_2017_val",) -SOLVER: - IMS_PER_BATCH: 64 - BASE_LR: 0.0001 - STEPS: (55440,) - MAX_ITER: 92400 - WARMUP_FACTOR: 1.0 - WARMUP_ITERS: 10 - WEIGHT_DECAY: 0.0001 - OPTIMIZER: "ADAMW" - BACKBONE_MULTIPLIER: 0.1 - CLIP_GRADIENTS: - ENABLED: True - CLIP_TYPE: "full_model" - CLIP_VALUE: 0.01 - NORM_TYPE: 2.0 -INPUT: - MIN_SIZE_TRAIN: (480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800) - CROP: - ENABLED: True - TYPE: "absolute_range" - SIZE: (384, 600) - FORMAT: "RGB" -TEST: - EVAL_PERIOD: 4000 -DATALOADER: - FILTER_EMPTY_ANNOTATIONS: False - NUM_WORKERS: 4 -VERSION: 2 diff --git a/PyTorch/contrib/cv/detection/DETR/d2/converter.py b/PyTorch/contrib/cv/detection/DETR/d2/converter.py deleted file mode 100644 index 42882ce86e..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/converter.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -Helper script to convert models trained with the main version of DETR to be used with the Detectron2 version. -""" -import json -import argparse - -import numpy as np -import torch - - -def parse_args(): - parser = argparse.ArgumentParser("D2 model converter") - - parser.add_argument("--source_model", default="../detr.pth", type=str, help="Path or url to the DETR model to convert") - parser.add_argument("--output_model", default="../detr_.pth", type=str, help="Path where to save the converted model") - return parser.parse_args() - - -def main(): - args = parse_args() - - # D2 expects contiguous classes, so we need to remap the 92 classes from DETR - # fmt: off - coco_idx = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, - 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, - 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91] - # fmt: on - - coco_idx = np.array(coco_idx) - - if args.source_model.startswith("https"): - checkpoint = torch.hub.load_state_dict_from_url(args.source_model, map_location="cpu", check_hash=True) - else: - checkpoint = torch.load(args.source_model, map_location="cpu") - model_to_convert = checkpoint["model"] - - model_converted = {} - for k in model_to_convert.keys(): - old_k = k - if "backbone" in k: - k = k.replace("backbone.0.body.", "") - if "layer" not in k: - k = "stem." + k - for t in [1, 2, 3, 4]: - k = k.replace(f"layer{t}", f"res{t + 1}") - for t in [1, 2, 3]: - k = k.replace(f"bn{t}", f"conv{t}.norm") - k = k.replace("downsample.0", "shortcut") - k = k.replace("downsample.1", "shortcut.norm") - k = "backbone.0.backbone." + k - k = "detr." + k - print(old_k, "->", k) - if "class_embed" in old_k: - v = model_to_convert[old_k].detach() - if v.shape[0] == 92: - shape_old = v.shape - model_converted[k] = v[coco_idx] - print("Head conversion: changing shape from {} to {}".format(shape_old, model_converted[k].shape)) - continue - model_converted[k] = model_to_convert[old_k].detach() - - model_to_save = {"model": model_converted} - torch.save(model_to_save, args.output_model) - - -if __name__ == "__main__": - main() diff --git a/PyTorch/contrib/cv/detection/DETR/d2/detr/__init__.py b/PyTorch/contrib/cv/detection/DETR/d2/detr/__init__.py deleted file mode 100644 index a618f82887..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/detr/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from .config import add_detr_config -from .detr import Detr -from .dataset_mapper import DetrDatasetMapper diff --git a/PyTorch/contrib/cv/detection/DETR/d2/detr/config.py b/PyTorch/contrib/cv/detection/DETR/d2/detr/config.py deleted file mode 100644 index 9ea267dd6f..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/detr/config.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from detectron2.config import CfgNode as CN - - -def add_detr_config(cfg): - """ - Add config for DETR. - """ - cfg.MODEL.DETR = CN() - cfg.MODEL.DETR.NUM_CLASSES = 80 - - # For Segmentation - cfg.MODEL.DETR.FROZEN_WEIGHTS = '' - - # LOSS - cfg.MODEL.DETR.GIOU_WEIGHT = 2.0 - cfg.MODEL.DETR.L1_WEIGHT = 5.0 - cfg.MODEL.DETR.DEEP_SUPERVISION = True - cfg.MODEL.DETR.NO_OBJECT_WEIGHT = 0.1 - - # TRANSFORMER - cfg.MODEL.DETR.NHEADS = 8 - cfg.MODEL.DETR.DROPOUT = 0.1 - cfg.MODEL.DETR.DIM_FEEDFORWARD = 2048 - cfg.MODEL.DETR.ENC_LAYERS = 6 - cfg.MODEL.DETR.DEC_LAYERS = 6 - cfg.MODEL.DETR.PRE_NORM = False - - cfg.MODEL.DETR.HIDDEN_DIM = 256 - cfg.MODEL.DETR.NUM_OBJECT_QUERIES = 100 - - cfg.SOLVER.OPTIMIZER = "ADAMW" - cfg.SOLVER.BACKBONE_MULTIPLIER = 0.1 diff --git a/PyTorch/contrib/cv/detection/DETR/d2/detr/dataset_mapper.py b/PyTorch/contrib/cv/detection/DETR/d2/detr/dataset_mapper.py deleted file mode 100644 index f428a4939b..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/detr/dataset_mapper.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -import copy -import logging - -import numpy as np -import torch - -from detectron2.data import detection_utils as utils -from detectron2.data import transforms as T -from detectron2.data.transforms import TransformGen - -__all__ = ["DetrDatasetMapper"] - - -def build_transform_gen(cfg, is_train): - """ - Create a list of :class:`TransformGen` from config. - Returns: - list[TransformGen] - """ - if is_train: - min_size = cfg.INPUT.MIN_SIZE_TRAIN - max_size = cfg.INPUT.MAX_SIZE_TRAIN - sample_style = cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING - else: - min_size = cfg.INPUT.MIN_SIZE_TEST - max_size = cfg.INPUT.MAX_SIZE_TEST - sample_style = "choice" - if sample_style == "range": - assert len(min_size) == 2, "more than 2 ({}) min_size(s) are provided for ranges".format(len(min_size)) - - logger = logging.getLogger(__name__) - tfm_gens = [] - if is_train: - tfm_gens.append(T.RandomFlip()) - tfm_gens.append(T.ResizeShortestEdge(min_size, max_size, sample_style)) - if is_train: - logger.info("TransformGens used in training: " + str(tfm_gens)) - return tfm_gens - - -class DetrDatasetMapper: - """ - A callable which takes a dataset dict in Detectron2 Dataset format, - and map it into a format used by DETR. - - The callable currently does the following: - - 1. Read the image from "file_name" - 2. Applies geometric transforms to the image and annotation - 3. Find and applies suitable cropping to the image and annotation - 4. Prepare image and annotation to Tensors - """ - - def __init__(self, cfg, is_train=True): - if cfg.INPUT.CROP.ENABLED and is_train: - self.crop_gen = [ - T.ResizeShortestEdge([400, 500, 600], sample_style="choice"), - T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE), - ] - else: - self.crop_gen = None - - self.mask_on = cfg.MODEL.MASK_ON - self.tfm_gens = build_transform_gen(cfg, is_train) - logging.getLogger(__name__).info( - "Full TransformGens used in training: {}, crop: {}".format(str(self.tfm_gens), str(self.crop_gen)) - ) - - self.img_format = cfg.INPUT.FORMAT - self.is_train = is_train - - def __call__(self, dataset_dict): - """ - Args: - dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. - - Returns: - dict: a format that builtin models in detectron2 accept - """ - dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below - image = utils.read_image(dataset_dict["file_name"], format=self.img_format) - utils.check_image_size(dataset_dict, image) - - if self.crop_gen is None: - image, transforms = T.apply_transform_gens(self.tfm_gens, image) - else: - if np.random.rand() > 0.5: - image, transforms = T.apply_transform_gens(self.tfm_gens, image) - else: - image, transforms = T.apply_transform_gens( - self.tfm_gens[:-1] + self.crop_gen + self.tfm_gens[-1:], image - ) - - image_shape = image.shape[:2] # h, w - - # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, - # but not efficient on large generic data structures due to the use of pickle & mp.Queue. - # Therefore it's important to use torch.Tensor. - dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) - - if not self.is_train: - # USER: Modify this if you want to keep them for some reason. - dataset_dict.pop("annotations", None) - return dataset_dict - - if "annotations" in dataset_dict: - # USER: Modify this if you want to keep them for some reason. - for anno in dataset_dict["annotations"]: - if not self.mask_on: - anno.pop("segmentation", None) - anno.pop("keypoints", None) - - # USER: Implement additional transformations if you have other types of data - annos = [ - utils.transform_instance_annotations(obj, transforms, image_shape) - for obj in dataset_dict.pop("annotations") - if obj.get("iscrowd", 0) == 0 - ] - instances = utils.annotations_to_instances(annos, image_shape) - dataset_dict["instances"] = utils.filter_empty_instances(instances) - return dataset_dict diff --git a/PyTorch/contrib/cv/detection/DETR/d2/detr/detr.py b/PyTorch/contrib/cv/detection/DETR/d2/detr/detr.py deleted file mode 100644 index 95f89dff3e..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/detr/detr.py +++ /dev/null @@ -1,261 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -import logging -import math -from typing import List - -import numpy as np -import torch -import torch.distributed as dist -import torch.nn.functional as F -from scipy.optimize import linear_sum_assignment -from torch import nn - -from detectron2.layers import ShapeSpec -from detectron2.modeling import META_ARCH_REGISTRY, build_backbone, detector_postprocess -from detectron2.structures import Boxes, ImageList, Instances, BitMasks, PolygonMasks -from detectron2.utils.logger import log_first_n -from fvcore.nn import giou_loss, smooth_l1_loss -from models.backbone import Joiner -from models.detr import DETR, SetCriterion -from models.matcher import HungarianMatcher -from models.position_encoding import PositionEmbeddingSine -from models.transformer import Transformer -from models.segmentation import DETRsegm, PostProcessPanoptic, PostProcessSegm -from util.box_ops import box_cxcywh_to_xyxy, box_xyxy_to_cxcywh -from util.misc import NestedTensor -from datasets.coco import convert_coco_poly_to_mask - -__all__ = ["Detr"] - - -class MaskedBackbone(nn.Module): - """ This is a thin wrapper around D2's backbone to provide padding masking""" - - def __init__(self, cfg): - super().__init__() - self.backbone = build_backbone(cfg) - backbone_shape = self.backbone.output_shape() - self.feature_strides = [backbone_shape[f].stride for f in backbone_shape.keys()] - self.num_channels = backbone_shape[list(backbone_shape.keys())[-1]].channels - - def forward(self, images): - features = self.backbone(images.tensor) - masks = self.mask_out_padding( - [features_per_level.shape for features_per_level in features.values()], - images.image_sizes, - images.tensor.device, - ) - assert len(features) == len(masks) - for i, k in enumerate(features.keys()): - features[k] = NestedTensor(features[k], masks[i]) - return features - - def mask_out_padding(self, feature_shapes, image_sizes, device): - masks = [] - assert len(feature_shapes) == len(self.feature_strides) - for idx, shape in enumerate(feature_shapes): - N, _, H, W = shape - masks_per_feature_level = torch.ones((N, H, W), dtype=torch.bool, device=device) - for img_idx, (h, w) in enumerate(image_sizes): - masks_per_feature_level[ - img_idx, - : int(np.ceil(float(h) / self.feature_strides[idx])), - : int(np.ceil(float(w) / self.feature_strides[idx])), - ] = 0 - masks.append(masks_per_feature_level) - return masks - - -@META_ARCH_REGISTRY.register() -class Detr(nn.Module): - """ - Implement Detr - """ - - def __init__(self, cfg): - super().__init__() - - self.device = torch.device(cfg.MODEL.DEVICE) - - self.num_classes = cfg.MODEL.DETR.NUM_CLASSES - self.mask_on = cfg.MODEL.MASK_ON - hidden_dim = cfg.MODEL.DETR.HIDDEN_DIM - num_queries = cfg.MODEL.DETR.NUM_OBJECT_QUERIES - # Transformer parameters: - nheads = cfg.MODEL.DETR.NHEADS - dropout = cfg.MODEL.DETR.DROPOUT - dim_feedforward = cfg.MODEL.DETR.DIM_FEEDFORWARD - enc_layers = cfg.MODEL.DETR.ENC_LAYERS - dec_layers = cfg.MODEL.DETR.DEC_LAYERS - pre_norm = cfg.MODEL.DETR.PRE_NORM - - # Loss parameters: - giou_weight = cfg.MODEL.DETR.GIOU_WEIGHT - l1_weight = cfg.MODEL.DETR.L1_WEIGHT - deep_supervision = cfg.MODEL.DETR.DEEP_SUPERVISION - no_object_weight = cfg.MODEL.DETR.NO_OBJECT_WEIGHT - - N_steps = hidden_dim // 2 - d2_backbone = MaskedBackbone(cfg) - backbone = Joiner(d2_backbone, PositionEmbeddingSine(N_steps, normalize=True)) - backbone.num_channels = d2_backbone.num_channels - - transformer = Transformer( - d_model=hidden_dim, - dropout=dropout, - nhead=nheads, - dim_feedforward=dim_feedforward, - num_encoder_layers=enc_layers, - num_decoder_layers=dec_layers, - normalize_before=pre_norm, - return_intermediate_dec=deep_supervision, - ) - - self.detr = DETR( - backbone, transformer, num_classes=self.num_classes, num_queries=num_queries, aux_loss=deep_supervision - ) - if self.mask_on: - frozen_weights = cfg.MODEL.DETR.FROZEN_WEIGHTS - if frozen_weights != '': - print("LOAD pre-trained weights") - weight = torch.load(frozen_weights, map_location=lambda storage, loc: storage)['model'] - new_weight = {} - for k, v in weight.items(): - if 'detr.' in k: - new_weight[k.replace('detr.', '')] = v - else: - print(f"Skipping loading weight {k} from frozen model") - del weight - self.detr.load_state_dict(new_weight) - del new_weight - self.detr = DETRsegm(self.detr, freeze_detr=(frozen_weights != '')) - self.seg_postprocess = PostProcessSegm - - self.detr.to(self.device) - - # building criterion - matcher = HungarianMatcher(cost_class=1, cost_bbox=l1_weight, cost_giou=giou_weight) - weight_dict = {"loss_ce": 1, "loss_bbox": l1_weight} - weight_dict["loss_giou"] = giou_weight - if deep_supervision: - aux_weight_dict = {} - for i in range(dec_layers - 1): - aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) - weight_dict.update(aux_weight_dict) - losses = ["labels", "boxes", "cardinality"] - if self.mask_on: - losses += ["masks"] - self.criterion = SetCriterion( - self.num_classes, matcher=matcher, weight_dict=weight_dict, eos_coef=no_object_weight, losses=losses, - ) - self.criterion.to(self.device) - - pixel_mean = torch.Tensor(cfg.MODEL.PIXEL_MEAN).to(self.device).view(3, 1, 1) - pixel_std = torch.Tensor(cfg.MODEL.PIXEL_STD).to(self.device).view(3, 1, 1) - self.normalizer = lambda x: (x - pixel_mean) / pixel_std - self.to(self.device) - - def forward(self, batched_inputs): - """ - Args: - batched_inputs: a list, batched outputs of :class:`DatasetMapper` . - Each item in the list contains the inputs for one image. - For now, each item in the list is a dict that contains: - - * image: Tensor, image in (C, H, W) format. - * instances: Instances - - Other information that's included in the original dicts, such as: - - * "height", "width" (int): the output resolution of the model, used in inference. - See :meth:`postprocess` for details. - Returns: - dict[str: Tensor]: - mapping from a named loss to a tensor storing the loss. Used during training only. - """ - images = self.preprocess_image(batched_inputs) - output = self.detr(images) - - if self.training: - gt_instances = [x["instances"].to(self.device) for x in batched_inputs] - - targets = self.prepare_targets(gt_instances) - loss_dict = self.criterion(output, targets) - weight_dict = self.criterion.weight_dict - for k in loss_dict.keys(): - if k in weight_dict: - loss_dict[k] *= weight_dict[k] - return loss_dict - else: - box_cls = output["pred_logits"] - box_pred = output["pred_boxes"] - mask_pred = output["pred_masks"] if self.mask_on else None - results = self.inference(box_cls, box_pred, mask_pred, images.image_sizes) - processed_results = [] - for results_per_image, input_per_image, image_size in zip(results, batched_inputs, images.image_sizes): - height = input_per_image.get("height", image_size[0]) - width = input_per_image.get("width", image_size[1]) - r = detector_postprocess(results_per_image, height, width) - processed_results.append({"instances": r}) - return processed_results - - def prepare_targets(self, targets): - new_targets = [] - for targets_per_image in targets: - h, w = targets_per_image.image_size - image_size_xyxy = torch.as_tensor([w, h, w, h], dtype=torch.float, device=self.device) - gt_classes = targets_per_image.gt_classes - gt_boxes = targets_per_image.gt_boxes.tensor / image_size_xyxy - gt_boxes = box_xyxy_to_cxcywh(gt_boxes) - new_targets.append({"labels": gt_classes, "boxes": gt_boxes}) - if self.mask_on and hasattr(targets_per_image, 'gt_masks'): - gt_masks = targets_per_image.gt_masks - gt_masks = convert_coco_poly_to_mask(gt_masks.polygons, h, w) - new_targets[-1].update({'masks': gt_masks}) - return new_targets - - def inference(self, box_cls, box_pred, mask_pred, image_sizes): - """ - Arguments: - box_cls (Tensor): tensor of shape (batch_size, num_queries, K). - The tensor predicts the classification probability for each query. - box_pred (Tensor): tensors of shape (batch_size, num_queries, 4). - The tensor predicts 4-vector (x,y,w,h) box - regression values for every queryx - image_sizes (List[torch.Size]): the input image sizes - - Returns: - results (List[Instances]): a list of #images elements. - """ - assert len(box_cls) == len(image_sizes) - results = [] - - # For each box we assign the best class or the second best if the best on is `no_object`. - scores, labels = F.softmax(box_cls, dim=-1)[:, :, :-1].max(-1) - - for i, (scores_per_image, labels_per_image, box_pred_per_image, image_size) in enumerate(zip( - scores, labels, box_pred, image_sizes - )): - result = Instances(image_size) - result.pred_boxes = Boxes(box_cxcywh_to_xyxy(box_pred_per_image)) - - result.pred_boxes.scale(scale_x=image_size[1], scale_y=image_size[0]) - if self.mask_on: - mask = F.interpolate(mask_pred[i].unsqueeze(0), size=image_size, mode='bilinear', align_corners=False) - mask = mask[0].sigmoid() > 0.5 - B, N, H, W = mask_pred.shape - mask = BitMasks(mask.cpu()).crop_and_resize(result.pred_boxes.tensor.cpu(), 32) - result.pred_masks = mask.unsqueeze(1).to(mask_pred[0].device) - - result.scores = scores_per_image - result.pred_classes = labels_per_image - results.append(result) - return results - - def preprocess_image(self, batched_inputs): - """ - Normalize, pad and batch the input images. - """ - images = [self.normalizer(x["image"].to(self.device)) for x in batched_inputs] - images = ImageList.from_tensors(images) - return images diff --git a/PyTorch/contrib/cv/detection/DETR/d2/train_net.py b/PyTorch/contrib/cv/detection/DETR/d2/train_net.py deleted file mode 100644 index 82f692922e..0000000000 --- a/PyTorch/contrib/cv/detection/DETR/d2/train_net.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -""" -DETR Training Script. - -This script is a simplified version of the training script in detectron2/tools. -""" -import os -import sys -import itertools - -# fmt: off -sys.path.insert(1, os.path.join(sys.path[0], '..')) -# fmt: on - -import time -from typing import Any, Dict, List, Set - -import torch - -import detectron2.utils.comm as comm -from d2.detr import DetrDatasetMapper, add_detr_config -from detectron2.checkpoint import DetectionCheckpointer -from detectron2.config import get_cfg -from detectron2.data import MetadataCatalog, build_detection_train_loader -from detectron2.engine import DefaultTrainer, default_argument_parser, default_setup, launch -from detectron2.evaluation import COCOEvaluator, verify_results - -from detectron2.solver.build import maybe_add_gradient_clipping - - -class Trainer(DefaultTrainer): - """ - Extension of the Trainer class adapted to DETR. - """ - - @classmethod - def build_evaluator(cls, cfg, dataset_name, output_folder=None): - """ - Create evaluator(s) for a given dataset. - This uses the special metadata "evaluator_type" associated with each builtin dataset. - For your own dataset, you can simply create an evaluator manually in your - script and do not have to worry about the hacky if-else logic here. - """ - if output_folder is None: - output_folder = os.path.join(cfg.OUTPUT_DIR, "inference") - return COCOEvaluator(dataset_name, cfg, True, output_folder) - - @classmethod - def build_train_loader(cls, cfg): - if "Detr" == cfg.MODEL.META_ARCHITECTURE: - mapper = DetrDatasetMapper(cfg, True) - else: - mapper = None - return build_detection_train_loader(cfg, mapper=mapper) - - @classmethod - def build_optimizer(cls, cfg, model): - params: List[Dict[str, Any]] = [] - memo: Set[torch.nn.parameter.Parameter] = set() - for key, value in model.named_parameters(recurse=True): - if not value.requires_grad: - continue - # Avoid duplicating parameters - if value in memo: - continue - memo.add(value) - lr = cfg.SOLVER.BASE_LR - weight_decay = cfg.SOLVER.WEIGHT_DECAY - if "backbone" in key: - lr = lr * cfg.SOLVER.BACKBONE_MULTIPLIER - params += [{"params": [value], "lr": lr, "weight_decay": weight_decay}] - - def maybe_add_full_model_gradient_clipping(optim): # optim: the optimizer class - # detectron2 doesn't have full model gradient clipping now - clip_norm_val = cfg.SOLVER.CLIP_GRADIENTS.CLIP_VALUE - enable = ( - cfg.SOLVER.CLIP_GRADIENTS.ENABLED - and cfg.SOLVER.CLIP_GRADIENTS.CLIP_TYPE == "full_model" - and clip_norm_val > 0.0 - ) - - class FullModelGradientClippingOptimizer(optim): - def step(self, closure=None): - all_params = itertools.chain(*[x["params"] for x in self.param_groups]) - torch.nn.utils.clip_grad_norm_(all_params, clip_norm_val) - super().step(closure=closure) - - return FullModelGradientClippingOptimizer if enable else optim - - optimizer_type = cfg.SOLVER.OPTIMIZER - if optimizer_type == "SGD": - optimizer = maybe_add_full_model_gradient_clipping(torch.optim.SGD)( - params, cfg.SOLVER.BASE_LR, momentum=cfg.SOLVER.MOMENTUM - ) - elif optimizer_type == "ADAMW": - optimizer = maybe_add_full_model_gradient_clipping(torch.optim.AdamW)( - params, cfg.SOLVER.BASE_LR - ) - else: - raise NotImplementedError(f"no optimizer type {optimizer_type}") - if not cfg.SOLVER.CLIP_GRADIENTS.CLIP_TYPE == "full_model": - optimizer = maybe_add_gradient_clipping(cfg, optimizer) - return optimizer - - -def setup(args): - """ - Create configs and perform basic setups. - """ - cfg = get_cfg() - add_detr_config(cfg) - cfg.merge_from_file(args.config_file) - cfg.merge_from_list(args.opts) - cfg.freeze() - default_setup(cfg, args) - return cfg - - -def main(args): - cfg = setup(args) - - if args.eval_only: - model = Trainer.build_model(cfg) - DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(cfg.MODEL.WEIGHTS, resume=args.resume) - res = Trainer.test(cfg, model) - if comm.is_main_process(): - verify_results(cfg, res) - return res - - trainer = Trainer(cfg) - trainer.resume_or_load(resume=args.resume) - return trainer.train() - - -if __name__ == "__main__": - args = default_argument_parser().parse_args() - print("Command Line Args:", args) - launch( - main, - args.num_gpus, - num_machines=args.num_machines, - machine_rank=args.machine_rank, - dist_url=args.dist_url, - args=(args,), - ) -- Gitee From 95cf61a0272fb2d58ccff1134add973532d51465 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:20:09 +0000 Subject: [PATCH 53/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20Py?= =?UTF-8?q?Torch/contrib/cv/detection/DETR/.keep?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- PyTorch/contrib/cv/detection/DETR/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 PyTorch/contrib/cv/detection/DETR/.keep diff --git a/PyTorch/contrib/cv/detection/DETR/.keep b/PyTorch/contrib/cv/detection/DETR/.keep deleted file mode 100644 index e69de29bb2..0000000000 -- Gitee From ff2d458ec7d0a7b2c8a3542c4c919c3086634fb5 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:20:33 +0000 Subject: [PATCH 54/59] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=20PyTorch/con?= =?UTF-8?q?trib/cv/detection/DETR/train=5Fnpu=5Fhw.py=20=E4=B8=BA=20PyTorc?= =?UTF-8?q?h/contrib/cv/detection/DETR/train=5Fnpu.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../contrib/cv/detection/DETR/{train_npu_hw.py => train_npu.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename PyTorch/contrib/cv/detection/DETR/{train_npu_hw.py => train_npu.py} (100%) diff --git a/PyTorch/contrib/cv/detection/DETR/train_npu_hw.py b/PyTorch/contrib/cv/detection/DETR/train_npu.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/train_npu_hw.py rename to PyTorch/contrib/cv/detection/DETR/train_npu.py -- Gitee From 50af2e7f0e312d1895930a4d3aaada227a8937a0 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:21:17 +0000 Subject: [PATCH 55/59] update PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh. --- PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh b/PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh index 0fb18080be..6e9b336509 100644 --- a/PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh +++ b/PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh @@ -80,7 +80,7 @@ if [ x"${etp_flag}" != x"true" ];then fi -nohup taskset -c 0-23 python3.7 train_npu_hw.py \ +nohup taskset -c 0-23 python3.7 -u train_npu.py \ --coco_path=${data_path} \ --workers=${workers} \ --gpu=${ASCEND_DEVICE_ID} \ -- Gitee From 28d82b35071b1abfc96bb692359263baba60a28b Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:21:39 +0000 Subject: [PATCH 56/59] =?UTF-8?q?=E9=87=8D=E5=91=BD=E5=90=8D=20PyTorch/con?= =?UTF-8?q?trib/cv/detection/DETR=20=E4=B8=BA=20PyTorch/contrib/cv/detecti?= =?UTF-8?q?on/DETR=5Ffor=5FPyTorch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cv/detection/{DETR => DETR_for_PyTorch}/LICENSE | 0 .../__pycache__/OXInterface.cpython-37.pyc | Bin .../__pycache__/coco_eval.cpython-37.pyc | Bin .../__pycache__/engine.cpython-36.pyc | Bin .../__pycache__/engine.cpython-37.pyc | Bin .../__pycache__/hubconf.cpython-36.pyc | Bin .../__pycache__/hubconf.cpython-37.pyc | Bin .../__pycache__/npu_fused_adamw.cpython-37.pyc | Bin .../__pycache__/train_npu_hw.cpython-37.pyc | Bin .../{DETR => DETR_for_PyTorch}/coco_eval.py | 0 .../{DETR => DETR_for_PyTorch}/datasets/__init__.py | 0 .../datasets/__pycache__/__init__.cpython-36.pyc | Bin .../datasets/__pycache__/__init__.cpython-37.pyc | Bin .../datasets/__pycache__/coco.cpython-36.pyc | Bin .../datasets/__pycache__/coco.cpython-37.pyc | Bin .../datasets/__pycache__/coco_eval.cpython-36.pyc | Bin .../datasets/__pycache__/coco_eval.cpython-37.pyc | Bin .../__pycache__/panoptic_eval.cpython-36.pyc | Bin .../__pycache__/panoptic_eval.cpython-37.pyc | Bin .../datasets/__pycache__/transforms.cpython-36.pyc | Bin .../datasets/__pycache__/transforms.cpython-37.pyc | Bin .../{DETR => DETR_for_PyTorch}/datasets/coco.py | 0 .../datasets/coco_eval.py | 0 .../datasets/coco_panoptic.py | 0 .../datasets/panoptic_eval.py | 0 .../datasets/transforms.py | 0 .../detection/{DETR => DETR_for_PyTorch}/engine.py | 0 .../detection/{DETR => DETR_for_PyTorch}/hubconf.py | 0 .../{DETR => DETR_for_PyTorch}/models/__init__.py | 0 .../models/__pycache__/__init__.cpython-36.pyc | Bin .../models/__pycache__/__init__.cpython-37.pyc | Bin .../models/__pycache__/backbone.cpython-36.pyc | Bin .../models/__pycache__/backbone.cpython-37.pyc | Bin .../models/__pycache__/detr.cpython-36.pyc | Bin .../models/__pycache__/detr.cpython-37.pyc | Bin .../models/__pycache__/matcher.cpython-36.pyc | Bin .../models/__pycache__/matcher.cpython-37.pyc | Bin .../__pycache__/position_encoding.cpython-36.pyc | Bin .../__pycache__/position_encoding.cpython-37.pyc | Bin .../models/__pycache__/segmentation.cpython-36.pyc | Bin .../models/__pycache__/segmentation.cpython-37.pyc | Bin .../models/__pycache__/transformer.cpython-36.pyc | Bin .../models/__pycache__/transformer.cpython-37.pyc | Bin .../{DETR => DETR_for_PyTorch}/models/backbone.py | 0 .../{DETR => DETR_for_PyTorch}/models/detr.py | 0 .../{DETR => DETR_for_PyTorch}/models/matcher.py | 0 .../models/position_encoding.py | 0 .../models/segmentation.py | 0 .../models/transformer.py | 0 .../{DETR => DETR_for_PyTorch}/requirements.txt | 0 .../{DETR => DETR_for_PyTorch}/run_with_submitit.py | 0 .../{DETR => DETR_for_PyTorch}/test/env_npu.sh | 0 .../test/train_full_8p.sh | 0 .../test/train_performance_1p.sh | 0 .../test/train_performance_8p.sh | 0 .../{DETR => DETR_for_PyTorch}/train_npu.py | 0 .../{DETR => DETR_for_PyTorch}/util/__init__.py | 0 .../util/__pycache__/__init__.cpython-36.pyc | Bin .../util/__pycache__/__init__.cpython-37.pyc | Bin .../util/__pycache__/box_ops.cpython-36.pyc | Bin .../util/__pycache__/box_ops.cpython-37.pyc | Bin .../util/__pycache__/misc.cpython-36.pyc | Bin .../util/__pycache__/misc.cpython-37.pyc | Bin .../{DETR => DETR_for_PyTorch}/util/box_ops.py | 0 .../{DETR => DETR_for_PyTorch}/util/misc.py | 0 .../{DETR => DETR_for_PyTorch}/util/plot_utils.py | 0 .../{DETR => DETR_for_PyTorch}/validate.py | 0 67 files changed, 0 insertions(+), 0 deletions(-) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/LICENSE (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/__pycache__/OXInterface.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/__pycache__/coco_eval.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/__pycache__/engine.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/__pycache__/engine.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/__pycache__/hubconf.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/__pycache__/hubconf.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/__pycache__/npu_fused_adamw.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/__pycache__/train_npu_hw.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/coco_eval.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__init__.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/__init__.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/__init__.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/coco.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/coco.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/coco_eval.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/coco_eval.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/panoptic_eval.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/panoptic_eval.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/transforms.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/__pycache__/transforms.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/coco.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/coco_eval.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/coco_panoptic.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/panoptic_eval.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/datasets/transforms.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/engine.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/hubconf.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__init__.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/__init__.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/__init__.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/backbone.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/backbone.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/detr.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/detr.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/matcher.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/matcher.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/position_encoding.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/position_encoding.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/segmentation.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/segmentation.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/transformer.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/__pycache__/transformer.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/backbone.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/detr.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/matcher.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/position_encoding.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/segmentation.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/models/transformer.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/requirements.txt (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/run_with_submitit.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/test/env_npu.sh (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/test/train_full_8p.sh (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/test/train_performance_1p.sh (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/test/train_performance_8p.sh (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/train_npu.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/__init__.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/__pycache__/__init__.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/__pycache__/__init__.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/__pycache__/box_ops.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/__pycache__/box_ops.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/__pycache__/misc.cpython-36.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/__pycache__/misc.cpython-37.pyc (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/box_ops.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/misc.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/util/plot_utils.py (100%) rename PyTorch/contrib/cv/detection/{DETR => DETR_for_PyTorch}/validate.py (100%) diff --git a/PyTorch/contrib/cv/detection/DETR/LICENSE b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/LICENSE similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/LICENSE rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/LICENSE diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/OXInterface.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/OXInterface.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/__pycache__/OXInterface.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/OXInterface.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/coco_eval.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/coco_eval.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/__pycache__/coco_eval.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/coco_eval.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/engine.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/engine.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/__pycache__/engine.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/engine.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/engine.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/engine.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/__pycache__/engine.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/engine.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/hubconf.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/hubconf.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/__pycache__/hubconf.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/hubconf.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/hubconf.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/hubconf.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/__pycache__/hubconf.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/hubconf.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/npu_fused_adamw.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/npu_fused_adamw.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/__pycache__/npu_fused_adamw.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/npu_fused_adamw.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/__pycache__/train_npu_hw.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/train_npu_hw.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/__pycache__/train_npu_hw.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/__pycache__/train_npu_hw.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/coco_eval.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/coco_eval.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/coco_eval.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/coco_eval.py diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__init__.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__init__.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__init__.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__init__.py diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/__init__.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/__init__.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/__init__.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/__init__.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/__init__.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/coco.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/coco.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/coco.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/coco.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/coco_eval.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/coco_eval.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/coco_eval.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/coco_eval.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/coco_eval.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/panoptic_eval.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/panoptic_eval.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/panoptic_eval.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/panoptic_eval.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/panoptic_eval.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/panoptic_eval.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/panoptic_eval.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/panoptic_eval.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/transforms.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/transforms.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/transforms.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/__pycache__/transforms.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/__pycache__/transforms.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/coco.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/coco.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/coco.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/coco.py diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/coco_eval.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/coco_eval.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/coco_eval.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/coco_eval.py diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/coco_panoptic.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/coco_panoptic.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/coco_panoptic.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/coco_panoptic.py diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/panoptic_eval.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/panoptic_eval.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/panoptic_eval.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/panoptic_eval.py diff --git a/PyTorch/contrib/cv/detection/DETR/datasets/transforms.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/transforms.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/datasets/transforms.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/datasets/transforms.py diff --git a/PyTorch/contrib/cv/detection/DETR/engine.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/engine.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/engine.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/engine.py diff --git a/PyTorch/contrib/cv/detection/DETR/hubconf.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/hubconf.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/hubconf.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/hubconf.py diff --git a/PyTorch/contrib/cv/detection/DETR/models/__init__.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__init__.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__init__.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__init__.py diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/__init__.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/__init__.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/__init__.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/__init__.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/__init__.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/backbone.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/backbone.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/backbone.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/backbone.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/backbone.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/detr.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/detr.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/detr.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/detr.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/detr.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/matcher.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/matcher.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/matcher.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/matcher.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/matcher.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/position_encoding.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/position_encoding.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/position_encoding.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/position_encoding.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/position_encoding.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/position_encoding.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/position_encoding.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/position_encoding.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/segmentation.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/segmentation.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/segmentation.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/segmentation.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/segmentation.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/segmentation.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/segmentation.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/segmentation.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/transformer.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/transformer.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/transformer.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/transformer.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/__pycache__/transformer.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/transformer.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/__pycache__/transformer.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/__pycache__/transformer.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/models/backbone.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/backbone.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/backbone.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/backbone.py diff --git a/PyTorch/contrib/cv/detection/DETR/models/detr.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/detr.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/detr.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/detr.py diff --git a/PyTorch/contrib/cv/detection/DETR/models/matcher.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/matcher.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/matcher.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/matcher.py diff --git a/PyTorch/contrib/cv/detection/DETR/models/position_encoding.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/position_encoding.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/position_encoding.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/position_encoding.py diff --git a/PyTorch/contrib/cv/detection/DETR/models/segmentation.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/segmentation.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/segmentation.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/segmentation.py diff --git a/PyTorch/contrib/cv/detection/DETR/models/transformer.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/transformer.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/models/transformer.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/models/transformer.py diff --git a/PyTorch/contrib/cv/detection/DETR/requirements.txt b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/requirements.txt similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/requirements.txt rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/requirements.txt diff --git a/PyTorch/contrib/cv/detection/DETR/run_with_submitit.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/run_with_submitit.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/run_with_submitit.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/run_with_submitit.py diff --git a/PyTorch/contrib/cv/detection/DETR/test/env_npu.sh b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/env_npu.sh similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/test/env_npu.sh rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/env_npu.sh diff --git a/PyTorch/contrib/cv/detection/DETR/test/train_full_8p.sh b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_full_8p.sh similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/test/train_full_8p.sh rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_full_8p.sh diff --git a/PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_1p.sh similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/test/train_performance_1p.sh rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_1p.sh diff --git a/PyTorch/contrib/cv/detection/DETR/test/train_performance_8p.sh b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/test/train_performance_8p.sh rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh diff --git a/PyTorch/contrib/cv/detection/DETR/train_npu.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/train_npu.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/train_npu.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/train_npu.py diff --git a/PyTorch/contrib/cv/detection/DETR/util/__init__.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__init__.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/__init__.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__init__.py diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/__init__.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/__init__.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/__pycache__/__init__.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/__init__.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/__init__.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/__init__.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/__pycache__/__init__.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/__init__.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/box_ops.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/box_ops.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/__pycache__/box_ops.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/box_ops.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/box_ops.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/box_ops.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/__pycache__/box_ops.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/box_ops.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/misc.cpython-36.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/misc.cpython-36.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/__pycache__/misc.cpython-36.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/misc.cpython-36.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/util/__pycache__/misc.cpython-37.pyc b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/misc.cpython-37.pyc similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/__pycache__/misc.cpython-37.pyc rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/__pycache__/misc.cpython-37.pyc diff --git a/PyTorch/contrib/cv/detection/DETR/util/box_ops.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/box_ops.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/box_ops.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/box_ops.py diff --git a/PyTorch/contrib/cv/detection/DETR/util/misc.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/misc.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/misc.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/misc.py diff --git a/PyTorch/contrib/cv/detection/DETR/util/plot_utils.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/plot_utils.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/util/plot_utils.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/util/plot_utils.py diff --git a/PyTorch/contrib/cv/detection/DETR/validate.py b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/validate.py similarity index 100% rename from PyTorch/contrib/cv/detection/DETR/validate.py rename to PyTorch/contrib/cv/detection/DETR_for_PyTorch/validate.py -- Gitee From 8d534cd2f145cc78be453ce787674fbdd527ce07 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:22:38 +0000 Subject: [PATCH 57/59] update PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh. --- .../cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh index 3d36493e1b..32f5fa9db4 100644 --- a/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh +++ b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh @@ -77,7 +77,7 @@ for i in $(seq 0 7) do PID_START=$((KERNEL_NUM * i)) PID_END=$((PID_START + KERNEL_NUM - 1)) - taskset -c $PID_START-$PID_END python3.7 -u train_npu_hw.py \ + taskset -c $PID_START-$PID_END python3.7 -u train_npu.py \ --addr=$(hostname -I |awk '{print $1}') \ --workers=$(nproc) \ --multiprocessing_distributed \ -- Gitee From 918dbf36dc06fac798bf5cc158798b9331d7f5ad Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:23:10 +0000 Subject: [PATCH 58/59] update PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_full_8p.sh. --- .../contrib/cv/detection/DETR_for_PyTorch/test/train_full_8p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_full_8p.sh b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_full_8p.sh index 9522f2fb28..f5c6678e22 100644 --- a/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_full_8p.sh +++ b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_full_8p.sh @@ -77,7 +77,7 @@ for i in $(seq 0 7) do PID_START=$((KERNEL_NUM * i)) PID_END=$((PID_START + KERNEL_NUM - 1)) - taskset -c $PID_START-$PID_END python3.7 -u train_npu_hw.py \ + taskset -c $PID_START-$PID_END python3.7 -u train_npu.py \ --addr=$(hostname -I |awk '{print $1}') \ --workers=$(nproc) \ --multiprocessing_distributed \ -- Gitee From 59e56edb22b77ecd34c6d3f21da9aec30bd9bb35 Mon Sep 17 00:00:00 2001 From: eason_hw <94364678@qq.com> Date: Sat, 2 Apr 2022 07:23:41 +0000 Subject: [PATCH 59/59] update PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh. --- .../cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh index 32f5fa9db4..c4182b7a1f 100644 --- a/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh +++ b/PyTorch/contrib/cv/detection/DETR_for_PyTorch/test/train_performance_8p.sh @@ -12,7 +12,7 @@ export RANK_SIZE=8 data_path="" # 训练epoch -train_epochs=2 +train_epochs=1 # 学习率 learning_rate=0.0001 # 加载数据进程数 -- Gitee