質問編集履歴

情報不足

2020/05/25 04:06

投稿

Hiro051

スコア9

title CHANGED Viewed

File without changes

body CHANGED Viewed

@@ -30,7 +30,7 @@
 ```
 ### 該当のソースコード
+Alexnet.py
 ```Python
 import torch
 import torchvision
@@ -67,118 +67,277 @@
       batch_size=64,
       shuffle=False,
       num_workers=4)
+```
+dataset.py
+```Python
+import pickle
+from tqdm import tqdm
-class AlexNet(nn.Module):
+import numpy as np
+import pandas as pd
-    def __init__(self, num_classes):
-        super(AlexNet, self).__init__()
-        self.features = nn.Sequential(
-            nn.Conv2d(3, 64, kernel_size=3, padding=1),
-            nn.ReLU(inplace=True),
-            nn.MaxPool2d(kernel_size=2, stride=2),
-            nn.Conv2d(64, 192, kernel_size=5, padding=2),
-            nn.ReLU(inplace=True),
-            nn.MaxPool2d(kernel_size=2, stride=2),
-            nn.Conv2d(192, 384, kernel_size=3, padding=1),
-            nn.ReLU(inplace=True),
-            nn.Conv2d(384, 256, kernel_size=3, padding=1),
-            nn.ReLU(inplace=True),
-            nn.Conv2d(256, 256, kernel_size=3, padding=1),
-            nn.ReLU(inplace=True),
-            nn.MaxPool2d(kernel_size=2, stride=2),
-        )
-        self.classifier = nn.Sequential(
-            nn.Dropout(),
+import pathlib
-            nn.Linear(256 * 4 * 4, 4096),
-            nn.ReLU(inplace=True),
-            nn.Dropout(),
-            nn.Linear(4096, 4096),
-            nn.ReLU(inplace=True),
-            nn.Linear(4096, num_classes),
-        )
-    def forward(self, x):
+# == Base ==
-        x = self.features(x)
-        x = x.view(x.size(0), 256 * 4 * 4)
+DATA_DIR = pathlib.Path('/home') / 'radiology_datas'
-        x = self.classifier(x)
-        return x
-# select device
+# == Dataset ==
-num_classes = 4
+ADNI1 = DATA_DIR / 'ADNI1'
+ADNI2 = DATA_DIR / 'JHU-radiology' / '20170509'
+ADNI2_2 = DATA_DIR / 'JHU-radiology' / 'MNI_skull_stripped' / 'output'
-device = 'cuda' if torch.cuda.is_available() else 'cpu'
+PPMI = DATA_DIR / 'JHU-radiology' / 'PPMI'
-net = AlexNet(num_classes).to(device)
+FourRTNI = DATA_DIR / 'JHU-radiology' / '4RTNI'
-# optimizing
-criterion = nn.CrossEntropyLoss()
-optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4)
+BLACKLIST_DIR = DATA_DIR / 'util' / 'lists'
-# training
-num_epochs = 20
+DATA_CSV = {
+    'ADNI': DATA_DIR / 'JHU-radiology' / 'ADNIMERGE.csv',
+    'PPMI': DATA_DIR / 'JHU-radiology' / 'PPMI.csv',
-train_loss_list, train_acc_list, val_loss_list, val_acc_list = [], [], [], []
+    '4RTNI': FourRTNI / 'csv' / '4RTNI_DATA.csv',
+}
-### training
+DATA_DIRS_DICT = {
+    'ADNI1': ADNI1,
+    'ADNI2': ADNI2,
-for epoch in range(num_epochs):
+    'ADNI2-2': ADNI2_2,
+    'PPMI': PPMI,
-    train_loss, train_acc, val_loss, val_acc = 0, 0, 0, 0
+    '4RTNI': FourRTNI / 'SkullStripped',
+}
+DATA_PREFIX_DICT = {
+    'fullsize': 'fullsize',
+    'half': 'half_',
+}
-    # ====== train_mode ======
+# == Label Encoder ==
-    net.train()
+CLASS_MAP = {
-    for i, (images, labels) in enumerate(train_loader):
-      images, labels = images.to(device), labels.to(device)
+    'CN': 0,
+    'AD': 1,
-      optimizer.zero_grad()
+    'EMCI': 2,
-      outputs = net(images)
+    'LMCI': 3,
-      loss = criterion(outputs, labels)
+    'MCI': 4,
+    'SMC': 5,
-      train_loss += loss.item()
+    'Control': 6,
-      train_acc += (outputs.max(1)[1] == labels).sum().item()
-      loss.backward()
+    'FControl': 6,
-      optimizer.step()
+    'PD': 7,
+    'SWEDD': 8,
+    'Prodromal': 9,
+    'CBD': 10,
+    'PSP': 11,
+    'Oth': 12,
-    avg_train_loss = train_loss / len(train_loader.dataset)
-    avg_train_acc = train_acc / len(train_loader.dataset)
+}
-    # ====== val_mode ======
-    net.eval()
-    with torch.no_grad():
-      for images, labels in test_loader:
-        images = images.to(device)
-        labels = labels.to(device)
-        outputs = net(images)
-        loss = criterion(outputs, labels)
-        val_loss += loss.item()
-        val_acc += (outputs.max(1)[1] == labels).sum().item()
-    avg_val_loss = val_loss / len(test_loader.dataset)
-    avg_val_acc = val_acc / len(test_loader.dataset)
+def read_voxel(path):
-    print ('Epoch [{}/{}], Loss: {loss:.4f}, val_loss: {val_loss:.4f}, val_acc: {val_acc:.4f}'
+    '''
+    pathを受け取ってvoxelを返すだけ
+    Args
+    ----------
+    path : pathlib
+        pklファイルへのパス
+    Return
+    ----------
-                   .format(epoch+1, num_epochs, i+1, loss=avg_train_loss, val_loss=avg_val_loss, val_acc=avg_val_acc))
+    voxel : numpy.array
+        pklファイルの中身
+    '''
-    train_loss_list.append(avg_train_loss)
+    with open(path, 'rb')as rf:
+        voxel = pickle.load(rf)
-    train_acc_list.append(avg_train_acc)
+    return np.array(voxel).astype('f')
-    val_loss_list.append(avg_val_loss)
-    val_acc_list.append(avg_val_acc)
-# plot graph
-plt.figure()
-plt.plot(range(num_epochs), train_loss_list, color='blue', linestyle='-', label='train_loss')
-plt.plot(range(num_epochs), val_loss_list, color='green', linestyle='--', label='val_loss')
-plt.legend()
+def get_uid(path):
-plt.xlabel('epoch')
+    '''
+    pathを受け取ってuidを返すだけ
+    Args
+    ----------
+    path : pathlib
+        pklファイルへのパス
+    Return
+    ----------
+    uid : int
+        uid
-plt.ylabel('loss')
+    '''
+    uid = path.name
-plt.title('Training and validation loss')
+    for key, value in DATA_DIRS_DICT.items():
-plt.grid()
+        if str(value) in str(path):
-plt.show()
-plt.figure()
-plt.plot(range(num_epochs), train_acc_list, color='blue', linestyle='-', label='train_acc')
-plt.plot(range(num_epochs), val_acc_list, color='green', linestyle='--', label='val_acc')
-plt.legend()
-plt.xlabel('epoch')
+            if key == 'ADNI2':
-plt.ylabel('acc')
+                uid = path.name.split('_')[-2]
-plt.title('Training and validation accuracy')
-plt.grid()
+                uid = int(uid[1:])
-plt.show()
+            elif key == 'ADNI2-2':
+                uid = path.name.split('_')[-4]
+                uid = int(uid[1:])
+            elif key == 'PPMI':
+                uid = path.name.split('_')[-4]
+                uid = int(uid)
+            elif key == '4RTNI':
+                uid = path.name.split('_')[-4]
+                uid = int(uid)
+            return uid
+def collect_pids(dirs):
+    '''
+    ディレクトリ内に存在するpatiantを集める
-```
+    Args
+    ----------
+    path : pathlib
+        pklファイルへのパス
-```type
+    Return
+    ----------
+    pid : list of str
+        pids
+    '''
+    patiants = []
+    for dir_path in dirs:
+        [patiants.append(f.name) for f in dir_path.iterdir()]
+    return patiants
+def get_blacklist():
+    '''
+    brain/util/listsの中にいるblacklistたちをuidのリストで返す
+    Args
+    ----------
+    Return
+    ----------
+    uid : list of int
+        uids
+    '''
+    key = '**/uids.txt'
+    excluded_uid_paths = BLACKLIST_DIR.glob(key)
+    excluded_uids = []
+    for path in excluded_uid_paths:
+        with open(path, 'r') as rf:
+            [excluded_uids.append(int(uid.rstrip('\n'))) for uid in rf]
+    return excluded_uids
+def load_csv_data(pids):
+    df = pd.read_csv(DATA_CSV['ADNI'])
+    adni = df[['PTID', 'AGE', 'PTGENDER']]
+    adni.columns = ['PID', 'AGE', 'SEX']
+    df = pd.read_csv(DATA_CSV['PPMI'])
+    ppmi = df[['Subject', 'Age', 'Sex']]
+    ppmi.columns = ['PID', 'AGE', 'SEX']
+    df = pd.read_csv(DATA_CSV['4RTNI'])
+    fourrtni = df[['SUBID', 'AGE_AT_TP0', 'SEX']]
+    fourrtni.columns = ['PID', 'AGE', 'SEX']
+    df = adni.append(ppmi).append(fourrtni)
+    df.iloc[:, 2] = df['SEX'].apply(lambda x: x[0] if x in ['Male', 'Female'] else x)
+    df.iloc[:, 1] = df['AGE'].apply(lambda x: int(x))
+    df.iloc[:, 0] = df['PID'].apply(lambda x: str(x))
+    return df
+def load_data(
+        kinds=['ADNI2', 'ADNI2-2', 'PPMI', '4RTNI'],
+        classes=['CN', 'AD', 'MCI', 'EMCI', 'LMCI', 'SMC', 'Control', 'PD', 'SWEDD', 'Prodromal', 'PSP', 'CBD', 'Oth', 'FControl'],
+        size='half',
+        csv=False,
+        pids=[],
+        uids=[],
+        unique=False,
+        blacklist=False,
+        dryrun=False,
+):
+    '''
+    Args
+    ----------
+    kind : list
+        ADNI2, ADNI2-2, PPMI をリストで指定
+    classes : list
+        CN, AD, MCI, EMCI, LMCI, SMC,
+        Control, PD, SWEDD, Prodromal,
+        PSP, CBD, Oth,
+        をリストで指定
+    size    : str
+        fullsize, half
+    pids    : list of str
+        取得したい患者のpidをリストで指定
+    uids    : list of str
+        取得したい患者のuidをリストで指定
+    unique  : bool
+    blacklist  : bool
+    dryrun  : bool
+        trueの場合にvoxelを読み込まないでその他の情報だけ返す
+    Return
+    ----------
+    dataset: list
+        情報がいっぱい詰まったリストだよ
+    '''
+    dirs = []
+    for key in kinds:
+        for c in classes:
+            dirname = DATA_DIRS_DICT[key].resolve() / c
+            if dirname.exists():
+                dirs.append(DATA_DIRS_DICT[key].resolve() / c)
+    dataset = []
+    key = '**/*' + DATA_PREFIX_DICT[size] + '*.pkl'
+    if dryrun:
+        print(f'[--DRYRUN--]')
+        print(f'[SIZE] {size}')
+        print(f'[KINDS] {kinds}')
+        print(f'[CLASSES] {classes}')
+        print(f'[PATIANT] {len(pids)} of patiants')
+        print(f'[TARGET] {uids}')
+        print(f'[UNIQUE] {unique}')
+        print(f'[BLACKLIST] {blacklist}')
+    for dir_path in dirs:
+        for file_path in dir_path.glob(key):
+            data = {}
+            data['uid'] = get_uid(file_path)
+            data['pid'] = file_path.parent.name
+            data['label'] = dir_path.name
+            data['nu_label'] = CLASS_MAP[dir_path.name]
+            data['path'] = file_path
+            dataset.append(data)
+    if uids:
+        dataset = [data for data in dataset if data['uid'] in uids]
+    if unique:
+        dataset_unique = []
+        for pid in collect_pids(dirs):
+            # pidごとにdataを取り出しそれらのuidをソートして最新のものを選択
+            dataset_unique.append(
+                sorted([data for data in dataset if data['pid'] == pid], key=lambda data: data['uid'])[-1])
+        dataset = dataset_unique
+    if pids:
+        dataset = [data for data in dataset if data['pid'] in pids]
+    if blacklist:
+        exclude_uids = get_blacklist()
+        dataset = [data for data in dataset if data['uid'] not in exclude_uids]
+    if dryrun:
-$ print(type(dataset))
+        return np.array(dataset)
+    if csv:
+        df = load_csv_data([data['pid'] for data in dataset])
+        [data.update(
+            AGE=df[df.PID == data['pid']].AGE.values[0],
+            SEX=df[df.PID == data['pid']].SEX.values[0],
+        ) if data['pid'] in df.PID.values else data.update(
+            AGE=None,
+            SEX=None,
+        ) for data in dataset]
+    [data.update(voxel=read_voxel(data['path'])) for data in tqdm(dataset, leave=False)]
-<class 'numpy.ndarray'>
+    return np.array(dataset)
 ```