pythonのプログラムを実行していく上でメモリについてわからないことがありましたので質問させていただきます。
現在、pythonでプログラムを書いているのですが、おそらく自分のプログラムの構成の仕方が悪いせいで、メモリを多く使ってしまっています。
そこで、なるべくメモリを消費しないようなpythonのプログラムを作成したいと思っています。
以下が現在使っているpythonのプログラムコードとなっています。(長くなってしまい、読みづらかったらすみません。)
python
1# -*- coding: utf-8 -*- 2#@test {"output": "ignore"} 3import tensorflow as tf 4import numpy as np 5import pandas as pd 6import glob 7import os 8import math 9 10def read_file(filename): 11 ds_tmp = [] 12 with open(filename, 'r') as f: 13 print(filename) 14 for i in f: 15 txt = i.split() 16 txt = [ float(s) for s in txt] 17 ds_tmp.append(txt) 18 19 ds = np.array(ds_tmp) 20 return ds 21 22 23def process(label): 24 train_filename = "../text/text2/input_train_{}.txt".format(label) 25 test_filename = "../text/text2/input_test_{}.txt".format(label) 26 train_data = read_file(train_filename) 27 test_data = read_file(test_filename) 28 y_train = train_data[:, 4:]/50.0 # 4列目のみを抽出 29 x_train = train_data[:, :4] # 0~4列目を抽出 30 y_test = test_data[:, 4:]/50.0 31 x_test = test_data[:, :4] 32 33 num_examples = y_train.shape[0] 34 num_input = x_train.shape[1] 35 36 bias_with_x = np.insert(x_train, 0, 1, axis=1).astype(np.float32) 37 bias_with_x_test = np.insert(x_test, 0, 1, axis=1).astype(np.float32) 38 39 num_hidden1 = 40 40 num_hidden2 = 40 41 training_steps = 25000 42 learning_rate = 0.005 43 lambda_2 = 0.00000001 44 MAE_train_ave = 0 45 MAE_test_ave = 0 46 47 for seed in range(1,11): 48 losses = [] 49 50 with tf.Session() as sess: 51 # Set up all the tensors, variables, and operations. 52 input = tf.constant(bias_with_x) 53 target = tf.constant(np.transpose([y_train]).astype(np.float32)) 54 shape = [num_input+1, num_hidden1] 55 n = shape[0] * shape[1] 56 stddev = math.sqrt(2.0 / n) 57 initial = tf.truncated_normal(shape=shape, stddev=stddev, seed=seed) 58 weights = tf.Variable(initial, name=None) 59 bias = tf.Variable(tf.random_uniform([num_hidden1], minval=-2.0, maxval=2.0, seed=seed)) 60 61 shape = [num_hidden1, num_hidden2] 62 n = shape[0] * shape[1] 63 stddev = math.sqrt(2.0 / n) 64 initial = tf.truncated_normal(shape=shape, stddev=stddev, seed=seed) 65 weights1 = tf.Variable(initial, name=None) 66 bias1 = tf.Variable(tf.random_uniform([num_hidden2], minval=-2.0, maxval=2.0, seed=seed)) 67 68 shape = [num_hidden2, 1] 69 n = shape[0] * shape[1] 70 stddev = math.sqrt(2.0 / n) 71 initial = tf.truncated_normal(shape=shape, stddev=stddev, seed=seed) 72 weights2 = tf.Variable(initial, name=None) 73 bias2 = tf.Variable(tf.random_uniform([1], minval=-2.0, maxval=2.0, seed=seed)) 74 tf.global_variables_initializer().run() 75 76 y1 = tf.nn.relu(tf.matmul(input, weights)) 77 y2 = tf.nn.relu(tf.matmul(y1, weights1)) 78 yhat = tf.sigmoid(tf.matmul(y2, weights2)) 79 yerror = tf.subtract(yhat, target) 80 loss1 = tf.nn.l2_loss(yerror) 81 L2_sqr = tf.nn.l2_loss(weights) + tf.nn.l2_loss(weights1) + tf.nn.l2_loss(weights2) 82 loss = loss1 + lambda_2 * L2_sqr 83 84 input_test = tf.constant(bias_with_x_test) 85 target_test = tf.constant(np.transpose([y_test]).astype(np.float32)) 86 y1_test = tf.nn.relu(tf.matmul(input_test, weights)) 87 y2_test = tf.nn.relu(tf.matmul(y1_test, weights1)) 88 yhat_test = tf.sigmoid(tf.matmul(y2_test, weights2)) 89 90 update_weights = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss) 91 92 for i in range(training_steps): 93 update_weights.run() 94 losses.append(loss.eval()) 95 # Training is done, get the final values for the graphs 96 betas = weights.eval() 97 yhat = yhat.eval() 98 error = np.sqrt(((y_train-yhat)*(y_train-yhat)).mean()) #RMSEの算出 99 100 size = len(y_train) 101 MAE = (np.sum(np.abs(y_train-yhat)))/size 102 103 yhat_test = yhat_test.eval() 104 size = len(y_test) 105 MAE_test = (np.sum(np.abs(y_test-yhat_test)))/size 106 107 global MAE_train_ave 108 MAE_train_ave += MAE 109 110 global MAE_test_ave 111 MAE_test_ave += MAE_test 112 113 Train = MAE_train_ave/10.0 114 Test = MAE_test_ave/10.0 115 print('学習誤差 : %s' % Train) 116 print('テスト誤差 : %s' % Test) 117 118 119def main(): 120 with open('../text/text2/label_text2.txt', 'r') as label: 121 a_list = [line.rstrip() for line in label] 122 123 for label in a_list: 124 process(label) 125 126if __name__ == '__main__': 127 main() 128 129
※ご回答いただいた内容に質問させていただくこともあるかと思いますので、
※よろしければご返信いただければと思います。
回答1件
あなたの回答
tips
プレビュー