Kerasの音声認識でモデルサイズを合わせれなくて予測できない

Question

PythonのKerasの音声認識でモデルサイズを合わせれなくて予測できない。

予め学習させたモデルを読み込んで、PCのマイクで5秒録音した後に、確率が0.8が次に遷移するプログラムなのですが、
学習済みモデルのサイズに入力音声を合わせれなくてつまずいています。

機械学習とプログラムも初心者なのでよろしくお願いします。

import numpy as np
from tensorflow import keras #Colabで学習させたら
import tensorflow as tf
import librosa

model = keras.models.load_model('esc50-sp-tpu_epoch10.h5', compile=False)
print(model.summary())

import pyaudio
import time
import librosa.display
import matplotlib.pyplot as plt
import IPython.display as ipd

#基本情報の設定
SAMPLING_RATE = 44100
CHUNK = 1024
RECORD_SECONDS = 5
FFT_SIZE = 256
THRESHOLD = 0.8
HOP_LENGTH=128

STATES = ['airplane','breathing','brushing_teeth','can_opening','car_horn','cat','chainsaw','chirping_birds','church_bells','clapping','clock_alarm','clock_tick','coughing','cow','crackling_fire','crickets','crow','crying_baby','dog','door_wood_creaks','door_wood_knock','drinking_sipping','engine','fireworks','footsteps','frog','glass_breaking','hand_saw','helicopter','hen','insects','keyboard_typing','laughing','mouse_click','pig','pouring_water','rain','rooster','sea_waves','sheep','siren','sneezing','snoring','thunderstorm','toilet_flush','train','vacuum_cleaner','washing_machine','water_drops','wind','Unknown']
last_state = STATES.index('Unknown')

# display wave in plots
def show_wave(x):
    plt.plot(x)
    plt.show()

# display wave in spectrogram
def show_sp(sp, fs, HOP_LENGTH):
    librosa.display.specshow(sp, sr=fs, x_axis="time", y_axis="log", hop_length=hop_length)
    plt.colorbar(format='%+2.0f dB')
    plt.title('Spectrogram')
    plt.show()

def calculate_sp(x, n_fft=FFT_SIZE, hop_length=HOP_LENGTH):
    stft = librosa.stft(x, n_fft=n_fft, hop_length=hop_length,window='hamming')
    sp = librosa.amplitude_to_db(np.abs(stft))
    return sp

count = 0
predictions_in_60_sec = np.empty((0, len(STATES) - 1))

audio_interface = pyaudio.PyAudio()
audio_stream = audio_interface.open(format=pyaudio.paInt16,
                                    channels=1,
                                    rate=SAMPLING_RATE,
                                    input=True,
                                    frames_per_buffer=CHUNK)
audio_stream.start_stream()

try:
    while True:
        all = []
        for i in range(0, int(SAMPLING_RATE / CHUNK * RECORD_SECONDS)):
            rec = audio_stream.read(CHUNK) #音声を読み取って、
            all.append(rec) #データを追加
        data = b"".join(all)
        data = np.frombuffer(data,dtype="int16") / float(2**15)

        print(data)
        show_wave(data)
        
       # Pause the audio stream
        audio_stream.stop_stream()

        start = time.time()
        
        state = last_state

        D = calculate_sp(data)
        show_sp(D, SAMPLING_RATE, HOP_LENGTH)
        print("wave size:{0}\nspectrogram size:{1}\nsamping rate:{2}".format(data.shape, D.shape, SAMPLING_RATE))
        print(D)
        
        #配列をモデルに変換
        D = D.reshape((1,) + D.shape + (1,))
        print("reshape:",D)
        
        magnitude = D
        predictions = model.predict(magnitude,verbose=False)
        predictions_mean = predictions.mean(axis=0)

        elapsed_time = time.time() - start
        
        print('{0:s} ({1:.3f}, processed in {2:.3f} seconds)'.format(
            STATES[predictions_mean.argmax()],
            predictions_mean.max(),
            elapsed_time))

        if predictions_mean.max() > THRESHOLD:
            state = predictions_mean.argmax()

        if last_state != state:
            print('CHANGED: {0} > {1}'.format(
                STATES[last_state], STATES[state]))
            last_state = state

        # Resume the audio stream
        audio_stream.start_stream()

except KeyboardInterrupt:
    print('Requested to terminate')

finally:
    audio_stream.stop_stream()
    audio_stream.close()
    audio_interface.terminate()
    print('Terminated')

エラーログ

ValueError Traceback (most recent call last)
<ipython-input-12-106cde585066> in <module>
73
74 magnitude = D
---> 75 predictions = model.predict(magnitude,verbose=False)
76 predictions_mean = predictions.mean(axis=0)
77

~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in predict(self, x, batch_size, verbose, steps, max_queue_size, workers, use_multiprocessing)
1094 # batch size.
1095 x, _, _ = self._standardize_user_data(
-> 1096 x, check_steps=True, steps_name='steps', steps=steps)
1097
1098 if (self.run_eagerly or (isinstance(x, iterator_ops.EagerIterator) and

~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, batch_size, check_steps, steps_name, steps, validation_split, shuffle)
2380 feed_input_shapes,
2381 check_batch_axis=False, # Don't enforce the batch size.
-> 2382 exception_prefix='input')
2383
2384 if y is not None:

~/anaconda3/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_utils.py in standardize_input_data(data, names, shapes, check_batch_axis, exception_prefix)
360 'Error when checking ' + exception_prefix + ': expected ' +
361 names[i] + ' to have shape ' + str(shape) +
--> 362 ' but got array with shape ' + str(data_shape))
363 return data
364

ValueError: Error when checking input: expected input_1 to have shape (128, 1723, 1) but got array with shape (129, 1721, 1)

Comments

[t_obara]

与えているデータの意味を理解していますか？
各ステップで、与えているデータがどのように変化しているのかを理解してみてください。どのステップでサイズが変わっているのかを把握し、なぜその行が必要なのかを理解してみてください。

[kazuma_kishi]

モデルのInputLayerが(None, 128, 1723, 1)なので、(1, 128, 1723, 1)の配列に成形すればいいのでしょうか？

Answer 1

録音の時間を調整してデータ量を1720→1723に出来なかったので、0で足すことに。

        D = calculate_sp(data)

        #モデル用の配列に変換
        freq = 128
        T = 1723
        
        #モデルの入力に足りない列を0で先頭に追加
        t_add = T - D.shape[1]
        for i in range(int(t_add)):
            D = np.insert(D, 0, 0, axis=1)
        D = D.reshape(1, freq, T, 1)