tensorflow の学習がうまくいかない

前提・実現したいこと

以下サイトを参考にtensorflow object detection APIを使用して独自モデルの学習を試みています。
学習をどれだけ続けても精度が上がらず困っています。
https://github.com/EdjeElectronics/TensorFlow-Object-Detection-API-Tutorial-Train-Multiple-Objects-Windows-10
https://github.com/tensorflow/models/tree/master/research/object_detection

発生している問題・エラーメッセージ

添付画像に示すようにlossの推移がおかしい？ようで、作成したモデルで
動画の検出を行ってもかなり精度が低い検出となってしまいます。
学習にはvottにてアノテーションしたデータをroboflowにてtfrecordとしてtrain90%,test10%として出力し使用しました。実行に使用したpipelineは以下に示します。

エラーメッセージ

該当のソースコード

pipeline.config

python
1model {
2  ssd {
3    num_classes: 1
4    image_resizer {
5      fixed_shape_resizer {
6        height: 300
7        width: 300
8      }
9    }
10    feature_extractor {
11      type: "ssd_mobilenet_v1"
12      depth_multiplier: 1.0
13      min_depth: 16
14      conv_hyperparams {
15        regularizer {
16          l2_regularizer {
17            weight: 4e-05
18          }
19        }
20        initializer {
21          truncated_normal_initializer {
22            mean: 0.0
23            stddev: 0.03
24          }
25        }
26        activation: RELU_6
27        batch_norm {
28          decay: 0.9997
29          center: true
30          scale: true
31          epsilon: 0.001
32          train: true
33        }
34      }
35    }
36    box_coder {
37      faster_rcnn_box_coder {
38        y_scale: 10.0
39        x_scale: 10.0
40        height_scale: 5.0
41        width_scale: 5.0
42      }
43    }
44    matcher {
45      argmax_matcher {
46        matched_threshold: 0.5
47        unmatched_threshold: 0.5
48        ignore_thresholds: false
49        negatives_lower_than_unmatched: true
50        force_match_for_each_row: true
51      }
52    }
53    similarity_calculator {
54      iou_similarity {
55      }
56    }
57    box_predictor {
58      convolutional_box_predictor {
59        conv_hyperparams {
60          regularizer {
61            l2_regularizer {
62              weight: 4e-05
63            }
64          }
65          initializer {
66            truncated_normal_initializer {
67              mean: 0.0
68              stddev: 0.03
69            }
70          }
71          activation: RELU_6
72          batch_norm {
73            decay: 0.9997
74            center: true
75            scale: true
76            epsilon: 0.001
77            train: true
78          }
79        }
80        min_depth: 0
81        max_depth: 0
82        num_layers_before_predictor: 0
83        use_dropout: false
84        dropout_keep_probability: 0.8
85        kernel_size: 1
86        box_code_size: 4
87        apply_sigmoid_to_scores: false
88      }
89    }
90    anchor_generator {
91      ssd_anchor_generator {
92        num_layers: 6
93        min_scale: 0.2
94        max_scale: 0.95
95        aspect_ratios: 1.0
96        aspect_ratios: 2.0
97        aspect_ratios: 0.5
98        aspect_ratios: 3.0
99        aspect_ratios: 0.3333
100      }
101    }
102    post_processing {
103      batch_non_max_suppression {
104        score_threshold: 0.3
105        iou_threshold: 0.6
106        max_detections_per_class: 100
107        max_total_detections: 100
108      }
109      score_converter: SIGMOID
110    }
111    normalize_loss_by_num_matches: true
112    loss {
113      localization_loss {
114        weighted_smooth_l1 {
115        }
116      }
117      classification_loss {
118        weighted_sigmoid {
119        }
120      }
121      hard_example_miner {
122        num_hard_examples: 3000
123        iou_threshold: 0.99
124        loss_type: CLASSIFICATION
125        max_negatives_per_positive: 3
126        min_negatives_per_image: 0
127      }
128      classification_weight: 1.0
129      localization_weight: 1.0
130    }
131  }
132}
133train_config {
134  batch_size: 16
135  data_augmentation_options {
136    random_horizontal_flip {
137    }
138  }
139  data_augmentation_options {
140    ssd_random_crop {
141    }
142  }
143  optimizer {
144    rms_prop_optimizer {
145      learning_rate {
146        exponential_decay_learning_rate {
147          initial_learning_rate: 0.004
148          decay_steps: 800720
149          decay_factor: 0.95
150        }
151      }
152      momentum_optimizer_value: 0.9
153      decay: 0.9
154      epsilon: 1.0
155    }
156  }
157  fine_tune_checkpoint: "/content/drive/MyDrive/tensorflow1/models/research/object_detection/ssd_mobilenet_v1_coco_2018_01_28/model.ckpt"
158  from_detection_checkpoint: true
159  num_steps: 200000
160}
161train_input_reader {
162  label_map_path: "/content/drive/MyDrive/tensorflow1/models/research/object_detection/training/train_label_map.pbtxt"
163  tf_record_input_reader {
164    input_path: "/content/drive/MyDrive/tensorflow1/models/research/object_detection/train.tfrecord"
165  }
166}
167eval_config {
168  num_examples: 8000
169  max_evals: 10
170  use_moving_averages: false
171}
172eval_input_reader {
173  label_map_path: "/content/drive/MyDrive/tensorflow1/models/research/object_detection/training/train_label_map.pbtxt"
174  shuffle: false
175  num_readers: 1
176  tf_record_input_reader {
177    input_path: "/content/drive/MyDrive/tensorflow1/models/research/object_detection/test.tfrecord"
178  }
179}
180![![tensorboard](db99e5ae4210f30c9f3f5ea2e0d0dc13.jpeg)](8aea1d0321273f49a86e22bad586ac0d.jpeg)