質問編集履歴
3
補足情報を書いた。
title
CHANGED
|
File without changes
|
body
CHANGED
|
@@ -127,134 +127,6 @@
|
|
|
127
127
|
time.sleep(10)
|
|
128
128
|
|
|
129
129
|
##追記
|
|
130
|
-
適当にードを実行すると以下のようなエラー文が出ました。
|
|
131
|
-
|
|
132
|
-
エラー文の最後に、「TypeError: 'in <string>' requires string as left operand, not NoneType」と書いてあるので、for文の所でしょうか。
|
|
133
|
-
|
|
134
|
-
どこを、どう直せばよいかわからず質問しました。
|
|
135
|
-
教えて下さい。
|
|
136
|
-
よろしくお願いします。
|
|
137
|
-
|
|
138
|
-
##発生している問題・エラーメッセージ
|
|
139
|
-
Traceback (most recent call last):
|
|
140
|
-
File "d:/Pythonからspreadsheetsへ/venv1/lancerswork/scraping.py", line 111, in <module>
|
|
141
|
-
set_with_dataframe(sh, dict_df)
|
|
142
|
-
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\gspread_dataframe.py", line 241, in set_with_dataframe
|
|
143
|
-
_cellrepr(cell_value, allow_formulas))
|
|
144
|
-
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\gspread_dataframe.py", line 49, in _cellrepr
|
|
145
|
-
if pd.isnull(value) is True:
|
|
146
|
-
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\pandas\core\dtypes\missing.py", line 126, in isna
|
|
147
|
-
return _isna(obj)
|
|
148
|
-
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\pandas\core\dtypes\missing.py", line 141, in _isna_new
|
|
149
|
-
elif isinstance(
|
|
150
|
-
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\pandas\core\dtypes\generic.py", line 12, in _check
|
|
151
|
-
return getattr(inst, attr, "_typ") in comp
|
|
152
|
-
TypeError: 'in <string>' requires string as left operand, not NoneType
|
|
153
|
-
|
|
154
|
-
##該当のソースコード
|
|
155
|
-
```python
|
|
156
|
-
import requests
|
|
157
|
-
from bs4 import BeautifulSoup as bs
|
|
158
|
-
|
|
159
|
-
from selenium import webdriver
|
|
160
|
-
import time
|
|
161
|
-
from tqdm import tqdm
|
|
162
|
-
|
|
163
|
-
import json
|
|
164
|
-
import gspread
|
|
165
|
-
from googleapiclient import discovery
|
|
166
|
-
from oauth2client.service_account import ServiceAccountCredentials
|
|
167
|
-
from gspread_dataframe import get_as_dataframe
|
|
168
|
-
from gspread_dataframe import set_with_dataframe
|
|
169
|
-
import pandas as pd
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36"}
|
|
173
|
-
|
|
174
|
-
office_names = []
|
|
175
|
-
address_lists = []
|
|
176
|
-
|
|
177
|
-
profile_names = []
|
|
178
|
-
office_names2 = []
|
|
179
|
-
|
|
180
|
-
for page in range(1,3):
|
|
181
|
-
url = 'https://www.zeiri4.com/firm/search/?FirmSearchForm%5BPrefecture_id%5D=13&FirmSearchForm%5BIndustry_id%5D%5B0%5D=1&FirmSearchForm%5BIndustry_id%5D%5B1%5D=4&page={}'.format(page)
|
|
182
|
-
|
|
183
|
-
response = requests.get(url, headers=headers)
|
|
184
|
-
soup = bs(response.content, 'html.parser')
|
|
185
|
-
|
|
186
|
-
for office in soup.find_all('h2'):
|
|
187
|
-
offices = office.text.strip('\n')
|
|
188
|
-
office_names.append(offices)
|
|
189
|
-
time.sleep(10)
|
|
190
|
-
|
|
191
|
-
for address in soup.find_all('dl', class_='b-firmSearchPanel__datalist'):
|
|
192
|
-
address_ = address.find('dd')
|
|
193
|
-
address_lists.append(address_)
|
|
194
|
-
time.sleep(10)
|
|
195
|
-
|
|
196
|
-
url1 = 'https://www.bengo4.com/tokyo/f_12/?page={}'.format(page)
|
|
197
|
-
response1 = requests.get(url1, headers=headers)
|
|
198
|
-
soup1 = bs(response1.content, 'html.parser')
|
|
199
|
-
|
|
200
|
-
for name in soup1.find_all(class_='profile__name'):
|
|
201
|
-
name_ = name.text
|
|
202
|
-
profile_names.append(name_)
|
|
203
|
-
time.sleep(10)
|
|
204
|
-
|
|
205
|
-
for office2 in soup1.find_all('p', class_='office'):
|
|
206
|
-
office2_ = office2.text
|
|
207
|
-
office_names2.append(office2_)
|
|
208
|
-
time.sleep(10)
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
browser = webdriver.Chrome(r'D:\Pythonからspreadsheetsへ\venv1\lancerswork\chromedriver.exe')
|
|
212
|
-
browser.implicitly_wait(10)
|
|
213
|
-
|
|
214
|
-
address_lists2 = []
|
|
215
|
-
|
|
216
|
-
for page in range(1,3):
|
|
217
|
-
url2 = 'https://www.bengo4.com/tokyo/f_12/?page={}'.format(page)
|
|
218
|
-
browser.get(url2)
|
|
219
|
-
|
|
220
|
-
for address2 in browser.find_elements_by_class_name('address'):
|
|
221
|
-
address2_ = address2.text
|
|
222
|
-
address_lists2.append(address2_)
|
|
223
|
-
time.sleep(10)
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
|
|
227
|
-
credentials = ServiceAccountCredentials.from_json_keyfile_name('伏せておきます', scope)
|
|
228
|
-
gc = gspread.authorize(credentials)
|
|
229
|
-
SPREADSHEET_KEY = '1qPbI3Rf995Z53sU3sK60SlgMwtFkO13w4P-wBQlTcv8'
|
|
230
|
-
worksheet = gc.open_by_key(SPREADSHEET_KEY)
|
|
231
|
-
wb = worksheet.sheet1
|
|
232
|
-
|
|
233
|
-
mydict = {
|
|
234
|
-
'事務所名':office_names,
|
|
235
|
-
'住所':address_lists,
|
|
236
|
-
'':'',
|
|
237
|
-
'弁護士名':profile_names,
|
|
238
|
-
'事務所名1':office_names2,
|
|
239
|
-
'住所1':address_lists2,
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
dict_df = pd.DataFrame({key:pd.Series(value) for key, value in mydict.items()})
|
|
243
|
-
|
|
244
|
-
sh = gc.open_by_key('1qPbI3Rf995Z53sU3sK60SlgMwtFkO13w4P-wBQlTcv8').worksheet('シート1')
|
|
245
|
-
|
|
246
|
-
set_with_dataframe(sh, dict_df)
|
|
247
|
-
```
|
|
248
|
-
|
|
249
|
-
##試したこと
|
|
250
|
-
以下のスクレイピングをする最初の部分だけを残し、他の部分をコメントアウトして実行したときはspread sheetまで書き込むことができました。
|
|
251
|
-
|
|
252
|
-
for office in soup.find_all('h2'):
|
|
253
|
-
offices = office.text.strip('\n')
|
|
254
|
-
office_names.append(offices)
|
|
255
|
-
time.sleep(10)
|
|
256
|
-
|
|
257
|
-
##追記
|
|
258
130
|
下のように適当に作った辞書型のデータで実行してみるとうまくいきますが、スクレイピングのデータで実行するとエラーになります。
|
|
259
131
|
|
|
260
132
|
mydict={
|
2
補足情報を書いた。
title
CHANGED
|
File without changes
|
body
CHANGED
|
@@ -124,4 +124,142 @@
|
|
|
124
124
|
for office in soup.find_all('h2'):
|
|
125
125
|
offices = office.text.strip('\n')
|
|
126
126
|
office_names.append(offices)
|
|
127
|
-
time.sleep(10)
|
|
127
|
+
time.sleep(10)
|
|
128
|
+
|
|
129
|
+
##追記
|
|
130
|
+
適当にードを実行すると以下のようなエラー文が出ました。
|
|
131
|
+
|
|
132
|
+
エラー文の最後に、「TypeError: 'in <string>' requires string as left operand, not NoneType」と書いてあるので、for文の所でしょうか。
|
|
133
|
+
|
|
134
|
+
どこを、どう直せばよいかわからず質問しました。
|
|
135
|
+
教えて下さい。
|
|
136
|
+
よろしくお願いします。
|
|
137
|
+
|
|
138
|
+
##発生している問題・エラーメッセージ
|
|
139
|
+
Traceback (most recent call last):
|
|
140
|
+
File "d:/Pythonからspreadsheetsへ/venv1/lancerswork/scraping.py", line 111, in <module>
|
|
141
|
+
set_with_dataframe(sh, dict_df)
|
|
142
|
+
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\gspread_dataframe.py", line 241, in set_with_dataframe
|
|
143
|
+
_cellrepr(cell_value, allow_formulas))
|
|
144
|
+
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\gspread_dataframe.py", line 49, in _cellrepr
|
|
145
|
+
if pd.isnull(value) is True:
|
|
146
|
+
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\pandas\core\dtypes\missing.py", line 126, in isna
|
|
147
|
+
return _isna(obj)
|
|
148
|
+
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\pandas\core\dtypes\missing.py", line 141, in _isna_new
|
|
149
|
+
elif isinstance(
|
|
150
|
+
File "D:\Pythonからspreadsheetsへ\venv1\lib\site-packages\pandas\core\dtypes\generic.py", line 12, in _check
|
|
151
|
+
return getattr(inst, attr, "_typ") in comp
|
|
152
|
+
TypeError: 'in <string>' requires string as left operand, not NoneType
|
|
153
|
+
|
|
154
|
+
##該当のソースコード
|
|
155
|
+
```python
|
|
156
|
+
import requests
|
|
157
|
+
from bs4 import BeautifulSoup as bs
|
|
158
|
+
|
|
159
|
+
from selenium import webdriver
|
|
160
|
+
import time
|
|
161
|
+
from tqdm import tqdm
|
|
162
|
+
|
|
163
|
+
import json
|
|
164
|
+
import gspread
|
|
165
|
+
from googleapiclient import discovery
|
|
166
|
+
from oauth2client.service_account import ServiceAccountCredentials
|
|
167
|
+
from gspread_dataframe import get_as_dataframe
|
|
168
|
+
from gspread_dataframe import set_with_dataframe
|
|
169
|
+
import pandas as pd
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36"}
|
|
173
|
+
|
|
174
|
+
office_names = []
|
|
175
|
+
address_lists = []
|
|
176
|
+
|
|
177
|
+
profile_names = []
|
|
178
|
+
office_names2 = []
|
|
179
|
+
|
|
180
|
+
for page in range(1,3):
|
|
181
|
+
url = 'https://www.zeiri4.com/firm/search/?FirmSearchForm%5BPrefecture_id%5D=13&FirmSearchForm%5BIndustry_id%5D%5B0%5D=1&FirmSearchForm%5BIndustry_id%5D%5B1%5D=4&page={}'.format(page)
|
|
182
|
+
|
|
183
|
+
response = requests.get(url, headers=headers)
|
|
184
|
+
soup = bs(response.content, 'html.parser')
|
|
185
|
+
|
|
186
|
+
for office in soup.find_all('h2'):
|
|
187
|
+
offices = office.text.strip('\n')
|
|
188
|
+
office_names.append(offices)
|
|
189
|
+
time.sleep(10)
|
|
190
|
+
|
|
191
|
+
for address in soup.find_all('dl', class_='b-firmSearchPanel__datalist'):
|
|
192
|
+
address_ = address.find('dd')
|
|
193
|
+
address_lists.append(address_)
|
|
194
|
+
time.sleep(10)
|
|
195
|
+
|
|
196
|
+
url1 = 'https://www.bengo4.com/tokyo/f_12/?page={}'.format(page)
|
|
197
|
+
response1 = requests.get(url1, headers=headers)
|
|
198
|
+
soup1 = bs(response1.content, 'html.parser')
|
|
199
|
+
|
|
200
|
+
for name in soup1.find_all(class_='profile__name'):
|
|
201
|
+
name_ = name.text
|
|
202
|
+
profile_names.append(name_)
|
|
203
|
+
time.sleep(10)
|
|
204
|
+
|
|
205
|
+
for office2 in soup1.find_all('p', class_='office'):
|
|
206
|
+
office2_ = office2.text
|
|
207
|
+
office_names2.append(office2_)
|
|
208
|
+
time.sleep(10)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
browser = webdriver.Chrome(r'D:\Pythonからspreadsheetsへ\venv1\lancerswork\chromedriver.exe')
|
|
212
|
+
browser.implicitly_wait(10)
|
|
213
|
+
|
|
214
|
+
address_lists2 = []
|
|
215
|
+
|
|
216
|
+
for page in range(1,3):
|
|
217
|
+
url2 = 'https://www.bengo4.com/tokyo/f_12/?page={}'.format(page)
|
|
218
|
+
browser.get(url2)
|
|
219
|
+
|
|
220
|
+
for address2 in browser.find_elements_by_class_name('address'):
|
|
221
|
+
address2_ = address2.text
|
|
222
|
+
address_lists2.append(address2_)
|
|
223
|
+
time.sleep(10)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
scope = ['https://spreadsheets.google.com/feeds','https://www.googleapis.com/auth/drive']
|
|
227
|
+
credentials = ServiceAccountCredentials.from_json_keyfile_name('伏せておきます', scope)
|
|
228
|
+
gc = gspread.authorize(credentials)
|
|
229
|
+
SPREADSHEET_KEY = '1qPbI3Rf995Z53sU3sK60SlgMwtFkO13w4P-wBQlTcv8'
|
|
230
|
+
worksheet = gc.open_by_key(SPREADSHEET_KEY)
|
|
231
|
+
wb = worksheet.sheet1
|
|
232
|
+
|
|
233
|
+
mydict = {
|
|
234
|
+
'事務所名':office_names,
|
|
235
|
+
'住所':address_lists,
|
|
236
|
+
'':'',
|
|
237
|
+
'弁護士名':profile_names,
|
|
238
|
+
'事務所名1':office_names2,
|
|
239
|
+
'住所1':address_lists2,
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
dict_df = pd.DataFrame({key:pd.Series(value) for key, value in mydict.items()})
|
|
243
|
+
|
|
244
|
+
sh = gc.open_by_key('1qPbI3Rf995Z53sU3sK60SlgMwtFkO13w4P-wBQlTcv8').worksheet('シート1')
|
|
245
|
+
|
|
246
|
+
set_with_dataframe(sh, dict_df)
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
##試したこと
|
|
250
|
+
以下のスクレイピングをする最初の部分だけを残し、他の部分をコメントアウトして実行したときはspread sheetまで書き込むことができました。
|
|
251
|
+
|
|
252
|
+
for office in soup.find_all('h2'):
|
|
253
|
+
offices = office.text.strip('\n')
|
|
254
|
+
office_names.append(offices)
|
|
255
|
+
time.sleep(10)
|
|
256
|
+
|
|
257
|
+
##追記
|
|
258
|
+
下のように適当に作った辞書型のデータで実行してみるとうまくいきますが、スクレイピングのデータで実行するとエラーになります。
|
|
259
|
+
|
|
260
|
+
mydict={
|
|
261
|
+
'yoo':2,
|
|
262
|
+
'poo':3,
|
|
263
|
+
'foo':4,
|
|
264
|
+
'too':5
|
|
265
|
+
}
|
1
補足情報を書いた。
title
CHANGED
|
File without changes
|
body
CHANGED
|
@@ -116,4 +116,12 @@
|
|
|
116
116
|
sh = gc.open_by_key('1qPbI3Rf995Z53sU3sK60SlgMwtFkO13w4P-wBQlTcv8').worksheet('シート1')
|
|
117
117
|
|
|
118
118
|
set_with_dataframe(sh, dict_df)
|
|
119
|
-
```
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
##試したこと
|
|
122
|
+
以下のスクレイピングをする最初の部分だけを残し、他の部分をコメントアウトして実行したときはspread sheetまで書き込むことができました。
|
|
123
|
+
|
|
124
|
+
for office in soup.find_all('h2'):
|
|
125
|
+
offices = office.text.strip('\n')
|
|
126
|
+
office_names.append(offices)
|
|
127
|
+
time.sleep(10)
|