teratail
回答率
85.47%
質問するログイン新規登録
トップスクレイピングに関する質問bet365のサイトがseleniumでスクレイピングができません。詳しい方、教えていただけないでしょか？

編集履歴

回答編集履歴

8

不具合などあっても 責任持ちません

2020/09/27 18:30

投稿

Reach
Reach

スコア733

test CHANGED
@@ -1,6 +1,8 @@
1
1
  (動作不良のため削除します)
2
2
 
3
3
 
4
+
5
+ 不具合などあっても 責任持ちません
4
6
 
5
7
 
6
8
 

7

追記

2020/09/27 18:30

投稿

Reach
Reach

スコア733

test CHANGED
@@ -101,3 +101,87 @@
101
101
  bet365 = Bet365Driver()
102
102
 
103
103
  ```
104
+
105
+
106
+
107
+
108
+
109
+ Firefox版 ？
110
+
111
+
112
+
113
+ ```Python
114
+
115
+ from selenium import webdriver
116
+
117
+ from selenium.webdriver.firefox.options import Options
118
+
119
+
120
+
121
+ from time import sleep
122
+
123
+ import os
124
+
125
+
126
+
127
+
128
+
129
+ class Bet365Driver(object):
130
+
131
+ def __init__(self):
132
+
133
+
134
+
135
+ options = Options()
136
+
137
+ #options.add_argument("--headless")
138
+
139
+
140
+
141
+ profile = webdriver.FirefoxProfile()
142
+
143
+ profile.set_preference("general.useragent.override","Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko")
144
+
145
+ profile.set_preference('useAutomationExtension', False)
146
+
147
+ profile.set_preference('devtools.jsonview.enabled', False)
148
+
149
+ profile.set_preference("dom.webdriver.enabled", False)
150
+
151
+ profile.update_preferences()
152
+
153
+ self.driver = webdriver.Firefox(executable_path="settings.driver_path",firefox_profile=profile, options=options)
154
+
155
+
156
+
157
+ self.driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
158
+
159
+
160
+
161
+ self.driver.get('https://www.bet365.com/')
162
+
163
+
164
+
165
+ sleep (8)
166
+
167
+ list = self.driver.find_elements_by_tag_name('div')
168
+
169
+ print (len(list))
170
+
171
+ test_text = self.driver.find_element_by_xpath('//div[@class="ool-OpenOffersLhsModule_Offer "]').text
172
+
173
+ print(test_text.encode('cp932', "ignore"))
174
+
175
+ self.driver.close()
176
+
177
+ self.driver.quit()
178
+
179
+
180
+
181
+
182
+
183
+ if __name__ == '__main__':
184
+
185
+ bet365 = Bet365Driver()
186
+
187
+ ```

6

再掲

2020/09/27 18:22

投稿

Reach
Reach

スコア733

test CHANGED
@@ -1 +1,103 @@
1
- 動作不良のため削除します
1
+ (動作不良のため削除します)
2
+
3
+
4
+
5
+
6
+
7
+ Chromeで 機能 (弾かれることあり)
8
+
9
+ 不要なコードの記述が あります
10
+
11
+
12
+
13
+
14
+
15
+ ```Python
16
+
17
+ from selenium import webdriver
18
+
19
+ from selenium.webdriver.chrome.options import Options
20
+
21
+
22
+
23
+ from time import sleep
24
+
25
+ import os
26
+
27
+
28
+
29
+ class Bet365Driver(object):
30
+
31
+ def __init__(self):
32
+
33
+ userdata_dir = 'C:/Users/' + os.getlogin() + '/AppData/Local/Google/Chrome/User Data'
34
+
35
+ options = Options()
36
+
37
+ options.add_argument("--headless")
38
+
39
+ options.add_argument("--disable-dev-shm-usage")
40
+
41
+ options.add_argument('--disable-extensions')
42
+
43
+ options.add_argument('--disable-gpu')
44
+
45
+ options.add_argument('--lang=ja-JP')
46
+
47
+ options.add_argument("--proxy-server='direct://'")
48
+
49
+ options.add_argument("--proxy-bypass-list=*")
50
+
51
+ options.add_argument("--start-maximized")
52
+
53
+ options.add_experimental_option("excludeSwitches", ["enable-automation"])
54
+
55
+ options.add_experimental_option('useAutomationExtension', False)
56
+
57
+ options.add_argument("--disable-blink-features=AutomationControlled")
58
+
59
+ options.add_argument("--no-referrers")
60
+
61
+ #options.add_argument('--user-data-dir=' + userdata_dir) # Headless時は 必ずコメントアウト
62
+
63
+
64
+
65
+
66
+
67
+ self.driver = webdriver.Chrome(executable_path="settings.driver_path", options=options)
68
+
69
+
70
+
71
+ self.driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
72
+
73
+ self.driver.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4071.0 Safari/537.36'})
74
+
75
+
76
+
77
+ self.driver.get('https://www.bet365.com/')
78
+
79
+
80
+
81
+ sleep (8)
82
+
83
+ list = self.driver.find_elements_by_tag_name('div')
84
+
85
+ print (len(list))
86
+
87
+ test_text = self.driver.find_element_by_xpath('//div[@class="ool-OpenOffersLhsModule_Offer "]').text
88
+
89
+ print(test_text.encode('cp932', "ignore"))
90
+
91
+ self.driver.close()
92
+
93
+ self.driver.quit()
94
+
95
+
96
+
97
+
98
+
99
+ if __name__ == '__main__':
100
+
101
+ bet365 = Bet365Driver()
102
+
103
+ ```

5

動作不良のため削除

2020/09/27 17:05

投稿

Reach
Reach

スコア733

test CHANGED
@@ -1,55 +1 @@
1
- Chromeですが‥‥
1
+ 動作不良のため削除します
2
-
3
- ```Python
4
-
5
- from selenium import webdriver
6
-
7
- from selenium.webdriver.chrome.options import Options
8
-
9
- from time import sleep
10
-
11
-
12
-
13
- class Bet365Driver(object):
14
-
15
- def __init__(self):
16
-
17
- options = Options()
18
-
19
- options.add_argument("start-maximized")
20
-
21
- options.add_argument("--disable-dev-shm-usage")
22
-
23
- options.add_experimental_option("excludeSwitches", ["enable-automation"])
24
-
25
- options.add_experimental_option('useAutomationExtension', False)
26
-
27
- options.add_argument("--disable-blink-features=AutomationControlled")
28
-
29
- #options.log.level = 'trace'
30
-
31
- self.driver = webdriver.Chrome(executable_path=settings.driver_path, options=options)
32
-
33
- self.driver.get('https://www.bet365.com/')
34
-
35
- sleep(5)
36
-
37
- elems = self.driver.find_elements_by_tag_name("div")
38
-
39
- print(len(elems))
40
-
41
- print(self.driver.title)
42
-
43
- self.driver.close()
44
-
45
- self.driver.quit()
46
-
47
-
48
-
49
-
50
-
51
- if __name__ == '__main__':
52
-
53
- bet365 = Bet365Driver()
54
-
55
- ```

4

修正2

2020/08/22 22:18

投稿

Reach
Reach

スコア733

test CHANGED
@@ -44,4 +44,12 @@
44
44
 
45
45
  self.driver.quit()
46
46
 
47
+
48
+
49
+
50
+
51
+ if __name__ == '__main__':
52
+
53
+ bet365 = Bet365Driver()
54
+
47
55
  ```

3

修正2

2020/08/22 18:01

投稿

Reach
Reach

スコア733

test CHANGED
@@ -28,7 +28,7 @@
28
28
 
29
29
  #options.log.level = 'trace'
30
30
 
31
- self.driver = webdriver.Chrome(executable_path="C:/Drivers/chromedriver.exe", options=options)
31
+ self.driver = webdriver.Chrome(executable_path=settings.driver_path, options=options)
32
32
 
33
33
  self.driver.get('https://www.bet365.com/')
34
34
 

2

v

2020/08/22 17:57

投稿

Reach
Reach

スコア733

test CHANGED
@@ -5,6 +5,8 @@
5
5
  from selenium import webdriver
6
6
 
7
7
  from selenium.webdriver.chrome.options import Options
8
+
9
+ from time import sleep
8
10
 
9
11
 
10
12
 
@@ -30,14 +32,16 @@
30
32
 
31
33
  self.driver.get('https://www.bet365.com/')
32
34
 
35
+ sleep(5)
33
36
 
37
+ elems = self.driver.find_elements_by_tag_name("div")
34
38
 
39
+ print(len(elems))
35
40
 
41
+ print(self.driver.title)
36
42
 
43
+ self.driver.close()
37
44
 
38
-
39
- if __name__ == '__main__':
40
-
41
- bet365 = Bet365Driver()
45
+ self.driver.quit()
42
46
 
43
47
  ```

1

修正

2020/08/22 17:56

投稿

Reach
Reach

スコア733

test CHANGED
@@ -28,12 +28,6 @@
28
28
 
29
29
  self.driver = webdriver.Chrome(executable_path="C:/Drivers/chromedriver.exe", options=options)
30
30
 
31
- self.driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
32
-
33
- self.driver.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.53 Safari/537.36'})
34
-
35
- #print(self.driver.execute_script("return navigator.userAgent;"))
36
-
37
31
  self.driver.get('https://www.bet365.com/')
38
32
 
39
33