トップに関する質問 Python Webスクレイピングにおけるscheduleモジュール実行時のエラー

編集履歴

質問編集履歴

コード修正

2021/02/24 06:53

投稿

SMRKW

スコア2

test CHANGED Viewed

File without changes

test CHANGED Viewed

@@ -210,7 +210,7 @@
-  try: #cがある場合
+  try: #cが8個ない場合に対応
         for i in range(8):
@@ -276,7 +276,7 @@
-  try: #cがある場合
+  try: #cが8個ない場合に対応
       for i in range(10):
@@ -342,7 +342,7 @@
-  try: #cがある場合
+  try: #cが8個ない場合に対応
        for i in range(10):

コード修正

2021/02/24 06:53

投稿

SMRKW

スコア2

test CHANGED Viewed

File without changes

test CHANGED Viewed

@@ -14,6 +14,8 @@
 関数の中にtry、exceptを組み込んでいる影響でしょうか。
+※if elseに書き換えても同様のメッセージが発生しましたので更新したコードを記載いたします。
 ### 発生している問題・エラーメッセージ
@@ -148,7 +150,9 @@
-#1ページ目
+#page_1
   result1 = requests.get(url1)
@@ -158,218 +162,218 @@
+  if soup1.find("div", {"class":"main-inner-a"}) is None:
+      title_1_a = []
+      overview_1_a = []
+      link_1_a = []
+  else:
-  page_1_a = soup1.find("div", {"class":"main-inner-a"})
+      page_1_a = soup1.find("div", {"class":"main-inner-a"})
-  try: #aがある場合
-    title_1_a = page_1_a.find("h3").text.replace("\u3000","").replace("\n","")
+      title_1_a = page_1_a.find("h3").text.replace("\u3000","").replace("\n","")
-    overview_1_a = page_1_a.find("p").text.replace("\u3000","").replace("\n","")
+      overview_1_a = page_1_a.find("p").text.replace("\u3000","").replace("\n","")
-    link_1_a = page_1_a.find("a").get("href")
+      link_1_a = page_1_a.find("a").get("href")
+  if soup1.find("div", {"class":"main-inner-b"}) is None:
+      title_1_b = []
+      overview_1_b = []
+      link_1_b = []
+  else:
+      page_1_b = soup1.find("div", {"class":"main-inner-b"})
+      title_1_b = page_1_b.find("h3").text.replace("\u3000","").replace("\n","")
+      overview_1_b = page_1_b.find("p").text.replace("\u3000","").replace("\n","")
+      link_1_b = page_1_b.find("a").get("href")
+  title_1_c = []
+  overview_1_c = []
+  link_1_c = []
+  try: #cがある場合
+        for i in range(8):
+          page_1_c = soup1.find_all("div", {"class":"main-inner-c"})[i]
+          tmp_title = page_1_c.find("h3").text.replace("\u3000","").replace("\n","")
+          title_1_c.append(tmp_title)
+          tmp_overview = page_1_c.find_all("p")[2].text.replace("\u3000","").replace("\n","")
+          overview_1_c.append(tmp_overview)
+          tmp_link = page_1_c.find("a").get("href")
+          link_1_c.append(tmp_link)
+          i += 1
   except:
+    pass
+  title_1_all = pd.DataFrame([title_1_a, title_1_b] + title_1_c)
+  overview_1_all = pd.DataFrame([overview_1_a, overview_1_b] + overview_1_c)
+  link_1_all = pd.DataFrame([link_1_a, link_1_b] + link_1_c)
+  page1_df = pd.concat([title_1_all, overview_1_all, link_1_all], axis=1)
+#page_2
+  result2 = requests.get(url2)
+  c2 = result2.content
+  soup2 = BeautifulSoup(c2)
-    title_1_a = []
+  title_2_c = []
-    overview_1_a = []
+  overview_2_c = []
-    link_1_a = []
+  link_2_c = []
-  page_1_b = soup1.find("div", {"class":"main-inner-b"})
-  try: #bがある場合
+  try: #cがある場合
+      for i in range(10):
+            page_2_c = soup2.find_all("div", {"class":"main-inner-c"})[i]
-    title_1_b = page_1_b.find("h3").text.replace("\u3000","").replace("\n","")
+            tmp_title = page_2_c.find("h3").text.replace("\u3000","").replace("\n","")
+            title_2_c.append(tmp_title)
-    overview_1_b = page_1_b.find("p").text.replace("\u3000","").replace("\n","")
+            tmp_overview = page_2_c.find_all("p")[2].text.replace("\u3000","").replace("\n","")
+            overview_2_c.append(tmp_overview)
-    link_1_b = page_1_b.find("a").get("href")
+            tmp_link = page_2_c.find("a").get("href")
+            link_2_c.append(tmp_link)
+            i += 1
   except:
+      pass
-    title_1_b = []
+  title_2_all = pd.DataFrame(title_2_c)
-    overview_1_b = []
+  overview_2_all = pd.DataFrame(overview_2_c)
-    link_1_b = []
+  link_2_all = pd.DataFrame(link_2_c)
+  page2_df = pd.concat([title_2_all, overview_2_all, link_2_all], axis=1)
+#page_3
+  result3 = requests.get(url3)
+  c3 = result3.content
+  soup3 = BeautifulSoup(c3)
-  title_1_c = []
+  title_3_c = []
-  overview_1_c = []
+  overview_3_c = []
-  link_1_c = []
+  link_3_c = []
   try: #cがある場合
-        for i in range(8):
+       for i in range(10):
-          page_1_c = soup1.find_all("div", {"class":"main-inner-c"})[i]
+            page_3_c = soup3.find_all("div", {"class":"main-inner-c"})[i]
-          tmp_title = page_1_c.find("h3").text.replace("\u3000","").replace("\n","")
+            tmp_title = page_3_c.find("h3").text.replace("\u3000","").replace("\n","")
-          title_1_c.append(tmp_title)
+            title_3_c.append(tmp_title)
-          tmp_overview = page_1_c.find_all("p")[2].text.replace("\u3000","").replace("\n","")
+            tmp_overview = page_3_c.find_all("p")[2].text.replace("\u3000","").replace("\n","")
-          overview_1_c.append(tmp_overview)
+            overview_3_c.append(tmp_overview)
-          tmp_link = page_1_c.find("a").get("href")
+            tmp_link = page_3_c.find("a").get("href")
-          link_1_c.append(tmp_link)
+            link_3_c.append(tmp_link)
-          i += 1
+            i += 1
   except:
-    pass
-  title_1_all = pd.DataFrame([title_1_a, title_1_b] + title_1_c)
-  overview_1_all = pd.DataFrame([overview_1_a, overview_1_b] + overview_1_c)
-  link_1_all = pd.DataFrame([link_1_a, link_1_b] + link_1_c)
-  page1_df = pd.concat([title_1_all, overview_1_all, link_1_all], axis=1)
-#2ページ目
-  result2 = requests.get(url2)
-  c2 = result2.content
-  soup2 = BeautifulSoup(c2)
-  title_2_c = []
-  overview_2_c = []
-  link_2_c = []
-  try: #cがある場合
-      for i in range(10):
-            page_2_c = soup2.find_all("div", {"class":"main-inner-c"})[i]
-            tmp_title = page_2_c.find("h3").text.replace("\u3000","").replace("\n","")
-            title_2_c.append(tmp_title)
-            tmp_overview = page_2_c.find_all("p")[2].text.replace("\u3000","").replace("\n","")
-            overview_2_c.append(tmp_overview)
-            tmp_link = page_2_c.find("a").get("href")
-            link_2_c.append(tmp_link)
-            i += 1
-  except:
       pass
-  title_2_all = pd.DataFrame(title_2_c)
-  overview_2_all = pd.DataFrame(overview_2_c)
-  link_2_all = pd.DataFrame(link_2_c)
-  page2_df = pd.concat([title_2_all, overview_2_all, link_2_all], axis=1)
-#3ページ目
-  result3 = requests.get(url3)
-  c3 = result3.content
-  soup3 = BeautifulSoup(c3)
-  title_3_c = []
-  overview_3_c = []
-  link_3_c = []
-  try: #cがある場合
-       for i in range(10):
-            page_3_c = soup3.find_all("div", {"class":"main-inner-c"})[i]
-            tmp_title = page_3_c.find("h3").text.replace("\u3000","").replace("\n","")
-            title_3_c.append(tmp_title)
-            tmp_overview = page_3_c.find_all("p")[2].text.replace("\u3000","").replace("\n","")
-            overview_3_c.append(tmp_overview)
-            tmp_link = page_3_c.find("a").get("href")
-            link_3_c.append(tmp_link)
-            i += 1
-  except:
-      pass
   title_3_all = pd.DataFrame(title_3_c)
@@ -384,7 +388,7 @@
-#1～3ページ統合
+#total
@@ -392,6 +396,8 @@
   page_df.columns=["Title","Overview","Link"]
+  page_df.index = np.arange(1, len(page_df)+1)
   page_df.to_excel("result.xlsx")

コード修正

2021/02/24 06:52

投稿

SMRKW

スコア2

test CHANGED Viewed

File without changes

test CHANGED Viewed

@@ -428,7 +428,7 @@
 url2 = "https://runda.jp/search/?q=" + keyword + "&page=2&search_type=" + period
-url3 = "https://runda.jp/search/?q=" + keyword + "&page=2&search_type=" + period
+url3 = "https://runda.jp/search/?q=" + keyword + "&page=3&search_type=" + period

コード修正

2021/02/24 05:32

投稿

SMRKW

スコア2

test CHANGED Viewed

File without changes

test CHANGED Viewed

@@ -140,8 +140,6 @@
-#関数の定義
 ```python

コード修正

2021/02/24 05:28

投稿

SMRKW

スコア2

test CHANGED Viewed

File without changes

test CHANGED Viewed

@@ -140,14 +140,18 @@
+#関数の定義
+```python
 def scrayping():
+#1ページ目
   result1 = requests.get(url1)
   c1 = result1.content
@@ -250,6 +254,8 @@
+#2ページ目
   result2 = requests.get(url2)
@@ -314,6 +320,8 @@
+#3ページ目
   result3 = requests.get(url3)
@@ -378,7 +386,7 @@
-#page1~3統合
+#1～3ページ統合
@@ -442,6 +450,10 @@
+```

コードの修正

2021/02/24 05:27

投稿

SMRKW

スコア2

test CHANGED Viewed

File without changes

test CHANGED Viewed

@@ -148,10 +148,6 @@
-#page_1
   result1 = requests.get(url1)
   c1 = result1.content
@@ -254,8 +250,6 @@
-#page_2
   result2 = requests.get(url2)
@@ -320,8 +314,6 @@
-#page_3
   result3 = requests.get(url3)

誤字修正

2021/02/24 05:16

投稿

SMRKW

スコア2

test CHANGED Viewed

	@@ -1 +1 @@
1	- Python scheduleモジュール実行時のエラー
1	+ Python Webスクレイピングにおけるscheduleモジュール実行時のエラー

test CHANGED Viewed

File without changes