webスクレイピングを複数回行いたい。

Question

###前提・実現したいこと
webページのスクレイピングを複数回行いたいです。

具体的には、コマンドプロンプト上でpython ~.py プロレス 1と実行すれば
グーグルでプロレスと検索したときに表示される1ページ目の10件すべてをスクレイピングし、テキストとして保存する。
上記の事を実現したいです。

###発生している問題・エラーメッセージ
何かしらの形式がおかしいため、発生しているのかなと思っています。

1
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'
Traceback (most recent call last):
  File "bs.py", line 69, in <module>
    soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "C:\Users\RYO\AppData\Local\Programs\Python\Python36\lib\urllib\request.p
y", line 518, in open
    protocol = req.type
AttributeError: 'HTTPResponse' object has no attribute 'type'

###該当のソースコード

# -*- coding: utf-8 -*-
#BeautifulSoupとutllibをインポート
#from bs4 import BeautifulSoup
import urllib.request
import urllib.parse
import bs4
from bs4 import NavigableString,Declaration,Comment,BeautifulSoup
#コマンドライン引数指定 (1つのみ)
import sys

import ssl
ssl._create_default_https_context = ssl._create_unverified_context

links = [] #ページにあるリンク10個を格納する
maintexts = [] #txtファイルに書き込む文章

def getNavigableStrings(soup):
  if isinstance(soup, NavigableString):
    if type(soup) not in (Comment, Declaration) and soup.strip():
      yield soup
  elif soup.name not in ('script', 'style'):
    for c in soup.contents:
      for g in getNavigableStrings(c):
        yield g
#html抽出関数
def htmlExtraction(url):
# HTML を取得
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    html = opener.open(url)
    soup = BeautifulSoup(html,"html.parser")

    for url in soup.find_all("h3",{"class":"r"}):
        links.extend([url.a.get("href")])

query = "" #query encode
queryary = [] #query word
querytxt = "" #txtファイル名
pagenum = int(sys.argv[len(sys.argv) - 1]) #ページ番号
page = "" #何ページ目を参照するか

for i in range(1,len(sys.argv) - 1):
    queryary.extend([urllib.parse.quote(sys.argv[i])]) #sys.argv[1]~sys.argv[max]
    querytxt = querytxt + sys.argv[i] + " "
    if i == 1:
        query = queryary[i-1]
    else:
        query = query + "+" + queryary[i-1]

querytxt = querytxt + sys.argv[len(sys.argv) - 1]

url1 = "https://www.google.co.jp/search?q="

if int(sys.argv[len(sys.argv) - 1]) >= 2: #最後の引数はページ数 1ページ目参照の場合不要
    page = "&start=" + str((pagenum - 1) * 10) #最後の引数が2なら11件目から20件目を参照
url = url1 + query + page

htmlExtraction(url)

linkcnt = 0
#本文抽出
for link in links:
    linkcnt += 1
    try:
        opener = urllib.request.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
        html = opener.open("https://www.google.co.jp" + link)
        #soup = BeautifulSoup(html, "html.parser")
        soup = bs4.BeautifulSoup(urllib.request.urlopen(html).read(),"html.parser")
        text = '\n'.join(getNavigableStrings(soup))

        maintexts.extend([text]) #配列に入れてく

    except urllib.error.URLError:
        print(str(linkcnt) + "page URLerror")
    except:
        import traceback
        traceback.print_exc()




#ファイルに書き込み w:上書き a:追記
import codecs
file_object= codecs.open(querytxt + ".txt", "wb", "cp932", "ignore")
file_object.write(str(maintexts) + "\n")
file_object.close()

###補足情報(言語/FW/ツール等のバージョンなど)
より詳細な情報

Comments

[_Victorique__]

htmlが何かを明記してください。

[_Victorique__]

失礼、書いてありましたね

Answer 1

二回urlopenしてますね。

soup = bs4.BeautifulSoup(html.read(),"html.parser")

これで十分です。

Comments

[hanpanai-tumi]

ご回答ありがとうございます。無事解決出来ました！