Ruby 複数ページからのスクレイピング

Question

■分からないこと

複数ページからのスクレイピングのやり方を教えていただきたいです。
下記の2つのコードで「list_1」がベースになり、会社名を紐づけて「list_2」のEmailを取得したいと思っています。
list_2のURLの"https://job.rikunabi.com/2019/company/r294900083/"の最後の「r + ９桁数字」もランダムな数字になっているので、そこをどう書いていいのかも分かりません。

よろしくお願い致します。

■list_1

require 'open-uri'
require 'nokogiri'
require 'csv'
require 'byebug'

def get_data(uri, companies)
  html = open(uri).read
  documents = Nokogiri::HTML(html)
  documents.xpath("//a[@class='ts-h-search-cassetteTitleMain js-h-search-cassetteTitleMain']").each {|n| companies << n.text.strip}
  return companies
end

def main()
  uri = "https://job.rikunabi.com/2019/s/__13_0_______/"
  puts "What is the page? "
  page = gets.to_i

  companies = []
  data = get_data(uri,companies)
  (2..page).to_a.each do |idx|
    uri = "https://job.rikunabi.com/2019/s/__13_0_______/?moduleCd=2&isc=ps054&pn=#{idx}"
    data = get_data(uri,companies)
  end

  len = [companies.size].min - 1

  headers = ["会社名"]
  time = Time.new.strftime("%Y-%m-%d")
  CSV.open("rikunabi_tokyo_2019-#{time}.csv", "a",headers: headers, write_headers: true) do |csv|
    (0..len).to_a.each do |idx|
      csv_column_values = [companies[idx]]
      csv << csv_column_values
    end
  end
end

if __FILE__ == $0
  puts("Process Start")
  main()
  puts("Process Finished")
end

■list_2

require 'open-uri'
require 'nokogiri'
require 'csv'
require 'byebug'
require 'kconv'

def get_data(uri, data)
  companies = data[0]
  parameters = data[1]
  html = open(uri).read
  documents = Nokogiri::HTML(html.toutf8, nil, 'utf-8')
  companies << documents.xpath("//h1[@class='ts-h-company-mainTitle']").text
  parameters << documents.xpath("//div[@class='ts-h-company-sentence']")[1].text.strip.gsub(/(\r)/, " ")
  return [companies, parameters]
end

def main()
  uri = "https://job.rikunabi.com/2019/company/r294900083/"
  puts "What is the page? "
  page = gets.to_i

  companies = []
  parameters = []
  data = [companies, parameters]
  data = get_data(uri,data)
  (2..page).to_a.each do |idx|
    uri = "https://job.rikunabi.com/2019/company/r294900083/"
    data = get_data(uri,data)
  end

  len = [companies.size,parameters.size].min - 1

  headers = ["会社名","Email"]
  time = Time.new.strftime("%Y-%m-%d")
  CSV.open("rikunabi_tokyo_2019-#{time}.csv", "a",headers: headers, write_headers: true) do |csv|
    (0..len).to_a.each do |idx|
      csv_column_values = [companies[idx], parameters[idx]]
      csv << csv_column_values
    end
  end
end

if __FILE__ == $0
  puts("Process Start")
  main()
  puts("Process Finished")
end

Answer 1

require 'open-uri'
require 'nokogiri'

main_url = 'https://job.rikunabi.com/2019/s/__13_0_______/'

1.upto(84) do |i|
  doc = Nokogiri::HTML.parse(open("#{main_url}?moduleCd=100&isc=ps054&pn=#{i}"))

  doc.xpath('//*[contains(@id,"cassette-r")]/div[1]/div/a').each do |a|
    com_page = Nokogiri::HTML.parse(open("https://job.rikunabi.com#{a.xpath('@href').text}"))
    puts a.text
    puts com_page.xpath('//*[@id="company-data04"]/div').inner_html.gsub('<br>', "\n")
    puts "\n" + '-' * 80
    sleep(rand(3..5))
  end
end

Comments

[Komama]

ありがとうございます！
取得できました！
コードも書き方も非常に勉強になります！
しっかり理解して次に活かしたいと思います。
本当にありがとうございました！