爬蟲 & Python

from bs4 import BeautifulSoup
import requests

# fetch url
fetchUrl = "https://baito.mynavi.jp/tokyo/ss-1/ssm-1/sss-1/kd-61/"
r = requests.get(fetchUrl)

# new BeautifulSoup
soup = BeautifulSoup(r.text, "html.parser")

# 用 css selector 取得 element
element = soup.select(".searchResult > .researchCount")
print(element)

from bs4 import BeautifulSoup
import time
from selenium import webdriver
from selenium.webdriver.common.by import By

# 啟動 driver
option = webdriver.ChromeOptions()

# 背景開啟瀏覽器
# option.add_argument('headless')

# 開啟瀏覽器
driver = webdriver.Chrome(option)

# fetch url
fetchUrl = "https://baito.mynavi.jp/cl-002280403865/job-94305516/?reserve_w_29450608"
driver.get(fetchUrl)

# 等待頁面渲染
time.sleep(1)

# tab 会社情報
button = driver.find_element(By.CSS_SELECTOR, ".tab02")
button.click()
time.sleep(3)

# 取得 dom
html = driver.page_source
soup = BeautifulSoup(html, 'html.parser')
address = soup.select(".workPlaceAddressDiamondMark")
for a in address:
    print(a.text)

import pymysql

db = pymysql.connect(host='',
                     user='',
                     password='',
                     database='')

cursor = db.cursor()

targetUrl = [
    {'url': 'https://job-medley.com/est/pref13/employment3/', 'number': 200},
    {'url': 'https://job-medley.com/ans/pref14/employment3/', 'number': 132},
    {'url': 'https://job-medley.com/ans/pref27/employment3/', 'number': 126},
    ...
]

from urlList import melody
import requests
from bs4 import BeautifulSoup

for idx, list in enumerate(melody.targetUrl):
    page = 1
    count = 0
    while count < list['number']:
        # 從第 1 頁開始 (https://job-medley.com/est/pref13/employment3/?page=2)
        fetchUrl = list['url'] if page == 1 else list['url'] + "?page=" + str(page)
        
        # request 之前 sleep 1~10 秒
        time.sleep(random.randint(1, 10))
        r = requests.get(fetchUrl)

        # new BeautifulSoup
        soup = BeautifulSoup(r.text, "html.parser")

        # 取得一覽頁總件數
        shopCount = soup.select(".c-search-condition-title__important")[0].text

        # 取得資訊
        shopBlock = soup.select(".c-offers-sort-area .c-offers-sort-area__contents .o-gutter-row .o-gutter-row__item")
        for sb in shopBlock:
            data = {
                "job_url": sb.select(".job-url"), # 店鋪詳細 url
                "station": sb.select(".station")  # 店鋪所在地
            }

            # 儲存資料庫
            insert_mysql(fetchUrl, data)
    
            # 計算筆數
            count += 1

            # 若滿足件數，繼續撈取下一個一覽頁
            if count >= list['number']:
                break

        # 若已達一覽頁總件數，繼續撈取下一個一覽頁
        if count >= int(shopCount):
            break

import requests
from bs4 import BeautifulSoup

def select_jobs():
    sql = ("SELECT id, job_url, station, is_requested," +
           "CASE WHEN station LIKE '% 尼崎駅%' THEN 1 " +
           "     WHEN station LIKE '% 明石駅%' THEN 2 " +
           "     WHEN station LIKE '% 高松駅%' THEN 3 " +
           " FROM `melody`" +
           " WHERE is_requested = 0" +
           " ORDER BY ordering, id ASC" +
           " LIMIT 100")
           
def update_sql(id):
    try:
        sql = "UPDATE melody SET is_requested = 1 WHERE id = " + str(id)
        cursor.execute(sql)
        db.commit()
    except:
        write_log('sql update error: melody id - ' + str(id))
        db.rollback()

############################## main ##############################
jobs = select_jobs()
for job in jobs:
    #取得資訊
    fetchUrl = job['job_url']
    r = requests.get(fetchUrl, headers={'user-agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'})
    soup = BeautifulSoup(r.text, "html.parser")
    ldJson = soup.select('script[type="application/ld+json"]')
    ...
    
    # 儲存資料庫
    insert_mysql(fetchUrl, data)
    
    # 更新資料表 is_requested = 1
    update_sql(job['id'])

import googlemaps

gmaps = googlemaps.Client(key="[MY_TOKEN]")
search_result = gmaps.places(facilityName)
tel = ""
if len(search_result['results']) > 0:
    # 先找到 <位置ID>
    place_id = search_result['results'][0]['place_id']

    # 用 <位置ID> 找店家詳細
    place_details = gmaps.place(place_id=place_id, fields=['name', 'formatted_phone_number'])
    tel = place_details['result'].get('formatted_phone_number')

    # 印出 tel
    print(tel)

r = requests.get(fetchUrl, headers={'user-agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'})