Selenium HTTPリクエストのURLを記録する（Chrome, Python）

  • Selenium 4.9.0
  • Chrome 112
  • ChromeDriver 112.0.5615.49
  • Python 3.11
import time
import json
from selenium.webdriver import (
    Chrome,
    DesiredCapabilities,
)




desired_capabilities = DesiredCapabilities.CHROME
desired_capabilities["goog:loggingPrefs"] = {
    "performance": "ALL",
}


driver = Chrome(
    desired_capabilities=desired_capabilities,
)
driver.implicitly_wait(5)


driver.get("https://www.google.com/")


known_url_set = set()


while True:
    performance_log_entries = driver.get_log("performance")


    for log_entry in performance_log_entries:
        log_message = json.loads(log_entry.get("message", "{}")).get("message", {})


        method = log_message.get("method")
        params = log_message.get("params", {})


        if method == "Network.responseReceived":
            response = params.get("response", {})
            url = response.get("url")


            if url in known_url_set:
                continue
            known_url_set.add(url)


            print(url)


    time.sleep(1)