Python+webdriver visits a js-rendered web page why it cannot be fully rendered in headless
the following code:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
options = webdriver.ChromeOptions()
executable_path = "/Users/shurachow/Downloads/chromedriver"
-sharp options.add_argument("--disable-gpu")
-sharp options.add_argument("--headless")
options.add_argument("--hide-scrollbars")
mobileEmulation = {"deviceName": "iPhone 8"}
options.add_experimental_option("mobileEmulation", mobileEmulation)
options.add_argument("lang=zh_CN.UTF-8")
options.add_argument("Accept="application/json, text/javascript"")
prefs = {
"profile.default_content_setting_values": {
"images": 1,
"javascript": 1,
"geolocation": 1
}
}
options.add_experimental_option("prefs", prefs)
browser = webdriver.Chrome(executable_path=executable_path, chrome_options=options)
browser.get("https://daojia.jd.com")
browser.execute_script("""
var __s__script = document.createElement("script");
const scriptContent = `
localStorage.setItem("_current_address_", "{"city":"","longitude":114.30525,"latitude":30.59276,"areaCode":1381,"districtCode":1386,"address":"188","district":"","title":"","adcode":"420102","requestId":"92746b90-d3d0-11e8-8978-246e96285542","poiId":"","cityId":1381,"_hasLogin_":false}");
`;
__s__script.innerHTML = scriptContent;
document.head.prepend(__s__script)
""")
try:
WebDriverWait(browser, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".index-shop-wrap")))
print "good"
except:
print "bad"
browser.close()
if you open the comments of -- headless
, the
page cannot render .index-shop-wrap
, you will not get the result of print "good"
finally
on the contrary, in the case of chrome GUI, it is OK. Am I missing something?
look for advice.