import logging
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
try:
# Set Chrome options to headless
chrome_options = Options()
chrome_options.add_argument("--headless")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--window-size=1920,1200")
chrome_options.add_argument("--ignore-certificate-errors")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
# Initialize the Chrome driver
logger.info("Initializing Chrome WebDriver")
service = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=service, options=chrome_options)
if driver is None:
raise Exception("Failed to initialize WebDriver")
logger.info("Navigating to the webpage")
driver.get("https://www.nayaraenergy.com/petrol-pump-near-me")
# Ensure the page is fully loaded
driver.implicitly_wait(10) # Wait for up to 10 seconds for the elements to appear
# Get the cookies
logger.info("Retrieving cookies")
cookies = driver.get_cookies()
session_data = {}
# Print the cookies
for cookie in cookies:
print(cookie)
data = {
cookie['name']: cookie['value']
}
session_data.update(data)
print(session_data)
except Exception as e:
logger.error(f"An error occurred: {e}", exc_info=True)
finally:
if driver is not None:
logger.info("Closing the driver")
driver.quit()
I am trying to get the cookie information of the page –
The above code works to a certain extent, but fails to output all the cookie values
the whole data of the cookie is –
_fbp=fb.1.1722150867062.324436436556492225; _
hjSessionUser_3422175=some_data;
_gid=GA1.2.353944606.1722412003;
_hjSession_3422175=some_data
; _ga_6EDN8NZ3RQ=GS1.1.1722433679.6.0.1722433679.60.0.0; _ga_5WCBPP0WF1=GS1.1.1722433679.7.0.1722433679.0.0.0; _ga=GA1.2.1174399862.1722150866; _gat_gtag_UA_142048367_1=1; XSRF-TOKEN=eyJpdiI6IlA0MWIxZHR1WXZvaHBFZHhtU3E2dVE9PSIsInZhbHVlIjoieDV3ZVZRdEpxRWRreUVrWEZQQ0lUWEQ3dGRWUnhLa3JidjR5eGxIc3BVbGl3REZYWmN1SDdIb0dVdTZyN1lDTSIsIm1hYyI6IjgwNmExMTE4MDFkMTkyZGVlYmZkMzVlYjMyYTNhMzg0ODNiNDdmMjdiZTFkNmRlOTYxYzAxMWUwN2Q4YzgwNzAifQ%3D%3D; laravel_session=eyJpdiI6IjVoYkNuOG5hbXF3bVZZd0VGanpHNmc9PSIsInZhbHVlIjoiNm8yaGVXdWllbTQxWGJ1T3p5QkQ4Tmc1QXlhT2E2UG1KOUgyQnF5dkFkVHV4QWZobklGOEF1dG5weGNBNUdZdiIsIm1hYyI6IjljOWQ3MGI4MjgyM2Q1MDZiM2JjN2M1OTcxZWIxMDk0MzgyMTc3MTQ3ZDg1ODExNDdmY2EwN2JkMjJmNmY2Y2YifQ%3D%3D
This session has already expired, so no point trying the tokens
but when i use the get_cookies()
function, it doesn’t outputs the hjSessionUser_3422175 and hjSession_3422175
Chatgpt was of no use, I tried finding the get_cookie function in their github repo to understand the workings, but can’t seem to find it, guess it’s built in in the webdriver itself.