https://www.canparkavm.com.tr/tr/markalar
As i said at title, how can i access/convert JavaScript variable as JSON value on Python using the BeautifulSoup and requests libraries?
I tried this topic for solve my problem but did not worked actually: Extract JSON from HTML Script tag with BeautifulSoup in Python
from bs4 import BeautifulSoup
import requests, json
from subprocess import check_output
def main():
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
url = "https://www.canparkavm.com.tr/tr/markalar"
req = requests.get(url,headers=headers)
bs4 = BeautifulSoup(req.content,"lxml")
data = bs4.find("body").find("script").find_next("script")
with open('temp.js','w') as f:
f.write('stores = {};n'+data.text.strip()+'nprocess.stdout.write(JSON.stringify(stores));')
window_init_state = check_output(['node','temp.js'])
print(json.loads(window_init_state))
main()
Script tag is inside of HTML.
This is script tag which i tried to scrap JSON value from stores variable:
<script>
var language = "tr";
var stores = [
{
"store_categories": [
{
"store_category_tr_tags": [],
"store_category_en_tags": [],
"_id": "4yc8Q95d-",
"store_category_name_tr": "Yeme & İçme",
"store_category_name_en": "Food & Beverage",
"store_category_mallid": "NkYFwYyO-"
}
],
"store_tags": [],
"store_locations": [
{
"location_pin": {
"z": 0,
"x": 7475,
"y": 4043
},
"location_serviceids": [],
"location_belongs_to_nodes": [
"B1ZlzrSdA0F"
],
"_id": "EJ1xCinc-OZ",
"location_id": "K0_store_43",
"location_mallid": "NkYFwYyO-",
"location_floorid": {
"floor_number": 0,
"_id": "4Jk1uMxdW",
"floor_name_tr": "Zemin Kat",
"floor_name_en": "Ground Floor"
},
"location_storeid": "NJgYCfWg_-"
}
],
"store_promotions": [
"41yxPW1hB"
],
"store_will_use_big_logo": false,
"_id": "NJgYCfWg_-",
"store_slug": "alacati-muhallebicisi",
"store_mallid": "NkYFwYyO-",
"store_brandid": {
"_id": "4ym42U_JOb",
"brand_id": 2271,
"brand_name": "Alaçatı Muhallebici",
"brand_logo": "https://d3heiv85u05n2u.cloudfront.net/images/brands/2271_low.jpg?1429088370",
"brand_web": "http://www.alacatimuhallebicisi.com",
"brand_whitescale_logo": "https://d3heiv85u05n2u.cloudfront.net/images/brands/2271_whitescale_low.png?1482761569",
"brand_logo_high": "https://d3heiv85u05n2u.cloudfront.net/images/brands/2271_original.jpg?1429088370",
"brand_whitescale_logo_high": "https://d3heiv85u05n2u.cloudfront.net/images/brands/2271_whitescale_original.png?1482761569"
},
"store_name": "ALAÇATI MUHALLEBİCİSİ",
"store_email": "",
"store_web": "http://www.alacatimuhallebicisi.com",
"store_phone": "0216 504 51 18",
"store_description_tr": "",
"store_description_en": "",
"store_primary_image": {
"main": "https://s3-eu-central-1.amazonaws.com/klepierre/images/store_primary_image/NJgYCfWg_--main.jpeg?1470234948279"
}
},
{
"store_categories": [
{
"store_category_tr_tags": [],
"store_category_en_tags": [],
"_id": "NkSYmq9db",
"store_category_name_tr": "Sinema & Eğlence",
"store_category_name_en": "Fun",
"store_category_mallid": "NkYFwYyO-"
}
],
"store_tags": [],
"store_locations": [
{
"location_pin": {
"z": 3,
"x": 1832,
"y": 3989
},
"location_serviceids": [],
"location_belongs_to_nodes": [
"HkBHMHBORCY"
],
"_id": "Vyy4Cj29Zdb",
"location_id": "K3_store_123q124",
"location_mallid": "NkYFwYyO-",
"location_floorid": {
"floor_number": 3,
"_id": "V1bjvfxuZ",
"floor_name_tr": "Kat 3",
"floor_name_en": "Floor 3"
},
"location_storeid": "VJWEJQWxdZ"
},
{
"location_pin": {
"z": 3,
"x": 9885,
"y": 3872
},
"location_serviceids": [],
"location_belongs_to_nodes": [
"SkVIGSB_R0F"
],
"_id": "NyJS0o25b_Z",
"location_id": "K3_store_136q138",
"location_mallid": "NkYFwYyO-",
"location_floorid": {
"floor_number": 3,
"_id": "V1bjvfxuZ",
"floor_name_tr": "Kat 3",
"floor_name_en": "Floor 3"
},
"location_storeid": "VJWEJQWxdZ"
}
],
"store_promotions": [
"EkC2w99hS",
"r1X-WHouN"
],
"store_will_use_big_logo": false,
"_id": "VJWEJQWxdZ",
"store_slug": "atlantis-fun-city",
"store_mallid": "NkYFwYyO-",
"store_brandid": {
"_id": "ByPu9oyvB",
"brand_id": 5431,
"brand_name": "Atlantis Fun City",
"brand_logo": "https://ams3.digitaloceanspaces.com/cinebrand/images/brand/brand_logo/6d1a86e9-3d27-4dfa-9318-dc8855dcf839.png?1568811042923",
"brand_logo_high": "https://ams3.digitaloceanspaces.com/cinebrand/images/brand/brand_logo/0fb05193-2af0-4607-be17-a965d1b509c5.png?1568811042667"
},
"store_name": "ATLANTİS FUN CITY",
"store_email": "[email protected]",
"store_web": "",
"store_phone": "0544 258 34 22 ",
"store_description_tr": "",
"store_description_en": "",
"store_primary_image": {
"main": "https://s3-eu-west-1.amazonaws.com/canpark/images/store_primary_image/VJWEJQWxdZ-main.jpeg?1560243207859"
}
},
{
"store_categories": [
{
"store_category_tr_tags": [],
"store_category_en_tags": [],
"_id": "Eyvibc5uZ",
"store_category_name_tr": "Hazır Giyim",
"store_category_name_en": "Clothing",
"store_category_mallid": "NkYFwYyO-"
}
],
"store_tags": [],
"store_locations": [
{
"location_pin": {
"z": 1,
"x": 8974,
"y": 3054
},
"location_serviceids": [],
"location_belongs_to_nodes": [
"HJUfGrrOR0F"
],
"_id": "EkhZAi3qZ_b",
"location_id": "K1_store_79",
"location_mallid": "NkYFwYyO-",
"location_floorid": {
"floor_number": 1,
"_id": "VkeCvMxOZ",
"floor_name_tr": "Kat 1",
"floor_name_en": "Floor 1"
},
"location_storeid": "Vk7jAMZxOW"
}
],
"store_promotions": [
"Sy8FFoc2B",
"HywcrnjRH"
],
"store_will_use_big_logo": false,
"_id": "Vk7jAMZxOW",
"store_slug": "aker",
"store_mallid": "NkYFwYyO-",
"store_brandid": {
"_id": "4Jzz28OJub",
"brand_id": 535,
"brand_name": "Aker",
"brand_logo": "https://d3heiv85u05n2u.cloudfront.net/images/brands/535_low.jpg?1376916442",
"brand_web": "http://www.aker.com.tr",
"brand_whitescale_logo": "https://d3heiv85u05n2u.cloudfront.net/images/brands/535_whitescale_low.png?1472804453",
"brand_logo_high": "https://d3heiv85u05n2u.cloudfront.net/images/brands/535_original.jpg?1376916442",
"brand_whitescale_logo_high": "https://d3heiv85u05n2u.cloudfront.net/images/brands/535_whitescale_original.png?1472804453"
},
"store_name": "Aker",
"store_email": "[email protected]",
"store_web": "http://www.aker.com.tr",
"store_phone": "0216 999 37 83",
"store_description_tr": "",
"store_description_en": "",
"store_primary_image": {
"main": "https://s3-eu-central-1.amazonaws.com/klepierre/images/store_primary_image/Vk7jAMZxOW-main.jpeg?1470234918440"
}
},
{
"store_categories": [
{
"store_category_tr_tags": [],
"store_category_en_tags": [],
"_id": "Eyvibc5uZ",
"store_category_name_tr": "Hazır Giyim",
"store_category_name_en": "Clothing",
"store_category_mallid": "NkYFwYyO-"
}
],
"store_tags": [],
"store_locations": [
{
"location_pin": {
"z": 1,
"x": 4842,
"y": 3029
},
"location_serviceids": [],
"location_belongs_to_nodes": [
"ByK-GHSdCCK"
],
"_id": "4kz-Co2cWuZ",
"location_id": "K1_store_86",
"location_mallid": "NkYFwYyO-",
"location_floorid": {
"floor_number": 1,
"_id": "VkeCvMxOZ",
"floor_name_tr": "Kat 1",
"floor_name_en": "Floor 1"
},
"location_storeid": "EkNoCfZlub"
}
],
"store_promotions": [
"VyLvwZy2r"
],
"store_will_use_big_logo": false,
"_id": "EkNoCfZlub",
"store_slug": "altinyildiz-classics",
"store_mallid": "NkYFwYyO-",
"store_brandid": {
"_id": "V1nYjVTsN",
"brand_id": 4363,
"brand_name": "Altınyıldız Classics",
"brand_logo": "https://d3heiv85u05n2u.cloudfront.net/images/brands/4363_low.jpg?1521447270",
"brand_logo_high": "https://d3heiv85u05n2u.cloudfront.net/images/brands/4363_original.jpg?1521447270",
"brand_whitescale_logo": "https://ams3.digitaloceanspaces.com/cinebrand/images/brand/brand_whitescale_logo/ae8e1fff-241b-4046-9943-be8ffa704cec.png?1559033647723",
"brand_whitescale_logo_high": "https://ams3.digitaloceanspaces.com/cinebrand/images/brand/brand_whitescale_logo/8008a25a-95a3-4da8-8b56-90340ccadfd0.png?1559033633724",
"brand_web": ""
},
"store_name": "Altınyıldız Classics",
"store_email": "[email protected]",
"store_web": "",
"store_phone": "0216 519 16 06",
"store_description_tr": "",
"store_description_en": "",
"store_primary_image": {
"main": "https://s3-eu-west-1.amazonaws.com/canpark/images/store_primary_image/EkNoCfZlub-main.jpeg?1559032279151"
}
},
{
"store_categories": [
{
"store_category_tr_tags": [],
"store_category_en_tags": [],
"_id": "4yc8Q95d-",
"store_category_name_tr": "Yeme & İçme",
"store_category_name_en": "Food & Beverage",
"store_category_mallid": "NkYFwYyO-"
}
],
"store_tags": [],
"store_locations": [
{
"location_pin": {
"z": 3,
"x": 2274,
"y": 4510
},
"location_serviceids": [],
"location_belongs_to_nodes": [
"rkESfSBu0RY"
],
"_id": "41R7Rs3cbOb",
"location_id": "K3_store_125",
"location_mallid": "NkYFwYyO-",
"location_floorid": {
"floor_number": 3,
"_id": "V1bjvfxuZ",
"floor_name_tr": "Kat 3",
"floor_name_en": "Floor 3"
},
"location_storeid": "EkNq8ZwTZ"
}
],
"store_promotions": [],
"store_will_use_big_logo": false,
"_id": "EkNq8ZwTZ",
"store_slug": "arden",
"store_brandid": {
"_id": "NJpvrWvpW",
"brand_id": 3433,
"brand_name": "Arden",
"brand_logo": "http…</script>