Why blocks parsing?

Question

Python

Good day
I did the site parsing:

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import csv
import os
from threading import *
import requests
from concurrent.futures import ThreadPoolExecutor, wait
import time

firefox_profile = webdriver.FirefoxProfile()
firefox_profile.set_preference("thatoneguydotnet.QuickJava.startupStatus.Images", 2)  ## Turns images off
firefox_profile.set_preference("thatoneguydotnet.QuickJava.startupStatus.AnimatedImage", 2)  ## Turns animated

URL = 'https://novgorod.shop.megafon.ru/mobile'
HEDARS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}
FILE = 'meg.csv'
direct= os.getcwd()
def get_himl(url):
    driver = (direct + '/geckodriver')
    option = webdriver.FirefoxOptions()
    option.set_preference('dom.webdriver.enabled', False)
    #option.headless = True
    browser = webdriver.Firefox(executable_path=driver, options=option)
    time.sleep(3)
    browser.get(url)
    time.sleep(5)
    r = browser.page_source
    time.sleep(3)
    browser.quit()
    return r

def seve_file(item,path):
    with open(path, 'w', newline='',encoding='utf-8') as file:
        writer = csv.writer(file, delimiter = ';')
        writer.writerow(['бренд', 'модель', 'цена', 'акции'])
        for items in item:
            writer.writerow([items['brand'], items['title'], items['prise'], items['sale']])


def multiple_replace(target_str, replace_values):
    for i, j in replace_values.items():
        target_str = target_str.replace(i, j)
    return target_str

def get_content(html):
    soup = BeautifulSoup(html, 'html.parser')
    items = soup.find_all(class_='b-goods-list__item')
    phone = []
    for items in items:
        prise = items.find('span', class_='b-price-good-list__value b-price__value')
        if prise:
            prise = prise.get_text()
        else:
            prise = ''
        action = items.find_all('div', class_='c-actionTip__wrapper c-actionTip__text c-actionTip__text--small')
        sale = []
        if action:
            for action in action:
                sale += action
            sale = (' '.join(map(str,sale)))
            replace_values = {'<br/>': '','<p class="c-actionTip__price">':'', '\n':'','<p>':'','</p>':'','VoLTE ':''}
            sale = multiple_replace(sale, replace_values)
        else:
            sale = ''
        name = items.find('a', class_='b-good__title-link').get_text()
        name = name.split()
        name = ' '.join(name[0:-1]).lower()
        brand = name.split()[0]
        phone.append({
            'brand' : brand,
            'title' : name,
            'prise' : prise,
            'sale' : sale
        })
    return (phone)

def pars():
    for i in range(1,2):
        print(f'анаизруем {i} страницу ')
        a = (URL + '?page=' + str(i) + '&archVal=0')
        html = get_himl(a)
        phone.extend(get_content(html))
        seve_file(phone, FILE)
        print(f'найдено ' + str(len(phone)) + ' телефонов')

if __name__ == "__main__":
    phone = []
    pars()

But when I open the page, I get a message:

Access Denied
You don't have permission to access "/mobile?page=1&archVal=0" on this server.
Reference ID: 6d3f093e-e839-421f-b93e-94e3419d86e7
Description: 42

Access Denied Can

anyone tell me how to bypass the protection?

Answer 1

How to bypass selenium blocking by website?
if it didn't help
https://stackoverflow.com/questions/33225947/can-a...