A
A
Andrew2019-01-24 23:30:24
Python
Andrew, 2019-01-24 23:30:24

Selenium python 3 substitutes other values ​​in the proxy, what should I do?

There will be two questions :)
Through the Pool I do a mass launch of browsers, in python3 , selenium - chromedriver.
I substitute a proxy with authorization (via a plugin). I first look at which proxy I took (print), but it actually loads a completely different proxy. What could be the reason? Or is it " selenium thread safe "?
And the whole problem is that there is no definite dependence: it can open 8 out of 8 unique ones, or it can open 2-7 identical ones.
The code:

#!/usr/bin/python3
from multiprocessing import Process, Pool, Queue, freeze_support
from multiprocessing.pool import ThreadPool

from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
import time, random
import re, os, requests

from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
import zipfile, subprocess

def brows(accs):
            time.sleep(random.randint(2,5))
            print(accs)
            def bigproxy():
                manifest_json = """
                    {
                    "version": "1.0.0",
                    "manifest_version": 2,
                    "name": "Chrome Proxy",
                    "permissions": [
                        "proxy",
                        "tabs",
                        "unlimitedStorage",
                        "storage",
                        "<all_urls>",
                        "webRequest",
                        "webRequestBlocking"
                    ],
                    "background": {
                        "scripts": ["background.js"]
                    },
                    "minimum_chrome_version":"22.0.0"
                }
                """
                
                background_js = """    
                var config = {
                    mode: "fixed_servers",
                    rules: {
                      singleProxy: {
                        scheme: "http",
                        host: """+'"'+ipadr+'"'+""", 
                        port: parseInt("""+port_adr+""")
                      },
                      bypassList: ["bdseo.ru"]
                    }
                  };

                chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});

                function callbackFn(details) {
                    return {
                        authCredentials: {
                            username: """+'"'+login_pr+'"'+""",
                            password: """+'"'+pass_pr+'"'+"""
                        }    
                    };
                }

                chrome.webRequest.onAuthRequired.addListener(
                        callbackFn,
                        {urls: ["<all_urls>"]},
                        ['blocking']
                );
                """
                pluginfile = os.getcwd() + '/plugins/' + 'proxy_auth_plugin.zip'

                with zipfile.ZipFile(pluginfile, 'w') as zp:
                    zp.writestr("manifest.json", manifest_json)
                    zp.writestr("background.js", background_js)

                global co
                co = Options()
                co.add_argument("--start-maximized")    
                userag = 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_4_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/11.0 Mobile/15E148 Safari/604.1'
                userag = common_user[random.randint(0, len(common_user) - 1)].replace('\n', '')
                co.add_argument("user-agent=" + userag) #добавляем user agent    
                for plug in os.listdir(os.getcwd() + '/plugins'):
                    co.add_extension(os.getcwd() + '/plugins/' + plug) #'multilogin.zip')
                    
            def anyheaders():
                        global common_user
                        with open(os.getcwd() + '//common_user.txt', 'r') as hed_te: #часто используемые юзер-агенты
                                common_user = hed_te.readlines()  
            anyheaders()
            
            global  login_pr, pass_pr, ipadr, port_adr
            aco = accs.split('@')        
            username_p = aco[0].split(':')[0]
            pass_p = aco[0].split(':')[1]
            proxy = {'address': aco[1],
             'username': username_p,
             'password': pass_p}
            
            ipadr = aco[1].split(':')[0]
            port_adr = aco[1].split(':')[1]
            login_pr = proxy['username']
            pass_pr = proxy['password']
        
            bigproxy()
            global browser        
            browser = webdriver.Chrome(executable_path = os.getcwd() + '//settings/chromedriver.exe', chrome_options=co)
            
            url = 'http://2ip.ru'
            browser.get(url)
           

            #browser.quit()
            time.sleep(1)

def main2():                  
        def proxylist():
                        global proxys
                        with open(os.getcwd() + '//proxy.txt') as fg:
                                proxys = fg.readlines()                       
        proxylist()        
        try:
            time.sleep(2.5)
            pol = Pool(len(proxys))
            output = pol.map(brows, proxys)
            for an in output:                
                print('.')
        except Exception as erro:
            print(erro)
            pass  
if __name__ == '__main__':
        freeze_support()        
        try:        
            main2()
        except Exception as eror:
            print(eror)
            time.sleep(20)

The second question:
I do it through the for loop, without multiprocessing, and so the browser opens, loads the page, and then closes it after some time, although there is no "request" for quit ().
Code (everything is the same except for the main2() function:
def main2():                  
        def proxylist():
                        global proxys
                        with open(os.getcwd() + '//proxy.txt') as fg:
                                proxys = fg.readlines()                       
        proxylist()        
        try:
            for proxy in proxys:
                  brows(proxy)
        except Exception as erro:
            print(erro)
            pass

There are no errors, at most at the start of the first chromedriver Warning pops up, but everything works "according to the script".
Thank you. I spent 3 days, I can not pick it up.

Answer the question

In order to leave comments, you need to log in

Didn't find what you were looking for?

Ask your question

Ask a Question

731 491 924 answers to any question