S
S
SHADRIN2019-12-25 16:13:01
Python
SHADRIN, 2019-12-25 16:13:01

Problem with multiprocessing processes, what to do?

class First(Thread):
    def __init__(self, Links, Potok, numbers):
        Thread.__init__(self)
        self.Links = Links
        self.Potok = Potok
        self.numbers = numbers

    def run(self):
        LinksMP = self.MainPage(self.Links)
        Links = LinksMP
        proxy = self.ProxyList()

        procs = []
        name = codecs.open('name.txt', 'r', "utf_8_sig")
        name = name.read()
        name = name.split('\n')
        for index, number in enumerate(self.numbers):

            proc = Process(target=self.Multiproc, args=(Links, name, proxy,))
            procs.append(proc)
            proc.start()
        
        for proc in procs:
            proc.join()

    def ProxyList(self):
        #Тут парсит прокси убрал место занимает
        return proxy

    def MainPage(self, Links):
        #Парсит список ссылок
        return Links

    def Multiproc(self, Links, name, proxy):
        while True:
            global ValCicle
            if len(proxy) == 0:
                proxy = self.ProxyList()

            Linkes = 0
            for line in Links:
                Linkes += 1

            proxies = 0
            for line in proxy:
                proxies += 1

            names = 0
            for line in name:
                names += 1

            proxys = random.randrange(0, proxies)
            names = random.randrange(0, names)
            Linkss = random.randrange(0, Linkes)

            TT = random.randrange(25, 60)
            chrome_options = Options()  

            #chrome_options.add_argument("--headless") 
            chrome_options.add_argument("user-agent="+ name[names]) 
            chrome_options.add_argument('--proxy-server=' + proxy[proxys])
            driver = webdriver.Chrome(executable_path='chromedriver.exe', chrome_options=chrome_options)

            try:
                
                driver.get(Links[Linkss])
                time.sleep(5)
                driver.get(Links[Linkss])
                time.sleep(5)
                driver.get(Links[Linkss])
                time.sleep(TT)
                Down = driver.find_element_by_xpath('//div[@class="zen-comments"]').location_once_scrolled_into_view
                time.sleep(TT)
                ValCicle += 1
                VAL.configure(text=ValCicle) 
                driver.close()

            except:
                del proxy[proxys]
                driver.close()
                pass

This is the code itself, I shortened it a bit to make it easier
Traceback (most recent call last):
  File "C:\Users\Сергей\AppData\Local\Programs\Python\Python38-32\lib\threading.py", line 932, in _bootstrap_inner
    self.run()
  File "GUI.py", line 40, in run
    proc.start()
  File "C:\Users\Сергей\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\process.py", line 121, in start
    self._popen = self._Popen(self)
  File "C:\Users\Сергей\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\context.py", line 224, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "C:\Users\Сергей\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\context.py", line 326, in _Popen
    return Popen(process_obj)
  File "C:\Users\Сергей\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\popen_spawn_win32.py", line 93, in __init__
    reduction.dump(process_obj, to_child)
  File "C:\Users\Сергей\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle '_thread.lock' object
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Users\Сергей\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\spawn.py", line 107, in spawn_main
    new_handle = reduction.duplicate(pipe_handle,
  File "C:\Users\Сергей\AppData\Local\Programs\Python\Python38-32\lib\multiprocessing\reduction.py", line 79, in duplicate
    return _winapi.DuplicateHandle(
OSError: [WinError 6] Неверный дескриптор

The question is that you need to put locks, I don’t understand what it is, reading the dock, how not everything is about that, give advice.
If you remove it from Class, then everything works as usual, it does not give any errors

Answer the question

In order to leave comments, you need to log in

1 answer(s)
A
antonksa, 2019-12-25
@antonksa

Re-read pep8. It hurts my eyes to read

self.Links = Links
self.Potok = Potok
driver.get(Links[Linkss])

There are also all sorts of books, our evangelists, Grigory Petrov, for example, filmed a video "how to name variables so that they don't throw tomatoes at you."
And in general, coders on vend should suffer.

Didn't find what you were looking for?

Ask your question

Ask a Question

731 491 924 answers to any question