How to properly organize a multi-threaded console application in python?

Question

Python

I want to make a simple framework for a multi-threaded application that prints a string and sleeps at the right time. Here is an example code:

import threading
import thread
from threading import Lock
import sys
from random import randint
import time

my_lock = Lock()
class printer(threading.Thread):
    def __init__(self, string):
        threading.Thread.__init__(self)
        self.string = string
    #with my_lock:
    #   print string
    def run(self):
        sys.stdout.write(self.string + '\n')
        sleep_time = randint(1, 10)
        time.sleep(sleep_time)


max_threads = 5
f = open("strings.txt", "r")
data = f.readlines()
data_count = len(data)
print data_count
threads = []
i = 0
flag = True
curr_threads = 0

while(flag == True):


    if(i >= data_count):
        flag = False

    if(flag != False):
        curr_threads = len(threads)
        while(curr_threads < max_threads):
            print i
            data_string = data[i]
            t = printer(data_string) 
            threads.append(t)
            t.daemon = True
            t.start()
            i = i + 1
            if ( i >= data_count):
                break

    for thr in threads:
        t_id = str(thr.ident)
        print "checking " + t_id
        if thr.is_alive():
            t.join()
            print "handling"
            threads.remove(t)

But, firstly, for some reason, the main thread falls asleep, and secondly, I don’t know how to properly process running threads. I thought that I needed to go through all the running threads and if they are Alive then do a join and remove it from the list, and so on. But join blocks the main program, so if some thread "sleeps" for a long time, then this is unproductive, and I can't remove it from the list: ValueError: list.remove(x): x not in list. How to correct the situation?

Answer 1

import sys
import time
import Queue
import threading
import random


MAX_THREADS = 5


def start_thread(*args, **kwargs):
    t = threading.Thread(*args, **kwargs)
    t.daemon = True
    t.start()

    return t


def dumb_print(q):
    line = q.get()
    time.sleep(random.randint(1, 10))

    sys.stdout.write(line)

    q.task_done()


def main_loop():
    q = Queue.Queue(MAX_THREADS)

    with open('strings.txt', 'r') as f:
        data = f.readlines()

    for line in data:
        q.put(line)
        start_thread(target=dumb_print, args=(q,))

    q.join()


if __name__ == '__main__':
    sys.stdout.write('start\n')

    loop = start_thread(target=main_loop)

    while loop.is_alive():
        sys.stdout.write('still alive\n')
        time.sleep(random.randint(1, 5))

    sys.stdout.write('done\n')

I advise you to read the documentation on Queue
ps as already mentioned: if you have calculations on the processor, and not IO, then it is better to look towards the multiprocessing module , since the API is almost the same.

Answer 2

No need to write multi-threaded programs in python, because GIL.
If you need good multithreading - Java in hand.