Celery "hangs" and does not receive a task for execution?

Question

Python

There are registered tasks: A task

that parses views from a telegram post, a fully working method, if not wrapped in a task:

@task(name='Telegram statistic task')
def telegram_views(url, post_id, message_id, user_id):
    def count_of_views(views):
        if views.isdigit():
            return int(views)
        else:
            char_index = views.find('K')
            views = views[0:char_index:]
            if '.' in views:
                count_views = int(''.join(views.split('.'))) * 100
                return count_views
            else:
                return int(views[0:char_index:]) * 100

    html_doc = requests.get(url + '?embed=1')
    soup = BeautifulSoup(html_doc.content, 'html.parser')
    views = soup.find('span', class_='tgme_widget_message_views')
    post_views = count_of_views(views.text)
    filtered_data = {}
    if views is not None:

        # Update post
        filtered_data.update({post_id: {}})
        filtered_data.get(post_id).update({
            'response': {
                'views': post_views
            },
            'post_info': {
                'message_id': message_id,
                'user_id': user_id,
                'social_network': 'tl',
            }
            })
        return filtered_data
    else:
        return 'empty'

A task that throws other tasks for execution to collect statistics:

@task()
@timethis
def collect_stats():
    start = round(time.time()) - 60
    stop = round(time.time())
    print('Start time: %d\nStop time: %d' % (start, stop))

    mes = Message.objects.filter(next_timestamp__range=(start, stop)).filter(status='done')
    print('MESSAGES: %s' % mes)

    # Celery tasks
    tasks = []

    # Value of limit posts in one request
    vk_posts_limit = 100
    fb_posts_limit = 50
    ok_posts_limit = 100

    # TODO
    # Found out limit for odnoklassniki

    # Posts array of social network
    vk_public_posts = []
    fb_public_posts = []
    ok_public_posts = []

    # Timestamp for all checks
    timestamp = round(time.time())

    for one_mes in mes:
        user = UserProfile.objects.get(id=one_mes.owner_id)
        posts = Post.objects.filter(message_id=one_mes.message_id)

        for post in posts:
            if post.__dict__.get('social_network') == 'vk':

                # Public community
                if post.community.is_closed is False:
                    vk_public_posts.append({'post_id': post.post_id, 'message_id': one_mes.id, 'user_id': user.id})

                # For private communities
                else:
                    tasks.append(vk_stats.s(user.vk_token, [{'post_id': post.post_id, 'message_id': one_mes.id, 'user_id': user.id}]))

            elif post.__dict__.get('social_network') == 'fb':

                # Public community
                if post.community.is_closed is False:
                    fb_public_posts.append({'post_id': post.post_id, 'message_id': one_mes.id, 'user_id': user.id})

                # For private communities
                else:
                    tasks.append(fb_stats.s(user.fb_token, [{'post_id': post.post_id, 'message_id': one_mes.id, 'user_id': user.id}]))

            elif post.__dict__.get('social_network') == 'ok':

                # Public community
                if post.community.is_closed is False:
                    ok_public_posts.append({'post_id': post.post_id, 'message_id': one_mes.id, 'user_id': user.id})

                # For private communities
                else:
                    tasks.append(ok_stats.s(user.ok_token, [{'post_id': post.post_id, 'message_id': one_mes.id, 'user_id': user.id}]))

            elif post.__dict__.get('social_network') == 'tw':
                tasks.append(tw_stats.s(user.tw_token, user.tw_secret, {'post_id': post.post_id, 'message_id': one_mes.id, 'user_id': user.id}))

            elif post.__dict__.get('social_network') == 'tl':
                # Here the condition is satisfied
                # Это последняя рабочая область куда может дойти выполнение кода, дальше логи ничего не отображают
                tasks.append(telegram_views.s(post.get('post_link'), post.get('post_id'), one_mes.id, user.id))

            elif post.__dict__.get('social_network') == 'insta':
                # TODO
                # tasks.append
                pass



    # Separation posts
    vk_first, *vk_after = array_separation(vk_public_posts, vk_posts_limit)
    fb_first, *fb_after = array_separation(fb_public_posts, fb_posts_limit)
    ok_first, *ok_after = array_separation(ok_public_posts, ok_posts_limit)

    # Append posts to tasks
    tasks.append(vk_stats.s(vk_service_key_access, vk_first))
    for pack in vk_after:
        tasks.append(vk_stats.s(vk_service_key_access, pack))

    tasks.append(fb_stats.s(fb_app_token, fb_first))
    for pack in fb_after:
        tasks.append(fb_stats.s(fb_app_token, pack))

    tasks.append(ok_stats.s(ok_eternal_token, ok_first))
    for pack in ok_after:
        tasks.append(ok_stats.s(ok_eternal_token, pack))

    print('Tasks: %s' % str(tasks))

    callback = stats_result.s(tasks_response=tasks, timestamp=timestamp)
    chord(tasks)(callback)

If the selection contains a post from telegram:

celery_1     | [2018-08-01 10:26:54,497: WARNING/ForkPoolWorker-2] MESSAGES: <QuerySet [<Message: 84 7 Test  1532678399>]>

at this point, the Celery log stops and nothing happens.
Sample without post from telegram (normal operation):

celery_1     | [2018-08-01 10:28:54,584: WARNING/ForkPoolWorker-14] MESSAGES: <QuerySet []>
celery_1     | [2018-08-01 10:28:54,615: WARNING/ForkPoolWorker-14] Tasks: [Vkontakte statistic task('', []), Facebook statistic task('', []), Odnoklassniki statistic task('tkn1Ud9dcxkz8dsBv71SEsgUo5QJIYZjbG0GqA5zzUeTjq6rv7d1uuua9zcFTPRoTSIIH0', [])]
celery_1     | [2018-08-01 10:28:54,627: INFO/MainProcess] Received task: Vkontakte statistic task[faecd67a-0073-42d9-9efa-d8374662441d]  
celery_1     | [2018-08-01 10:28:54,632: INFO/MainProcess] Received task: Facebook statistic task[5c158f76-11f3-479b-992a-061119502191]  
celery_1     | [2018-08-01 10:28:54,636: INFO/MainProcess] Received task: Odnoklassniki statistic task[3657b25e-676f-4e28-8fd0-c38b5cd6e21e]  
celery_1     | [2018-08-01 10:28:54,645: INFO/MainProcess] Received task: Data after receiving statistics[a870c00e-6211-483f-8100-afc633881c8a]

How can such behavior be explained and where to look for the cause?

Answer 1

look in the logs,
leave only the problematic one and study