update user point profile picture fetcher for expired CDNs link retrieval (#672)

* updating how user profile pics are updated so that there's bigger emphasis on retrieving users based on whether their discord CDN has expired rather than their bucket number

* also resettng the number of attempts when a user's bucket is reset

* adding a separation of function for getting users with expired images and users whose bucket_number has come

* separating out the current logic for syncing users in the background into their own functions

* updating reset command so it also resets the avatar discord channel

Author: modernNeo

.wall_e_models

@@ -39,7 +39,8 @@ def __init__(self):
         self.bucket_update_in_progress = False
         self.ensure_xp_roles_exist_and_have_right_users.start()
         self.process_leveling_profile_data_for_lurkers.start()
-        self.process_leveling_profile_data_for_active_users.start()
+        # self.process_leveling_profile_data_for_active_users.start() # will re-enable when all the current users have
+        # buckets again
 
     @commands.Cog.listener(name="on_ready")
     async def get_guild(self):
@@ -472,15 +473,124 @@ async def assign_roles_on_member_join(self, member: discord.Member):
     @tasks.loop(hours=1)
     async def process_leveling_profile_data_for_lurkers(self):
         """
-        Goes through all the UserPoint objects that have been marked indicating their profile has been updated
-         and needs to have wall_e's database updated for the leveling website
-        :return:
+        Goes through all the UserPoint objects whose avatar CDN link has expired or who don't yet have a bucket number
+        and ensure their information has been updated for the leveling website
         """
-        if self.user_points is None or self.levelling_website_avatar_channel is None or self.guild is None:
+        not_ready_to_process_lurkers = (
+                self.user_points is None or self.levelling_website_avatar_channel is None or self.guild is None or
+                self.bucket_update_in_progress
+        )
+        if not_ready_to_process_lurkers:
             return
         self.logger.debug("[Leveling process_leveling_profile_data_for_lurkers()] background task starting")
         await self._set_bucket_numbers()
         self.logger.debug("[Leveling process_leveling_profile_data_for_lurkers()] null bucket_number has been set")
+
+        entry = await self._get_current_bucket_number()
+
+        user_ids_to_update = set()
+        user_ids_to_update.update(await UserPoint.get_users_with_current_bucket_number(entry.bucket_number_completed))
+
+        user_ids_to_update.update(await UserPoint.get_users_with_expired_images())
+        self.logger.debug(
+            f"[Leveling process_leveling_profile_data_for_lurkers()] {user_ids_to_update} "
+            f"potential updates retrieved for bucket {entry.bucket_number_completed}"
+        )
+        await self._update_users(user_ids_to_update)
+        await ProfileBucketInProgress.async_save(entry)
+
+    async def _set_bucket_numbers(self):
+        """
+        Assigns a bucket_number to any new UserPoints that don't yet have one
+
+        The logic is implemented by creating a dictionary with a bucket for each hour of the month with the lowest
+         number of days so that this algorithm can also work on leap years.
+         The idea is that each user will be set a certain hour of the month when it should be updated. And it will
+          only be updated in that time ASSUMING that the user is a lurker who is not regularly sending messages.
+           Because if the user is regular sending messages, chances are any changes in their profile will be caught
+        by get_updated_user_logs
+        :return:
+        """
+        if self.bucket_update_in_progress:
+            return
+        self.bucket_update_in_progress = True
+        user_points = [user_point for user_point in self.user_points.values() if user_point.bucket_number is None]
+        if len(user_points) == 0:
+            return
+
+        users_to_update = self._setup_bucket_number_for_new_users()
+
+        self.logger.debug(
+            f"[Leveling _set_bucket_numbers()] updating {len(users_to_update)} user_point objects' bucket_number"
+        )
+        await UserPoint.async_bulk_update(users_to_update, ["bucket_number"])
+        self.bucket_update_in_progress = False
+        self.logger.debug(
+            f"[Leveling _set_bucket_numbers()] updated {len(users_to_update)} user_point objects' date_to_check"
+        )
+
+    def _setup_bucket_number_for_new_users(self) -> list:
+        """
+        :return: the users who need to have their bucket_number updated
+        """
+        date_buckets = self._initialize_bucket_with_number_of_current_users()
+        users_to_update = []
+        for user_id in self.user_points.keys():
+            if self.user_points[user_id].bucket_number is None:
+                lowest_bucket_number = self._get_bucket_number_with_lowest_number_of_users(date_buckets)
+                date_buckets[lowest_bucket_number] += 1
+                self.user_points[user_id].bucket_number = lowest_bucket_number
+                users_to_update.append(self.user_points[user_id])
+        return users_to_update
+
+    def _initialize_bucket_with_number_of_current_users(self) -> dict:
+        """
+        :return: the bucket dict with the users that currently have a bucket_number attached
+         already reflected on it
+        """
+        date_buckets = Leveling._initialize_blank_bucket()
+
+        # populate the date_buckets values with the number of users that currently exist in those buckets
+        for user_id in self.user_points.keys():
+            if self.user_points[user_id].bucket_number is not None:
+                date_buckets[self.user_points[user_id].bucket_number] += 1
+        return date_buckets
+
+    @staticmethod
+    def _initialize_blank_bucket() -> dict:
+        """
+        :return: a blank bucket dict that has the slots necessary to determine when people should be divided into
+         slots/buckets within a 2-week period
+        """
+        date_buckets = {}
+        bucket_number = 1
+        for day in range(1, 14):  # discord CDN links apparently expire after 2 weeks and need to be re-retrieved
+            for hour in range(1, 24):
+                date_buckets[bucket_number] = 0
+                bucket_number += 1
+        return date_buckets
+
+    @staticmethod
+    def _get_bucket_number_with_lowest_number_of_users(data_buckets):
+        """
+        :param data_buckets:
+        :return: the bucket_number which has the lowest number of users
+        """
+        low_load_bucket_number = None
+        min_value = None
+        for curr_bucket_number, number_of_user_to_checks in data_buckets.items():
+            if min_value is None:
+                low_load_bucket_number = curr_bucket_number
+                min_value = number_of_user_to_checks
+            elif min_value > number_of_user_to_checks:
+                low_load_bucket_number = curr_bucket_number
+                min_value = number_of_user_to_checks
+        return low_load_bucket_number
+
+    async def _get_current_bucket_number(self) -> ProfileBucketInProgress:
+        """
+        :return: the bucket_number to work on in the current iteration
+        """
         entry = await ProfileBucketInProgress.retrieve_entry()
         if entry is None:
             entry = await ProfileBucketInProgress.create_entry()
@@ -494,12 +604,15 @@ async def process_leveling_profile_data_for_lurkers(self):
             entry.bucket_number_completed += 1
             if entry.bucket_number_completed > max_bucket_number:
                 entry.bucket_number_completed = 1
-        updated_user_ids = await UserPoint.get_users_that_need_leveling_info_updated(entry.bucket_number_completed)
+        return entry
+
+    async def _update_users(self, updated_user_ids):
+        """
+        iterates through the given list of user_ids and updates them
+        :param updated_user_ids:
+        :return:
+        """
         total_number_of_updates_needed = len(updated_user_ids)
-        self.logger.debug(
-            f"[Leveling process_leveling_profile_data_for_lurkers()] {total_number_of_updates_needed} "
-            f"potential updates retrieved for bucket {entry.bucket_number_completed}"
-        )
         for index, user_id in enumerate(updated_user_ids):
             self.logger.debug(
                 f"[Leveling process_leveling_profile_data_for_lurkers()] attempting to get updated "
@@ -516,62 +629,6 @@ async def process_leveling_profile_data_for_lurkers(self):
                     pass
             if member:
                 await self._update_member_profile_data(member, user_id, index, total_number_of_updates_needed)
-        await ProfileBucketInProgress.async_save(entry)
-
-    async def _set_bucket_numbers(self):
-        """
-        Takes any [new] UserPoints that don't yet have a bucket_number set
-
-        The logic is implemented by creating a dictionary with a bucket for each hour of the month with the lowest
-         number of days so that this algorithm can also work on leap years.
-         The idea is that each user will be set a certain hour of the month when it should be updated. And it will
-          only be updated in that time ASSUMING that the user is a lurker who is not regularly sending messages.
-           Because if the user is regular sending messages, chances are any changes in their profile will be caught
-        by get_updated_user_logs
-        :return:
-        """
-        user_points = [user_point for user_point in self.user_points.values() if user_point.bucket_number is None]
-        if len(user_points) == 0:
-            return
-        if self.bucket_update_in_progress:
-            return
-        self.bucket_update_in_progress = True
-        date_buckets = {}
-        bucket_number = 1
-        for day in range(1, 14):  # discord CDN links apparently expire after 2 weeks and need to be re-retrieved
-            for hour in range(1, 24):
-                date_buckets[bucket_number] = 0
-                bucket_number += 1
-        for user_id in self.user_points.keys():
-            if self.user_points[user_id].bucket_number is not None:
-                date_buckets[self.user_points[user_id].bucket_number] += 1
-
-        def get_bucket_number_with_lowest_user_points(data_buckets_local):
-            low_load_bucket_number = None
-            min_value = None
-            for curr_bucket_number, number_of_user_to_checks in data_buckets_local.items():
-                if min_value is None:
-                    low_load_bucket_number = curr_bucket_number
-                    min_value = number_of_user_to_checks
-                elif min_value > number_of_user_to_checks:
-                    low_load_bucket_number = curr_bucket_number
-                    min_value = number_of_user_to_checks
-            return low_load_bucket_number
-        users_to_update = []
-        for user_id in self.user_points.keys():
-            if self.user_points[user_id].bucket_number is None:
-                lowest_bucket_number = get_bucket_number_with_lowest_user_points(date_buckets)
-                date_buckets[lowest_bucket_number] += 1
-                self.user_points[user_id].bucket_number = lowest_bucket_number
-                users_to_update.append(self.user_points[user_id])
-        self.logger.debug(
-            f"[Leveling _set_bucket_numbers()] updating {len(users_to_update)} user_point objects' bucket_number"
-        )
-        await UserPoint.async_bulk_update(users_to_update, ["bucket_number"])
-        self.bucket_update_in_progress = False
-        self.logger.debug(
-            f"[Leveling _set_bucket_numbers()] updated {len(users_to_update)} user_point objects' date_to_check"
-        )
 
     @tasks.loop(seconds=2)
     async def process_leveling_profile_data_for_active_users(self):
@@ -619,20 +676,23 @@ async def _update_member_profile_data(self, member, updated_user_id, index, tota
         if member:
             try:
                 if self.user_points[member.id].leveling_update_attempt >= 5:
-                    self.logger.warn(
+                    self.logger.error(
                         f"[Leveling _update_member_profile_data()] "
                         f"attempt {self.user_points[member.id].leveling_update_attempt} to update the member profile"
                         f" data in the database for member {member} {index + 1}/{total_number_of_updates_needed}"
                     )
-                user_updated = await self.user_points[member.id].update_leveling_profile_info(
-                    self.logger, member, self.levelling_website_avatar_channel,
-                    updated_user_log_id=updated_user_log_id
-                )
-                if user_updated:
-                    self.logger.debug(
-                        f"[Leveling _update_member_profile_data()] updated the member profile data"
-                        f" in the database for member {member} {index + 1}/{total_number_of_updates_needed}"
+                else:
+                    # leveling_update_attempt is reset to 0 in update_leveling_profile_info if member is successfully
+                    # updated THIS time
+                    user_updated = await self.user_points[member.id].update_leveling_profile_info(
+                        self.logger, member, self.levelling_website_avatar_channel,
+                        updated_user_log_id=updated_user_log_id
                     )
+                    if user_updated:
+                        self.logger.debug(
+                            f"[Leveling _update_member_profile_data()] updated the member profile data"
+                            f" in the database for member {member} {index + 1}/{total_number_of_updates_needed}"
+                        )
             except Exception as e:
                 self.logger.error(
                     f"[Leveling _update_member_profile_data()] unable to update the member profile"
@@ -648,9 +708,9 @@ async def _update_member_profile_data(self, member, updated_user_id, index, tota
             )
             await self.user_points[member.id].async_save()
 
-    @app_commands.command(name="reset_bucket_number")
+    @app_commands.command(name="reset_user_profiles")
     @app_commands.checks.has_any_role("Bot_manager")
-    async def reset_bucket_number(self, interaction: discord.Interaction):
+    async def reset_user_profiles(self, interaction: discord.Interaction):
         await interaction.response.defer()
         if self.bucket_update_in_progress:
             e_obj = await embed(
@@ -666,8 +726,18 @@ async def reset_bucket_number(self, interaction: discord.Interaction):
         users_to_update = []
         for user_id in self.user_points.keys():
             self.user_points[user_id].bucket_number = None
+            self.user_points[user_id].leveling_update_attempt = 0
+            self.user_points[user_id].avatar_url = None
+            self.user_points[user_id].avatar_url_message_id = None
+            self.user_points[user_id].leveling_message_avatar_url = None
+            self.user_points[user_id].discord_avatar_link_expiry_date = None
             users_to_update.append(self.user_points[user_id])
-        await UserPoint.async_bulk_update(users_to_update, ["bucket_number"])
+        await UserPoint.async_bulk_update(
+            users_to_update,
+            ["bucket_number", "leveling_update_attempt", "avatar_url", "avatar_url_message_id",
+             "leveling_message_avatar_url", "discord_avatar_link_expiry_date"
+             ]
+        )
         e_obj = await embed(
             self.logger, interaction=interaction,
             description=f'{len(users_to_update)} bucket_numbers reset to None'
@@ -676,6 +746,18 @@ async def reset_bucket_number(self, interaction: discord.Interaction):
             await interaction.followup.send(embed=e_obj)
             await asyncio.sleep(5)
             await interaction.delete_original_response()
+        if self.levelling_website_avatar_channel is not None:
+            await self.levelling_website_avatar_channel.delete()
+        leveling_website_avatar_images_channel_id = await bot.bot_channel_manager.create_or_get_channel_id(
+            self.logger, self.guild, wall_e_config.get_config_value('basic_config', 'ENVIRONMENT'),
+            'leveling_website_avatar_images'
+        )
+        self.levelling_website_avatar_channel: discord.TextChannel = discord.utils.get(
+            self.guild.channels, id=leveling_website_avatar_images_channel_id
+        )
+        self.logger.debug(
+            f"[Leveling get_leveling_avatar_channel()] bot channel {self.levelling_website_avatar_channel} acquired."
+        )
         self.bucket_update_in_progress = False
 
     @commands.command(