diff --git a/LICENSE.TXT b/LICENSE similarity index 100% rename from LICENSE.TXT rename to LICENSE diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 09bd4c474..15104c626 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -1,7 +1,8 @@ import zipfile import json import csv -from collections import OrderedDict +import re +from collections import OrderedDict, defaultdict from io import StringIO from django.http import HttpResponse from tempfile import SpooledTemporaryFile @@ -36,6 +37,12 @@ from api.permissions import IsOrganizerOrCollaborator from django.db import transaction from django.conf import settings +from django.contrib.auth import get_user_model +User = get_user_model() + +import logging +logger = logging.getLogger(__name__) + class CompetitionViewSet(ModelViewSet): @@ -431,9 +438,12 @@ def collect_leaderboard_data(self, competition, phase_pk=None): generated_columns = OrderedDict() for task in phase['tasks']: for col in leaderboard.columns.all(): + # key by column_key-taskid generated_columns.update({f'{col.key}-{task["id"]}': f'{task["name"]}({task["id"]})-{col.title}'}) for submission in phase['submissions']: - submission_key = f'{submission["owner"]}-{submission["parent"] or submission["id"]}' + # Use submission id to make each submission (parent or child) unique. + # This prevents multiple children of the same parent being merged. + submission_key = f'{submission["owner"]}-{submission["id"]}' if submission_key not in leaderboard_data[leaderboard_titles[phase['id']]].keys(): leaderboard_data[leaderboard_titles[phase['id']]].update({submission_key: OrderedDict()}) if 'fact_sheet_answers' in submission.keys() and submission['fact_sheet_answers']: @@ -442,9 +452,17 @@ def collect_leaderboard_data(self, competition, phase_pk=None): for col_title in generated_columns.values(): leaderboard_data[leaderboard_titles[phase['id']]][submission_key].update({col_title: ""}) for score in submission['scores']: - score_column = generated_columns[f'{score["column_key"]}-{submission["task"]}'] - leaderboard_data[leaderboard_titles[phase['id']]][submission_key].update({score_column: score['score']}) + # make task id lookup robust: submission['task'] might be an int or dict with an 'id' key + task_id = submission['task'] if isinstance(submission['task'], int) else (submission['task'].get('id') if submission.get('task') else None) + if task_id is None: + # skip if no task info + continue + key = f'{score["column_key"]}-{task_id}' + if key in generated_columns: + score_column = generated_columns[key] + leaderboard_data[leaderboard_titles[phase['id']]][submission_key].update({score_column: score['score']}) return leaderboard_data + @action(detail=True, methods=['GET'], renderer_classes=[JSONRenderer, CSVRenderer, ZipRenderer]) def results(self, request, pk, format='json'): @@ -767,104 +785,455 @@ def rerun_submissions(self, request, pk): return Response({"count": rerun_count}) else: raise PermissionDenied(error_message) - - # @swagger_auto_schema(responses={200: PhaseResultsSerializer}) + @extend_schema(responses={200: PhaseResultsSerializer}) @action(detail=True, methods=['GET'], permission_classes=[AllowAny]) def get_leaderboard(self, request, pk): - phase = self.get_object() - if phase.competition.fact_sheet: - fact_sheet_keys = [(phase.competition.fact_sheet[question]['key'], phase.competition.fact_sheet[question]['title']) - for question in phase.competition.fact_sheet if phase.competition.fact_sheet[question]['is_on_leaderboard'] == 'true'] - else: + try: + phase = self.get_object() + logger.info("===== LEADERBOARD ENTRY =====") + logger.debug(f"Requested phase pk: {pk}; resolved phase id: {getattr(phase, 'id', None)}") + fact_sheet_keys = None - query = LeaderboardPhaseSerializer(phase).data - response = { - 'title': query['leaderboard']['title'], - 'id': phase.id, - 'submissions': [], - 'tasks': [], - 'fact_sheet_keys': fact_sheet_keys or None, - 'primary_index': query['leaderboard']['primary_index'] - } - columns = [col for col in query['columns']] - submissions_keys = {} - submission_detailed_results = {} - for submission in query['submissions']: - submission_key = f"{submission['owner']}{submission['parent'] or submission['id']}" - # gather detailed result from submissions for each task - # detailed_results are gathered based on submission key - # `id` is used to fetch the right detailed result in detailed results page - # `detailed_result` url is not needed - submission_detailed_results.setdefault(submission_key, []).append({ - # 'detailed_result': submission['detailed_result'], - 'task': submission['task'], - 'id': submission['id'] - }) - - if submission_key not in submissions_keys: - submissions_keys[submission_key] = len(response['submissions']) - response['submissions'].append({ - 'id': submission['id'], - 'owner': submission['display_name'] or submission['owner'], - 'scores': [], - 'detailed_results': [], - 'fact_sheet_answers': submission['fact_sheet_answers'], - 'slug_url': submission['slug_url'], - 'organization': submission['organization'], - 'created_when': submission['created_when'] - }) - for score in submission['scores']: - - # to check if a column is found - # this is useful because of `hidden` field - # if a column is hidden it will not be shown here so - # we will not return that score to the front-end - column_found = False - # default precision is set to 2 - precision = 2 - # default hidden is set to false - hidden = False - - # loop over columns to find a column with the same index - # replace default precision with column precision - for col in columns: - if col["index"] == score["index"]: - precision = col["precision"] - hidden = col["hidden"] - column_found = True - break - - tempScore = score - tempScore['task_id'] = submission['task'] - # round the score to 'precision' decimal points - tempScore['score'] = str(round(float(tempScore["score"]), precision)) - - # only add scores to the scores list - # if this column is found - # and - # column is not hidden - if column_found and not hidden: - response['submissions'][submissions_keys[submission_key]]['scores'].append(tempScore) - - # put detailed results in its submission - for k, v in submissions_keys.items(): - response['submissions'][v]['detailed_results'] = submission_detailed_results[k] - - for task in query['tasks']: - # This can be used to rendered variable columns on each task - tempTask = { - 'name': task['name'], - 'id': task['id'], - 'colWidth': len(columns), - 'columns': [], + try: + if phase.competition.fact_sheet: + fact_sheet_keys = [ + (phase.competition.fact_sheet[q]['key'], phase.competition.fact_sheet[q]['title']) + for q in phase.competition.fact_sheet + if str(phase.competition.fact_sheet[q].get('is_on_leaderboard', '')).lower() == 'true' + ] + except Exception: + logger.exception("Error reading competition.fact_sheet; continuing without fact_sheet_keys.") + + try: + query = LeaderboardPhaseSerializer(phase).data + except Exception: + logger.exception("Failed to serialize phase with LeaderboardPhaseSerializer.") + return Response({"detail": "Failed to serialize leaderboard data."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + serialized_submissions = list(query.get('submissions', [])) + + logger.warning("===== LEADERBOARD DEBUG START =====") + logger.warning(f"Phase ID: {getattr(phase, 'id', None)}") + logger.warning(f"Number of serialized submissions: {len(serialized_submissions)}") + logger.debug(f"Serialized submissions sample (first 8): {serialized_submissions[:8]}") + + response = { + 'title': query.get('leaderboard', {}).get('title'), + 'id': phase.id, + 'submissions': [], + 'tasks': [], + 'groups': [], + 'fact_sheet_keys': fact_sheet_keys or None, + 'primary_index': query.get('leaderboard', {}).get('primary_index') } + + columns = [col for col in query.get('columns', []) or []] + logger.debug(f"Columns loaded: {[c.get('key') for c in columns]}") + + group_order = [] + try: + part_groups = getattr(phase.competition, "participant_groups", None) + if part_groups is not None: + try: + group_order = [g.name for g in part_groups.all()] + except Exception: + try: + group_order = list(part_groups) + except Exception: + group_order = [] + except Exception: + logger.exception("Error reading competition.participant_groups; ignoring.") + logger.debug(f"Initial group_order from competition: {group_order}") + + tasks_in_query = query.get('tasks') or [] + logger.debug(f"Tasks in query count: {len(tasks_in_query)}") + + child_parent_group = defaultdict(dict) + children_by_parent = defaultdict(list) + seen_groups = set() + for s in serialized_submissions: + if not s.get('parent'): + continue + parent_id = s['parent'] + gid_raw = None + org = s.get('organization') + if isinstance(org, dict): + gid_raw = org.get('name') + elif isinstance(org, str) and org: + gid_raw = org + if not gid_raw: + sd = s.get('status_details') + if isinstance(sd, dict): + gid_raw = sd.get('group') or sd.get('group_name') or sd.get('name') + elif sd: + gid_raw = str(sd) + if not gid_raw: + owner = s.get('owner') + if isinstance(owner, dict) and owner.get('username'): + gid_raw = owner.get('username') + elif isinstance(owner, str): + gid_raw = owner + else: + gid_raw = f'group-{s.get("id")}' + + gid_final = gid_raw + matched = None + if group_order: + low_raw = (gid_raw or "").lower() + group_order_lower = [g.lower() for g in group_order] + if low_raw in group_order_lower: + matched = group_order[group_order_lower.index(low_raw)] + if not matched: + for g in group_order: + if low_raw and low_raw in (g or "").lower(): + matched = g + break + if (g or "").lower() and (g or "").lower() in low_raw: + matched = g + break + if not matched: + digits = re.findall(r'\d+', gid_raw or "") + if digits: + for d in digits: + for g in group_order: + if d in (g or ""): + matched = g + break + if matched: + break + if matched: + gid_final = matched + logger.info(f"Child {s.get('id')} gid_raw='{gid_raw}' matched to canonical group '{gid_final}'") + else: + logger.debug(f"Child {s.get('id')} gid_raw='{gid_raw}' did NOT match any canonical group; keeping raw key.") + + child_parent_group[parent_id][gid_final] = s + children_by_parent[parent_id].append(s) + seen_groups.add(gid_final) + logger.debug(f"Mapped child {s.get('id')} -> parent {parent_id} as group key '{gid_final}' (raw='{gid_raw}')") + + if not group_order and seen_groups: + if len(seen_groups) > 1: + group_order = sorted(list(seen_groups)) + logger.info(f"Fallback group_order from seen_groups (multiple keys): {group_order}") + else: + if len(tasks_in_query) > 1: + logger.info("Detected multi-task phase and only one seen_group -> skip using seen_groups as groups (treat as multi-task legacy).") + else: + group_order = sorted(list(seen_groups)) + logger.info(f"Fallback group_order from seen_groups (single key, single-task): {group_order}") + + if group_order: + response['groups'] = [{'name': g, 'colCount': len(columns)} for g in group_order] + logger.info(f"Group order determined: {group_order}") + else: + logger.info("No participant groups found / used for this leaderboard.") + + submissions_keys = {} + submission_detailed_results = {} + row_parent_map = {} + + is_multi_task = (not group_order) and (len(tasks_in_query) > 1) + logger.debug(f"is_multi_task={is_multi_task} (group_order present? {bool(group_order)} tasks_count={len(tasks_in_query)})") + + if group_order: + parent_ids = set() + + for s in serialized_submissions: + if s.get('parent'): + parent_ids.add(s.get('parent')) + else: + parent_ids.add(s.get('id')) + + for parent_id in parent_ids: + parent_entry = next( + (x for x in serialized_submissions if x.get('id') == parent_id and not x.get('parent')), + None + ) + + if parent_entry: + owner = parent_entry.get('display_name') or parent_entry.get('owner') + slug = parent_entry.get('slug_url') + org = parent_entry.get('organization') + created = parent_entry.get('created_when') + fact_sheet = parent_entry.get('fact_sheet_answers') + else: + child = children_by_parent.get(parent_id, [None])[0] + owner = (child.get('display_name') or child.get('owner')) if child else "unknown" + slug = child.get('slug_url') if child else None + org = child.get('organization') if child else None + created = child.get('created_when') if child else None + fact_sheet = child.get('fact_sheet_answers') if child else {} + + submission_key = str(parent_id) + submissions_keys[submission_key] = len(response['submissions']) + + response['submissions'].append({ + 'id': parent_id, + 'owner': owner, + 'scores': [], + 'detailed_results': [], + 'fact_sheet_answers': fact_sheet, + 'slug_url': slug, + 'organization': org, + 'created_when': created + }) + + row_parent_map[submission_key] = parent_id + + else: + for submission in serialized_submissions: + if is_multi_task: + submission_key = f"{submission.get('owner')}" + else: + submission_key = f"{submission.get('owner')}{submission.get('parent') or submission.get('id')}" + + submission_detailed_results.setdefault(submission_key, []).append({ + 'task': submission.get('task'), + 'id': submission.get('id') + }) + + if submission_key not in submissions_keys: + submissions_keys[submission_key] = len(response['submissions']) + response['submissions'].append({ + 'id': submission.get('id'), + 'owner': submission.get('display_name') or submission.get('owner'), + 'scores': [], + 'detailed_results': [], + 'fact_sheet_answers': submission.get('fact_sheet_answers'), + 'slug_url': submission.get('slug_url'), + 'organization': submission.get('organization'), + 'created_when': submission.get('created_when') + }) + row_parent_map[submission_key] = submission.get('parent') if submission.get('parent') else submission.get('id') + + if not group_order and not is_multi_task: + parent_candidates = set() + for s in serialized_submissions: + if s.get('parent'): + parent_candidates.add(s.get('parent')) + else: + parent_candidates.add(s.get('id')) + + for parent_id in parent_candidates: + parent_entry = next((x for x in serialized_submissions if x.get('id') == parent_id and not x.get('parent')), None) + if parent_entry: + parent_owner = parent_entry.get('owner') + else: + first_child = children_by_parent.get(parent_id, [None])[0] + parent_owner = first_child.get('owner') if first_child else f'unknown-{parent_id}' + + submission_key = f"{parent_owner}{parent_id}" + if submission_key not in submissions_keys: + logger.info(f"Creating synthetic parent row for parent_id={parent_id} submission_key='{submission_key}'") + submissions_keys[submission_key] = len(response['submissions']) + if parent_entry: + row_owner = parent_entry.get('display_name') or parent_entry.get('owner') + slug = parent_entry.get('slug_url') + org = parent_entry.get('organization') + created = parent_entry.get('created_when') + fact_sheet = parent_entry.get('fact_sheet_answers') + else: + child = children_by_parent.get(parent_id, [None])[0] + row_owner = (child.get('display_name') or child.get('owner')) if child else parent_owner + slug = child.get('slug_url') if child else None + org = child.get('organization') if child else None + created = child.get('created_when') if child else None + fact_sheet = child.get('fact_sheet_answers') if child else {} + + response['submissions'].append({ + 'id': parent_id, + 'owner': row_owner, + 'scores': [], + 'detailed_results': [], + 'fact_sheet_answers': fact_sheet, + 'slug_url': slug, + 'organization': org, + 'created_when': created + }) + row_parent_map[submission_key] = parent_id + + for k, v in submissions_keys.items(): + response['submissions'][v]['detailed_results'] = submission_detailed_results.get(k, []) + + for parent_id, children in list(children_by_parent.items()): + try: + num_children = len(children) + num_groups = len(group_order) + assigned_groups = set(child_parent_group[parent_id].keys()) + if num_groups and num_children == num_groups and not set(group_order).issubset(assigned_groups): + logger.warning(f"Fallback mapping: parent {parent_id} has {num_children} children and competition defines {num_groups} groups; remapping children to groups by child id order.") + sorted_children = sorted(children, key=lambda c: (c.get('id') or 0)) + child_parent_group[parent_id].clear() + for idx, child in enumerate(sorted_children): + g = group_order[idx] + child_parent_group[parent_id][g] = child + child['_group_name'] = g + logger.info(f"Fallback assigned child {child.get('id')} -> parent {parent_id} -> group '{g}'") + children_by_parent[parent_id] = sorted_children + except Exception: + logger.exception(f"Error during fallback group distribution for parent {parent_id}") + + columns_by_index = {} for col in columns: - tempTask['columns'].append(col) - response['tasks'].append(tempTask) - return Response(response) + idx = col.get('index') + if idx is not None: + columns_by_index[int(idx)] = col + task_fallback_id = None + if tasks_in_query: + try: + task_fallback_id = tasks_in_query[0].get('id') + except Exception: + task_fallback_id = None + + if not group_order: + logger.debug("Filling scores in legacy (no groups) mode.") + for submission in serialized_submissions: + if is_multi_task: + submission_key = f"{submission.get('owner')}" + else: + submission_key = f"{submission.get('owner')}{submission.get('parent') or submission.get('id')}" + row_idx = submissions_keys.get(submission_key) + if row_idx is None: + continue + for score in submission.get('scores', []) or []: + score_index = score.get("index") + if score_index is None: + logger.debug(f"Skipping score without index on submission {submission.get('id')}: {score}") + continue + + col = columns_by_index.get(int(score_index)) + if not col: + logger.debug(f"No column found for score index {score_index} on submission {submission.get('id')}; skipping.") + continue + + if col.get("hidden", False): + logger.debug(f"Column index {score_index} is hidden; skipping score for submission {submission.get('id')}") + continue + precision = col.get("precision", 2) + try: + precision_int = int(precision) + except Exception: + precision_int = 2 + + tempScore = dict(score) + tempScore['task_id'] = submission.get('task') if submission.get('task') is not None else task_fallback_id + try: + tempScore['score'] = str(round(float(tempScore.get("score")), precision_int)) + except Exception: + tempScore['score'] = tempScore.get("score") + + response['submissions'][row_idx]['scores'].append(tempScore) + logger.debug(f"Added legacy score to row {row_idx} (submission {submission.get('id')}): index={score_index} val={tempScore['score']} task_id={tempScore['task_id']}") + else: + logger.debug("Filling scores in grouped mode (one sub-column per group).") + for submission_key, row_idx in submissions_keys.items(): + parent_id = row_parent_map.get(submission_key) + logger.debug(f"Building group cells for row {row_idx} parent_id={parent_id}") + for g in group_order: + child = child_parent_group.get(parent_id, {}).get(g) + if child: + logger.info(f"Found child {child.get('id')} for parent {parent_id} group '{g}'. child.scores={child.get('scores')}") + for col in columns: + found = None + for s_score in child.get('scores', []) or []: + try: + if int(s_score.get('index')) == int(col.get('index')): + found = s_score + break + except Exception: + continue + if found: + precision = col.get('precision') if col.get('precision') is not None else 2 + try: + score_val = str(round(float(found.get('score')), int(precision))) + except Exception: + score_val = found.get('score') + logger.debug(f" -> child {child.get('id')} matched column index {col.get('index')} score {score_val}") + response['submissions'][row_idx]['scores'].append({ + 'id': found.get('id'), + 'index': col.get('index'), + 'column_key': f"{col.get('key')}__group__{g}", + 'score': score_val, + 'task_id': child.get('task'), + 'group_name': g + }) + else: + logger.debug(f" -> child {child.get('id')} has NO score for column index {col.get('index')} -> placeholder") + response['submissions'][row_idx]['scores'].append({ + 'id': None, + 'index': col.get('index'), + 'column_key': f"{col.get('key')}__group__{g}", + 'score': 'n/a', + 'task_id': child.get('task'), + 'group_name': g + }) + else: + logger.debug(f"No child found for parent {parent_id} and group '{g}' -> adding placeholders") + for col in columns: + response['submissions'][row_idx]['scores'].append({ + 'id': None, + 'index': col.get('index'), + 'column_key': f"{col.get('key')}__group__{g}", + 'score': 'n/a', + 'task_id': None, + 'group_name': g + }) + + for task in query.get('tasks', []): + if not group_order: + tempTask = { + 'name': task.get('name'), + 'id': task.get('id'), + 'colWidth': len(columns), + 'columns': [], + } + for col in columns: + tempTask['columns'].append(col) + response['tasks'].append(tempTask) + else: + tempTask = { + 'name': task.get('name'), + 'id': task.get('id'), + 'colWidth': len(columns) * max(1, len(group_order)), + 'columns': [], + } + for g in group_order: + for col in columns: + tempTask['columns'].append({ + 'id': col.get('id'), + 'computation': col.get('computation'), + 'computation_indexes': col.get('computation_indexes'), + 'key': f"{col.get('key')}__group__{g}", + 'title': col.get('title'), + 'group_title': g, + 'sorting': col.get('sorting'), + 'index': col.get('index'), + 'hidden': col.get('hidden'), + 'precision': col.get('precision'), + 'group_name': g, + 'task_id': task.get('id'), + }) + response['tasks'].append(tempTask) + + logger.warning("FINAL RESPONSE PREPARED FOR FRONTEND") + logger.warning(f"Groups: {response.get('groups')}") + logger.warning(f"Number of rows: {len(response.get('submissions', []))}") + for row in response.get('submissions', []): + logger.debug(f"Row {row.get('id')} owner {row.get('owner')} - scores count {len(row.get('scores', []))}") + logger.debug(f"Row {row.get('id')} scores sample: {row.get('scores')[:6]}") + logger.warning("===== LEADERBOARD DEBUG END =====") + + return Response(response) + + except Exception: + logger.exception("Unhandled exception in get_leaderboard.") + return Response({"detail": "Internal server error building leaderboard."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + class CompetitionParticipantViewSet(ModelViewSet): queryset = CompetitionParticipant.objects.all() serializer_class = CompetitionParticipantSerializer @@ -942,3 +1311,40 @@ def send_email(self, request, pk): return Response({'detail': 'A message is required to send an email'}, status=status.HTTP_400_BAD_REQUEST) send_direct_participant_email(participant=participant, content=message) return Response({}, status=status.HTTP_200_OK) + + +def resolve_owner_display(owner_serialized): + """ + owner_serialized peut être : dict (avec id/username), int (id), str (username or 'admin-386'), or None. + Renvoie le username résolu; sinon renvoie une fallback string (owner_serialized) si possible; sinon None. + """ + if not owner_serialized: + return None + try: + if isinstance(owner_serialized, dict): + # prefère username si présent, sinon id -> lookup + if owner_serialized.get('username'): + return str(owner_serialized.get('username')) + owner_id = owner_serialized.get('id') or owner_serialized.get('pk') or owner_serialized.get('user_id') + if owner_id: + return users_by_id.get(int(owner_id)) + elif isinstance(owner_serialized, int): + return users_by_id.get(owner_serialized) + elif isinstance(owner_serialized, str): + # try exact username lookup + uname = users_by_username.get(owner_serialized) + if uname: + return uname + # maybe username contains dash-id like admin-386 -> we can try to match numeric id + if '-' in owner_serialized: + parts = owner_serialized.split('-')[::-1] + for p in parts: + if p.isdigit(): + uid = int(p) + if uid in users_by_id: + return users_by_id[uid] + # fallback: return the raw string (useful if serializer only had username string) + return owner_serialized + except Exception: + logger.exception("Error resolving owner display from serialized owner.") + return None \ No newline at end of file diff --git a/src/apps/competitions/admin.py b/src/apps/competitions/admin.py index 769888db8..d69698b4a 100644 --- a/src/apps/competitions/admin.py +++ b/src/apps/competitions/admin.py @@ -1,10 +1,13 @@ +from django import forms from django.contrib import admin from django.utils.translation import gettext_lazy as _ import json import csv from django.http import HttpResponse -from profiles.models import User +from profiles.models import CustomGroup, User from . import models +from django.contrib.auth.models import Group +from django.contrib.admin.widgets import FilteredSelectMultiple # General class used to make custom filter @@ -348,6 +351,46 @@ class PhaseExpansion(admin.ModelAdmin): ] +class CustomGroupAdminForm(forms.ModelForm): + users = forms.ModelMultipleChoiceField( + queryset=User.objects.all(), + required=False, + widget=FilteredSelectMultiple("Users", is_stacked=False), + help_text="Add/Remove users for this group." + ) + + class Meta: + model = CustomGroup + fields = ('name', 'permissions', 'queue', 'users') + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.instance and self.instance.pk: + self.fields['users'].initial = self.instance.user_set.all() + + +admin.site.unregister(Group) +@admin.register(CustomGroup) +class CustomGroupAdmin(admin.ModelAdmin): + form = CustomGroupAdminForm + list_display = ('name', 'queue') + search_fields = ('name',) + filter_horizontal = ('permissions',) + fieldsets = ( + (None, {'fields': ('name',)}), + ('Permissions', {'fields': ('permissions',)}), + ('Utilisateurs', {'fields': ('users',)}), + ('Options', {'fields': ('queue',)}), + ) + + def save_model(self, request, obj, form, change): + super().save_model(request, obj, form, change) + + def save_related(self, request, form, formsets, change): + super().save_related(request, form, formsets, change) + form.instance.user_set.set(form.cleaned_data['users']) + + admin.site.register(models.Competition, CompetitionExpansion) admin.site.register( models.CompetitionCreationTaskStatus, CompetitionCreationTaskStatusExpansion diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index fb7e5f068..edfd37283 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -13,7 +13,7 @@ from celery_config import app, app_for_vhost from leaderboards.models import SubmissionScore -from profiles.models import User, Organization +from profiles.models import CustomGroup, User, Organization from utils.data import PathWrapper from utils.storage import BundleStorage from PIL import Image @@ -55,6 +55,15 @@ class Competition(models.Model): make_programs_available = models.BooleanField(default=False) make_input_data_available = models.BooleanField(default=False) + participant_groups = models.ManyToManyField( + CustomGroup, + blank=True, + related_name='competitions', + verbose_name="group of participants", + help_text="Competition owner being able to create groups of users." + ) + + queue = models.ForeignKey('queues.Queue', on_delete=models.SET_NULL, null=True, blank=True, related_name='competitions') diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 4c5e495b8..b610b77bd 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -116,8 +116,17 @@ ] MAX_EXECUTION_TIME_LIMIT = int(os.environ.get('MAX_EXECUTION_TIME_LIMIT', 600)) # time limit of the default queue +def _send_to_compute_worker(submission, is_scoring, target_group=None): + logger.info( + "Site Worker ==> STARTING submission_id=%s is_scoring=%s " + "target_group_param=%s submission.queue=%s parent=%s", + submission.pk, + is_scoring, + getattr(target_group, "pk", None), + getattr(getattr(submission, "queue", None), "name", None), + getattr(submission.parent, "pk", None) if submission.parent else None, + ) -def _send_to_compute_worker(submission, is_scoring): run_args = { "user_pk": submission.owner.pk, "submissions_api_url": settings.SUBMISSIONS_API_URL, @@ -127,128 +136,359 @@ def _send_to_compute_worker(submission, is_scoring): "id": submission.pk, "is_scoring": is_scoring, } + logger.debug("Initial run_args: %s", run_args) + + if submission.phase.competition.enable_detailed_results and not getattr(submission.detailed_result, "name", None): + logger.info("Creating empty detailed_result for submission %s", submission.pk) + submission.detailed_result.save("detailed_results.html", ContentFile(b"")) + submission.save(update_fields=["detailed_result"]) + + if not getattr(submission.prediction_result, "name", None): + logger.info("Creating empty prediction_result for submission %s", submission.pk) + submission.prediction_result.save("prediction_result.zip", ContentFile(b"")) + submission.save(update_fields=["prediction_result"]) - if not submission.detailed_result.name and submission.phase.competition.enable_detailed_results: - submission.detailed_result.save('detailed_results.html', ContentFile(''.encode())) # must encode here for GCS - submission.save(update_fields=['detailed_result']) - if not submission.prediction_result.name: - submission.prediction_result.save('prediction_result.zip', ContentFile(''.encode())) # must encode here for GCS - submission.save(update_fields=['prediction_result']) - if not submission.scoring_result.name: - submission.scoring_result.save('scoring_result.zip', ContentFile(''.encode())) # must encode here for GCS - submission.save(update_fields=['scoring_result']) + if not getattr(submission.scoring_result, "name", None): + logger.info("Creating empty scoring_result for submission %s", submission.pk) + submission.scoring_result.save("scoring_result.zip", ContentFile(b"")) + submission.save(update_fields=["scoring_result"]) submission = Submission.objects.get(id=submission.id) task = submission.task + logger.debug("Reloaded submission %s from DB; task=%s", submission.pk, getattr(task, "pk", None)) + + if task is None: + fallback_task = submission.phase.tasks.first() + if fallback_task is not None: + logger.warning( + "Submission %s has no task set. Falling back to first task of phase: task=%s", + submission.pk, + fallback_task.pk, + ) + task = fallback_task + else: + logger.warning( + "Submission %s has no task and phase %s has no tasks. Proceeding with minimal payload.", + submission.pk, + submission.phase.pk, + ) - # priority of scoring tasks is higher, we don't want to wait around for - # many submissions to be scored while we're waiting for results - if is_scoring: - # higher numbers are higher priority - priority = 10 - else: - priority = 0 + priority = 10 if is_scoring else 0 + logger.debug("Computed task priority=%s", priority) if not is_scoring: - run_args['prediction_result'] = make_url_sassy( - path=submission.prediction_result.name, - permission='w' - ) + if getattr(submission.prediction_result, "name", None): + run_args["prediction_result"] = make_url_sassy( + path=submission.prediction_result.name, + permission="w", + ) + logger.debug("Prediction phase: prediction_result=%s", run_args["prediction_result"]) + else: + logger.debug("Prediction phase: no prediction_result.path available for submission %s", submission.pk) else: - if submission.phase.competition.enable_detailed_results: - run_args['detailed_results_url'] = make_url_sassy( + if submission.phase.competition.enable_detailed_results and getattr(submission.detailed_result, "name", None): + run_args["detailed_results_url"] = make_url_sassy( path=submission.detailed_result.name, - permission='w', - content_type='text/html' + permission="w", + content_type="text/html", ) - run_args['prediction_result'] = make_url_sassy( - path=submission.prediction_result.name, - permission='r' - ) - run_args['scoring_result'] = make_url_sassy( - path=submission.scoring_result.name, - permission='w' + logger.debug("Scoring phase: detailed_results_url=%s", run_args["detailed_results_url"]) + + if getattr(submission.prediction_result, "name", None): + run_args["prediction_result"] = make_url_sassy( + path=submission.prediction_result.name, + permission="r", + ) + if getattr(submission.scoring_result, "name", None): + run_args["scoring_result"] = make_url_sassy( + path=submission.scoring_result.name, + permission="w", + ) + logger.debug( + "Scoring phase: prediction_result=%s scoring_result=%s", + run_args.get("prediction_result"), + run_args.get("scoring_result"), ) - if task.ingestion_program: - if (task.ingestion_only_during_scoring and is_scoring) or (not task.ingestion_only_during_scoring and not is_scoring): - run_args['ingestion_program'] = make_url_sassy(task.ingestion_program.data_file.name) + if task is not None: + try: + if getattr(task, "ingestion_program", None): + if ( + (getattr(task, "ingestion_only_during_scoring", False) and is_scoring) + or (not getattr(task, "ingestion_only_during_scoring", False) and not is_scoring) + ): + ingestion_path = getattr(task.ingestion_program, "data_file", None) + if ingestion_path: + run_args["ingestion_program"] = make_url_sassy(ingestion_path.name) + logger.debug("Added ingestion_program=%s", run_args["ingestion_program"]) + else: + logger.debug("Task %s ingestion_program exists but no data_file", task.pk) + + if getattr(task, "input_data", None) and (not is_scoring or getattr(task, "ingestion_only_during_scoring", False)): + input_data_file = getattr(task.input_data, "data_file", None) + if input_data_file: + run_args["input_data"] = make_url_sassy(input_data_file.name) + logger.debug("Added input_data=%s", run_args["input_data"]) + + if is_scoring and getattr(task, "reference_data", None): + reference_file = getattr(task.reference_data, "data_file", None) + if reference_file: + run_args["reference_data"] = make_url_sassy(reference_file.name) + logger.debug("Added reference_data=%s", run_args["reference_data"]) + except Exception: + logger.exception("Error while resolving task assets for submission %s", submission.pk) + else: + logger.debug("No task available for submission %s: skipping ingestion/input/reference attachments", submission.pk) - if task.input_data and (not is_scoring or task.ingestion_only_during_scoring): - run_args['input_data'] = make_url_sassy(task.input_data.data_file.name) + run_args["ingestion_only_during_scoring"] = getattr(task, "ingestion_only_during_scoring", False) - if is_scoring and task.reference_data: - run_args['reference_data'] = make_url_sassy(task.reference_data.data_file.name) + program_data_path = None + if not is_scoring: + if getattr(submission.data, "data_file", None): + program_data_path = submission.data.data_file.name + else: + if task is not None and getattr(task, "scoring_program", None) and getattr(task.scoring_program, "data_file", None): + program_data_path = task.scoring_program.data_file.name - run_args['ingestion_only_during_scoring'] = task.ingestion_only_during_scoring + if program_data_path: + run_args["program_data"] = make_url_sassy(path=program_data_path) + logger.debug("Added program_data=%s", run_args["program_data"]) + else: + logger.debug("No program_data path available for submission %s (is_scoring=%s)", submission.pk, is_scoring) - run_args['program_data'] = make_url_sassy( - path=submission.data.data_file.name if not is_scoring else task.scoring_program.data_file.name + detail_names = ( + SubmissionDetails.DETAILED_OUTPUT_NAMES_PREDICTION + if not is_scoring + else SubmissionDetails.DETAILED_OUTPUT_NAMES_SCORING ) - if not is_scoring: - detail_names = SubmissionDetails.DETAILED_OUTPUT_NAMES_PREDICTION - else: - detail_names = SubmissionDetails.DETAILED_OUTPUT_NAMES_SCORING - for detail_name in detail_names: run_args[detail_name] = create_detailed_output_file(detail_name, submission) + logger.debug("Created detailed output %s=%s", detail_name, run_args[detail_name]) - logger.info(f"Task data for submission id = {submission.id}") - logger.info(run_args) + logger.info("Final task payload ready for submission %s", submission.pk) + logger.debug("run_args=%s", run_args) - # Pad timelimit so worker has time to cleanup - time_padding = 60 * 20 # 20 minutes + time_padding = 60 * 20 time_limit = submission.phase.execution_time_limit + time_padding + logger.debug( + "Computed time_limit=%s (base=%s padding=%s)", + time_limit, + submission.phase.execution_time_limit, + time_padding, + ) - if submission.phase.competition.queue: # if the competition is running on a custom queue, not the default queue - submission.queue_name = submission.phase.competition.queue.name or '' - run_args['execution_time_limit'] = submission.phase.execution_time_limit # use the competition time limit - submission.save() + target_vhost = None + try: + if target_group: + if getattr(target_group, "queue", None): + run_args["queue"] = target_group.queue.name + target_vhost = getattr(target_group.queue, "vhost", None) + try: + submission.queue = target_group.queue + submission.save(update_fields=["queue"]) + logger.debug("Persisted submission.queue=%s for submission %s", getattr(target_group.queue, "name", None), submission.pk) + except Exception: + logger.exception("Failed to persist submission.queue for submission %s", submission.pk) + logger.info( + "Submission %s forced to group %s queue=%s vhost=%s", + submission.pk, + getattr(target_group, "pk", None), + run_args.get("queue"), + target_vhost, + ) + else: + persisted_queue = None + try: + persisted_queue = getattr(submission, "queue", None) + except Exception: + persisted_queue = None + + if persisted_queue: + try: + run_args["queue"] = persisted_queue.name + target_vhost = getattr(persisted_queue, "vhost", None) + logger.info( + "Submission %s resolved queue=%s vhost=%s (by persisted submission.queue)", + submission.pk, + run_args.get("queue"), + target_vhost, + ) + except Exception: + logger.exception("Error reading persisted submission.queue for submission %s", submission.pk) + else: + competition = submission.phase.competition + user_group_ids = list(submission.owner.groups.values_list("id", flat=True)) + logger.debug( + "User %s group ids for competition %s: %s", + submission.owner.pk, + competition.pk, + user_group_ids, + ) + + comp_user_groups_qs = ( + competition.participant_groups + .filter(id__in=user_group_ids) + .select_related("queue") + ) + + group = ( + comp_user_groups_qs.filter(queue__isnull=False).first() + or comp_user_groups_qs.first() + ) + + if group and getattr(group, "queue", None): + run_args["queue"] = group.queue.name + target_vhost = getattr(group.queue, "vhost", None) + logger.info( + "Submission %s resolved group=%s queue=%s vhost=%s (by owner membership)", + submission.pk, + group.pk, + group.queue.name, + target_vhost, + ) + else: + logger.debug( + "Submission %s: no matching group with queue for competition %s via owner membership", + submission.pk, + competition.pk, + ) + + if submission.parent: + try: + sibling = ( + Submission.objects + .filter(parent=submission.parent) + .exclude(pk=submission.pk) + .filter(queue__isnull=False) + .select_related("queue") + .first() + ) + if sibling and getattr(sibling, "queue", None): + run_args["queue"] = sibling.queue.name + target_vhost = getattr(sibling.queue, "vhost", None) + # optionally persist to current child for future + try: + submission.queue = sibling.queue + submission.save(update_fields=["queue"]) + logger.debug("Persisted submission.queue from sibling=%s to submission=%s", sibling.pk, submission.pk) + except Exception: + logger.exception("Failed to persist submission.queue (from sibling) for submission %s", submission.pk) + + logger.info( + "Submission %s resolved queue=%s vhost=%s (by sibling child %s)", + submission.pk, + run_args.get("queue"), + target_vhost, + sibling.pk, + ) + except Exception: + logger.exception("Error while trying to resolve queue from sibling for submission %s", submission.pk) + except Exception: + logger.exception( + "Error while resolving routing for submission %s", submission.pk + ) + + if target_vhost is None: + comp_queue = getattr(submission.phase.competition, "queue", None) + if comp_queue: + run_args["queue"] = getattr(comp_queue, "name", None) + target_vhost = getattr(comp_queue, "vhost", None) + logger.info( + "Fallback to competition queue=%s vhost=%s for submission %s", + run_args.get("queue"), + target_vhost, + submission.pk, + ) - # Send to special queue? Using `celery_app` var name here since we'd be overriding the imported `app` - # variable above - celery_app = app_or_default() - with celery_app.connection() as new_connection: - new_connection.virtual_host = str(submission.phase.competition.queue.vhost) - task = celery_app.send_task( - 'compute_worker_run', + task_obj = None + try: + if target_vhost: + logger.info( + "Sending task to compute-worker with explicit vhost=%s submission=%s", + target_vhost, + submission.pk, + ) + celery_app = app_or_default() + with celery_app.connection() as new_connection: + new_connection.virtual_host = str(target_vhost) + task_obj = celery_app.send_task( + "compute_worker_run", + args=(run_args,), + queue="compute-worker", + soft_time_limit=time_limit, + connection=new_connection, + priority=priority, + ) + else: + logger.info( + "Sending task to compute-worker with default vhost submission=%s", + submission.pk, + ) + task_obj = app.send_task( + "compute_worker_run", args=(run_args,), - queue='compute-worker', + queue="compute-worker", soft_time_limit=time_limit, - connection=new_connection, priority=priority, ) - else: - task = app.send_task( - 'compute_worker_run', - args=(run_args,), - queue='compute-worker', - soft_time_limit=time_limit, - priority=priority, + except Exception: + logger.exception( + "Failed to enqueue compute_worker_run for submission %s", + submission.pk, + ) + task_obj = None + + if task_obj: + submission.celery_task_id = getattr(task_obj, "id", None) + logger.info( + "Task enqueued successfully submission=%s celery_task_id=%s", + submission.pk, + submission.celery_task_id, ) - submission.celery_task_id = task.id if submission.status == Submission.SUBMITTING: - # Don't want to mark an already-prepared submission as "submitted" again, so - # only do this if we were previously "SUBMITTING" submission.status = Submission.SUBMITTED + logger.debug("Submission %s status updated to SUBMITTED", submission.pk) - submission.save() + try: + submission.save() + logger.info("Submission %s saved successfully", submission.pk) + except Exception: + logger.exception( + "Failed to save submission after enqueue submission=%s", + submission.pk, + ) def create_detailed_output_file(detail_name, submission): - # Detail logs like stdout/etc. new_details = SubmissionDetails.objects.create(submission=submission, name=detail_name) new_details.data_file.save(f'{detail_name}.txt', ContentFile(''.encode())) # must encode here for GCS return make_url_sassy(new_details.data_file.name, permission="w") - def run_submission(submission_pk, tasks=None, is_scoring=False): - task_ids = [t.id for t in tasks] if tasks else None + """ + Legacy-compatible handling: + - tasks may be None + - tasks may contain None + - in those cases we fallback to phase.tasks (as legacy did) + """ + + task_ids = None + + if tasks: + cleaned_tasks = [t for t in tasks if t is not None] + + if cleaned_tasks: + task_ids = [t.id for t in cleaned_tasks] + else: + # behave like legacy: treat as None + task_ids = None + return _run_submission.apply_async((submission_pk, task_ids, is_scoring)) + def send_submission_message(submission, data): from channels.layers import get_channel_layer channel_layer = get_channel_layer() @@ -304,6 +544,68 @@ def _run_submission(submission_pk, task_pks=None, is_scoring=False): tasks = tasks.order_by('pk') + '''New section for group submission child ''' + try: + assigned_group = None + + if submission.parent is None: + groups_with_queue_qs = resolve_competition_groups(submission) + groups_with_queue = list(groups_with_queue_qs) + + if len(groups_with_queue) > 1: + submission.has_children = True + submission.save() + send_parent_status(submission) + + # If there are multiple tasks, create one child per (group, task), + for group in groups_with_queue: + if len(tasks) > 1: + for task_obj in tasks: + child_sub = Submission( + owner=submission.owner, + phase=submission.phase, + data=submission.data, + participant=submission.participant, + parent=submission, + task=task_obj, + fact_sheet_answers=submission.fact_sheet_answers, + queue=group.queue, + ) + child_sub.save(ignore_submission_limit=True) + send_child_id(submission, child_sub.id) + + try: + _send_to_compute_worker(child_sub, is_scoring, target_group=group) + except Exception: + logger.exception("Failed to send child submission %s to compute worker for group %s", child_sub.pk, getattr(group, 'pk', None)) + else: + child_sub = Submission( + owner=submission.owner, + phase=submission.phase, + data=submission.data, + participant=submission.participant, + parent=submission, + task=tasks[0], + fact_sheet_answers=submission.fact_sheet_answers, + queue=group.queue, + ) + child_sub.save(ignore_submission_limit=True) + send_child_id(submission, child_sub.id) + try: + _send_to_compute_worker(child_sub, is_scoring, target_group=group) + except Exception: + logger.exception("Failed to send child submission %s to compute worker for group %s", child_sub.pk, getattr(group, 'pk', None)) + return + + if len(groups_with_queue) == 1: + assigned_group = groups_with_queue[0] + + except Exception: + logger.exception("Error resolving participant groups for submission %s", submission.pk) + '''END BLOCK''' + + + if len(tasks) > 1: # The initial submission object becomes the parent submission and we create children for each task submission.has_children = True @@ -320,18 +622,34 @@ def _run_submission(submission_pk, task_pks=None, is_scoring=False): participant=submission.participant, parent=submission, task=task, - fact_sheet_answers=submission.fact_sheet_answers + fact_sheet_answers=submission.fact_sheet_answers, + queue=(assigned_group.queue if assigned_group else None), ) child_sub.save(ignore_submission_limit=True) - _send_to_compute_worker(child_sub, is_scoring=False) + + try: + _send_to_compute_worker(child_sub, is_scoring=is_scoring, target_group=assigned_group) + logger.info("Dispatched child submission %s for task %s (parent=%s)", child_sub.pk, task.pk, submission.pk) + except Exception: + logger.exception( + "Failed to send child submission %s to compute worker for group %s", + child_sub.pk, + getattr(assigned_group, 'pk', None) + ) + send_child_id(submission, child_sub.id) - else: - # The initial submission object is the only submission - if not submission.task: - submission.task = tasks[0] - submission.save() - _send_to_compute_worker(submission, is_scoring) + else: + try: + _send_to_compute_worker(submission, is_scoring=is_scoring, target_group=assigned_group) + logger.info("Dispatched original submission %s for single-task execution (assigned_group=%s)", + submission.pk, getattr(assigned_group, 'pk', None)) + except Exception: + logger.exception( + "Failed to send submission %s to compute worker for group %s", + submission.pk, + getattr(assigned_group, 'pk', None) + ) @app.task(queue='site-worker', soft_time_limit=60 * 60) # 1 hour timeout def unpack_competition(status_pk): @@ -780,3 +1098,16 @@ def submission_status_cleanup(): sub.parent.cancel(status=Submission.FAILED) else: sub.cancel(status=Submission.FAILED) + + +def resolve_competition_groups(submission): + competition = submission.phase.competition + + user_group_ids = submission.owner.groups.values_list("id", flat=True) + + return ( + competition.participant_groups + .select_related("queue") + .filter(id__in=user_group_ids) + .exclude(queue__isnull=True) + ) diff --git a/src/apps/competitions/unpackers/base_unpacker.py b/src/apps/competitions/unpackers/base_unpacker.py index bf9f686db..5fda9ef7e 100644 --- a/src/apps/competitions/unpackers/base_unpacker.py +++ b/src/apps/competitions/unpackers/base_unpacker.py @@ -6,6 +6,7 @@ from django.core.files import File from django.test import RequestFactory from django.utils import timezone +from django.db import transaction from api.serializers.competitions import CompetitionSerializer from api.serializers.leaderboards import LeaderboardSerializer @@ -14,6 +15,7 @@ from datasets.models import Data from queues.models import Queue from tasks.models import Task, Solution +from profiles.models import CustomGroup from utils.storage import md5 from .utils import CompetitionUnpackingException, zip_if_directory @@ -27,6 +29,7 @@ def __init__(self, competition_yaml, temp_directory, creator): self.created_tasks = [] self.created_solutions = [] self.created_datasets = [] + self.created_groups = [] # We'll make a fake request to pass to DRF serializers for request.user context self.fake_request = RequestFactory() @@ -198,6 +201,7 @@ def _unpack_terms(self): raise NotImplementedError def _unpack_image(self): + try: image_name = self.competition_yaml['image'] except KeyError: @@ -205,6 +209,125 @@ def _unpack_image(self): self.competition['logo'] = self._read_image(image_name) + def _unpack_groups(self): + """ + Parse groups from YAML. + Expected format: + groups: + - name: "Group A" + queue: "Queue Name" # optional + """ + raw = self.competition_yaml.get('groups') + + if not raw: + self.competition['participant_groups_raw'] = [] + return + + parsed = [] + + for g in raw: + name = (g.get('name') or '').strip() + + if not name: + raise CompetitionUnpackingException( + 'Each group must have a non-empty "name" field.' + ) + + parsed.append({ + 'name': name, + 'queue_field': g.get('queue') + }) + + self.competition['participant_groups_raw'] = parsed + + + + def _save_groups(self, competition): + """ + Create CustomGroup objects and attach them to competition. + If group already exists, it is reused. + """ + + groups_raw = self.competition.get('participant_groups_raw') or [] + + if not groups_raw: + return + + import uuid + + with transaction.atomic(): + for grp in groups_raw: + name = grp['name'].strip() + queue_field = grp.get('queue_field') + + if not name: + raise CompetitionUnpackingException( + "Participant group name cannot be empty." + ) + queue_obj = None + + if queue_field: + if isinstance(queue_field, int) or ( + isinstance(queue_field, str) and queue_field.isdigit() + ): + queue_obj = Queue.objects.filter(pk=int(queue_field)).first() + + else: + try: + uuid_value = uuid.UUID(str(queue_field)) + queue_obj = Queue.objects.filter(vhost=uuid_value).first() + except ValueError: + queue_obj = None + + if not queue_obj: + queues = Queue.objects.filter(name=queue_field) + + if queues.count() > 1: + raise CompetitionUnpackingException( + f"Multiple queues found with name '{queue_field}'. " + f"Use id or UUID instead for group '{name}'." + ) + + queue_obj = queues.first() + + if not queue_obj: + raise CompetitionUnpackingException( + f"Queue '{queue_field}' does not exist " + f"for group '{name}'." + ) + + if not queue_obj.is_public: + organizers = queue_obj.organizers.values_list( + 'username', flat=True + ) + + if ( + queue_obj.owner != self.creator + and self.creator.username not in organizers + ): + raise CompetitionUnpackingException( + f"You do not have access to queue '{queue_field}' " + f"for group '{name}'." + ) + + group, created = CustomGroup.objects.get_or_create( + name=name, + defaults={'queue': queue_obj} + ) + + if not created and queue_field: + if group.queue != queue_obj: + group.queue = queue_obj + group.save() + + if created: + self.created_groups.append(group) + + competition.participant_groups.add(group) + + + + def _unpack_queue(self): # Get Queue by vhost/uuid. If instance not returned, or we don't have access don't set it! vhost = self.competition_yaml.get('queue') @@ -343,6 +466,11 @@ def _save_competition(self): def _clean(self): for dataset in self.created_datasets: dataset.delete() + for group in getattr(self, 'created_groups', []): + try: + group.delete() + except Exception: + pass for task in self.created_tasks: task.delete() for solution in self.created_solutions: @@ -353,7 +481,15 @@ def save(self): self._save_tasks() self._save_solutions() self._save_leaderboards() - return self._save_competition() + self._unpack_groups() + + # Create competition + competition_instance = self._save_competition() + + # Create and attach groups + self._save_groups(competition_instance) + + return competition_instance except Exception as e: self._clean() raise e diff --git a/src/apps/competitions/urls.py b/src/apps/competitions/urls.py index 705c1d77c..afaa59892 100644 --- a/src/apps/competitions/urls.py +++ b/src/apps/competitions/urls.py @@ -13,4 +13,10 @@ path('upload/', views.CompetitionUpload.as_view(), name="upload"), path('public/', views.CompetitionPublic.as_view(), name="public"), path('/detailed_results//', views.CompetitionDetailedResults.as_view(), name="detailed_results"), + + # Groups + path('/groups/create/', views.competition_create_group, name='competition_create_group'), + path('/groups//update/', views.competition_update_group), + path('/groups//delete/', views.competition_delete_group), + ] diff --git a/src/apps/competitions/views.py b/src/apps/competitions/views.py index 33d4b7a78..1ff15f8da 100644 --- a/src/apps/competitions/views.py +++ b/src/apps/competitions/views.py @@ -3,6 +3,22 @@ from django.views.generic import TemplateView, DetailView from .models import Competition, CompetitionParticipant +from django.core.serializers.json import DjangoJSONEncoder + +from django.db.models import Q +import json +from django.contrib.auth.decorators import login_required +from django.views.decorators.http import require_POST +from django.shortcuts import get_object_or_404 +from django.http import JsonResponse, HttpResponseForbidden, HttpResponseBadRequest, HttpResponseRedirect +from django.urls import reverse +from django.db import transaction +from django.contrib import messages + +from django.core.exceptions import PermissionDenied +from profiles.models import CustomGroup, User +from queues.models import Queue + class CompetitionManagement(LoginRequiredMixin, TemplateView): @@ -21,6 +37,50 @@ class CompetitionUpdateForm(LoginRequiredMixin, DetailView): template_name = 'competitions/form.html' queryset = Competition.objects.all() + + def get_context_data(self, **kwargs): + ctx = super().get_context_data(**kwargs) + comp = self.object + + groups_qs = CustomGroup.objects.filter( + Q(id__in=comp.participant_groups.values_list('id', flat=True)) + ).select_related('queue').prefetch_related('user_set') + + participant_user_ids = list( + CompetitionParticipant.objects.filter(competition=comp) + .values_list('user_id', flat=True) + ) + + ctx['available_groups_json'] = json.dumps([ + { + 'id': g.id, + 'name': g.name, + 'queue': g.queue.name if g.queue else None, + 'members': [u.username for u in g.user_set.filter(pk__in=participant_user_ids)], + } + for g in groups_qs + ], cls=DjangoJSONEncoder) + + ctx['selected_group_ids_json'] = json.dumps( + list(comp.participant_groups.values_list('id', flat=True)), + cls=DjangoJSONEncoder + ) + + ctx['available_queues_json'] = json.dumps( + list(Queue.objects.all().values('id', 'name')), + cls=DjangoJSONEncoder + ) + + ctx['available_users_json'] = json.dumps( + list( + User.objects + .filter(pk__in=participant_user_ids, is_active=True) + .values('id', 'username', 'email') + ), + cls=DjangoJSONEncoder + ) + return ctx + def get_object(self, *args, **kwargs): competition = super().get_object(*args, **kwargs) @@ -76,7 +136,7 @@ def get_object(self, *args, **kwargs): # get participants from CompetitionParticipant where user=user and competition=competition is_participant = CompetitionParticipant.objects.filter(user=self.request.user, competition=competition).count() > 0 - # check if secret key provided is valid + # check if secret key provided is valid, valid_secret_key = self.request.GET.get('secret_key') == str(competition.secret_key) if ( @@ -104,3 +164,209 @@ def get_context_data(self, **kwargs): class CompetitionDetailedResults(TemplateView): template_name = 'competitions/detailed_results.html' + + +@login_required +@require_POST +def competition_create_group(request, pk): + competition = get_object_or_404(Competition, pk=pk) + + user = request.user + if not (user.is_superuser or user == competition.created_by or user in competition.collaborators.all()): + return HttpResponseForbidden("Not allowed") + + if request.content_type == 'application/json': + try: + payload = json.loads(request.body.decode()) + except Exception: + return HttpResponseBadRequest("Invalid JSON") + name = (payload.get('name') or '').strip() + queue_id = payload.get('queue_id') + user_ids = payload.get('user_ids') or [] + else: + name = (request.POST.get('name') or '').strip() + queue_id = request.POST.get('queue_id') or None + user_ids = request.POST.getlist('user_ids') or [] + if not user_ids and request.POST.get('user_ids'): + user_ids = [u.strip() for u in request.POST.get('user_ids').split(',') if u.strip()] + + if not name: + return HttpResponseBadRequest("Missing name") + + allowed_user_ids = set( + CompetitionParticipant.objects.filter(competition=competition) + .values_list('user_id', flat=True) + ) + + try: + with transaction.atomic(): + group = CustomGroup(name=name) + if queue_id: + try: + queue = Queue.objects.get(pk=queue_id) + group.queue = queue + except Queue.DoesNotExist: + group.queue = None + group.save() + + user_ids_int = [] + try: + user_ids_int = [int(u) for u in user_ids] + except Exception: + user_ids_int = [] + + if user_ids_int: + invalid = [uid for uid in user_ids_int if uid not in allowed_user_ids] + if invalid: + raise ValueError(f"Some users are not participants of this competition: {invalid}") + + users_qs = User.objects.filter(pk__in=user_ids_int) + group.user_set.set(users_qs) + + competition.participant_groups.add(group) + + members = list(group.user_set.values_list('username', flat=True)) + group_data = { + 'id': group.id, + 'name': group.name, + 'queue': group.queue.name if group.queue else None, + 'members': members, + } + except ValueError as e: + return HttpResponseBadRequest(str(e)) + except Exception as e: + return HttpResponseBadRequest("Error creating group: %s" % str(e)) + + if ( + request.content_type.startswith('application/json') + or request.headers.get('x-requested-with') == 'XMLHttpRequest' + or 'application/json' in request.headers.get('accept', '') + ): + return JsonResponse({'status': 'ok', 'group': group_data}) + + + messages.success(request, "Groupe créé") + return HttpResponseRedirect(reverse('competitions:edit', kwargs={'pk': competition.pk})) + + +@login_required +@require_POST +def competition_update_group(request, pk, group_id): + competition = get_object_or_404(Competition, pk=pk) + group = get_object_or_404(CustomGroup, pk=group_id) + + user = request.user + if not (user.is_superuser or user == competition.created_by or user in competition.collaborators.all()): + return HttpResponseForbidden("Not allowed") + + if not competition.participant_groups.filter(pk=group.pk).exists(): + return HttpResponseBadRequest("Group does not belong to this competition") + + if request.content_type == 'application/json': + try: + payload = json.loads(request.body.decode()) + except Exception: + return HttpResponseBadRequest("Invalid JSON") + name = (payload.get('name') or '').strip() + queue_id = payload.get('queue_id') + user_ids = payload.get('user_ids', []) or [] + else: + name = (request.POST.get('name') or '').strip() + queue_id = request.POST.get('queue_id') or None + user_ids = request.POST.getlist('user_ids[]') or [] + if not user_ids and request.POST.get('user_ids'): + user_ids = [u.strip() for u in request.POST.get('user_ids').split(',') if u.strip()] + + if not name: + return HttpResponseBadRequest("Missing name") + + allowed_user_ids = set( + CompetitionParticipant.objects.filter(competition=competition) + .values_list('user_id', flat=True) + ) + + try: + with transaction.atomic(): + group.name = name + if queue_id: + group.queue = Queue.objects.filter(pk=queue_id).first() + else: + group.queue = None + group.save() + + try: + user_ids_int = [int(u) for u in user_ids] + except Exception: + user_ids_int = [] + + if user_ids_int: + invalid = [uid for uid in user_ids_int if uid not in allowed_user_ids] + if invalid: + raise ValueError(f"Some users are not participants of this competition: {invalid}") + + group.user_set.set(User.objects.filter(pk__in=user_ids_int)) + except ValueError as e: + return HttpResponseBadRequest(str(e)) + except Exception as e: + return HttpResponseBadRequest("Error updating group: %s" % str(e)) + + resp = { + 'status': 'ok', + 'group': { + 'id': group.id, + 'name': group.name, + 'queue': group.queue.name if group.queue else None, + 'members': list(group.user_set.values_list('username', flat=True)), + } + } + + if ( + request.content_type.startswith('application/json') + or request.headers.get('x-requested-with') == 'XMLHttpRequest' + or 'application/json' in request.headers.get('accept', '') + ): + return JsonResponse(resp) + + + messages.success(request, "Groupe modifié") + return HttpResponseRedirect(reverse('competitions:edit', kwargs={'pk': competition.pk})) + + +@login_required +@require_POST +def competition_delete_group(request, pk, group_id): + competition = get_object_or_404(Competition, pk=pk) + + user = request.user + if not ( + user.is_superuser + or user == competition.created_by + or user in competition.collaborators.all() + ): + raise PermissionDenied("Not allowed") + + group = get_object_or_404( + CustomGroup, + pk=group_id, + competitions=competition + ) + + try: + with transaction.atomic(): + competition.participant_groups.remove(group) + group.delete() + + except Exception as e: + return HttpResponseBadRequest(f"Error deleting group: {str(e)}") + + if ( + request.content_type.startswith("application/json") + or request.headers.get("x-requested-with") == "XMLHttpRequest" + or "application/json" in request.headers.get("accept", "") + ): + return JsonResponse({"status": "ok", "group_id": group_id}) + + messages.success(request, "Groupe supprimé") + return HttpResponseRedirect( + reverse("competitions:edit", kwargs={"pk": competition.pk}) + ) \ No newline at end of file diff --git a/src/apps/profiles/models.py b/src/apps/profiles/models.py index 042c7d0a1..006adcc2a 100644 --- a/src/apps/profiles/models.py +++ b/src/apps/profiles/models.py @@ -1,7 +1,9 @@ +from queues.models import Queue import uuid from django.contrib.auth.models import PermissionsMixin, AbstractBaseUser, UserManager from django.db import models +from django.contrib.auth.models import Group from django.utils.timezone import now from django.utils.text import slugify from utils.data import PathWrapper @@ -343,3 +345,23 @@ class Membership(models.Model): class Meta: ordering = ["date_joined"] + + +class CustomGroup(Group): + + queue = models.ForeignKey(Queue, + null=True, + blank=True, + on_delete=models.SET_NULL, + related_name='custom_groups', + verbose_name="Queue assignée au groupe", + help_text="Queue à utiliser pour les utilisateurs membres de ce groupe (si définie)." + ) + + class Meta: + verbose_name = "Group" + verbose_name_plural = "Groups" + + def __str__(self): + return self.name + \ No newline at end of file diff --git a/src/settings/base.py b/src/settings/base.py index ed5b978d7..b1cf1f201 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -328,6 +328,48 @@ # ============================================================================= LOGGING_CONFIG = None +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + + 'formatters': { + 'verbose': { + 'format': '[{asctime}] {levelname} {processName} {name}:{lineno} {message}', + 'style': '{', + }, + }, + + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout', + 'formatter': 'verbose', + }, + }, + + 'root': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, + + 'loggers': { + 'celery': { + 'handlers': ['console'], + 'level': 'INFO', + 'propagate': False, + }, + 'django': { + 'handlers': ['console'], + 'level': 'INFO', + 'propagate': False, + }, + '__main__': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, + }, +} + # This makes Celery not override the default logger that is configured for the project @signals.setup_logging.connect diff --git a/src/static/riot/competitions/detail/leaderboards.tag b/src/static/riot/competitions/detail/leaderboards.tag index 11191123e..4049889dd 100644 --- a/src/static/riot/competitions/detail/leaderboards.tag +++ b/src/static/riot/competitions/detail/leaderboards.tag @@ -26,11 +26,21 @@ + Task: { task.name } + + + + + + + { group.name } + + # Participant @@ -38,8 +48,10 @@ ID {column.title} + + + - @@ -90,14 +102,11 @@ } self.bold_class = function(column, submission){ - // Return `text-bold` if submission has - // more than one scores and score index == leaderbaord.primary_index - // otherwise return empty string - return_class = '' // default class value - if(column.task_id != -1){ // factsheet check - if(submission.scores.length > 1){ // score length check + return_class = '' + if(column.task_id != -1){ + if(submission.scores.length > 1){ let column_index = _.get(column, 'index') - if(column_index === self.selected_leaderboard.primary_index){ // column index check + if(column_index === self.selected_leaderboard.primary_index){ return_class = 'text-bold' } } @@ -153,8 +162,8 @@ CODALAB.api.get_leaderboard_for_render(self.phase_id) .done(responseData => { self.selected_leaderboard = responseData + self.groups = responseData.groups || [] self.columns = [] - // Make fake task and columns for Metadata so it can be filtered like columns if(self.selected_leaderboard.fact_sheet_keys){ let fake_metadata_task = { id: -1, @@ -171,25 +180,16 @@ self.selected_leaderboard.tasks.unshift(fake_metadata_task) } for(task of self.selected_leaderboard.tasks){ - for(column of task.columns){ column.task_id = task.id self.columns.push(column) } - // -1 id is used for fact sheet answers - if(self.enable_detailed_results & self.show_detailed_results_in_leaderboard & task.id != -1){ - self.columns.push({ - task_id: task.id, - title: "Detailed Results" - }) - task.colWidth += 1 - } } self.filter_columns() $('#leaderboardTable').tablesort() self.update() }) - } + } self.get_detailed_result_submisison_id = function(column, submisison){ for (index in submisison.detailed_results) { diff --git a/src/static/riot/competitions/editor/_participation.tag b/src/static/riot/competitions/editor/_participation.tag index 2a5463527..2e916b480 100644 --- a/src/static/riot/competitions/editor/_participation.tag +++ b/src/static/riot/competitions/editor/_participation.tag @@ -1,150 +1,683 @@ -
-
- - -
-
-
- - + +
+ + +
+
+
+ + +
+
+
+
+ + +
+
+ + + +
+ +

A list of emails (one per line) of users who do not require competition organizer's approval to enter this competition.

+
+ Note:
+ Only valid emails are allowed
+ Empty lines are not allowed +
+ +
+
+ + + +
+ +
+ +
+ +
+
+
+
+ +
+ +
+
+ Queue: { group.queue || "Aucune" } + Membres: { group.members && group.members.length > 0 ? group.members.length : 0 } +
+ +
+ + { m } + +
+ +
+ + +
+ +
+
+
+ + + + +