Extractions, calculations and optimizations + some documentation

Makefile

@@ -14,8 +14,11 @@ download:
 	python -m src.download
 
 transform:
+ifeq ($(ALLOW_DUPES), true)
+	python -m src.transform --allow-dupes
+else
 	python -m src.transform
-
+endif
 
 all: download transform
 

data/examples/output/sessions.json

@@ -19,8 +19,9 @@
         "end": null,
         "talks_in_parallel": null,
         "talks_after": null,
-        "next_talk_code": null,
-        "prev_talk_code": null,
+        "talks_before": null,
+        "next_talk": null,
+        "prev_talk": null,
         "website_url": "https://ep2024.europython.eu/session/this-is-a-test-talk-from-a-test-speaker-about-a-test-topic"
     },
     "B8CD4F": {
@@ -43,8 +44,9 @@
         "end": null,
         "talks_in_parallel": null,
         "talks_after": null,
-        "next_talk_code": null,
-        "prev_talk_code": null,
+        "talks_before": null,
+        "next_talk": null,
+        "prev_talk": null,
         "website_url": "https://ep2024.europython.eu/session/a-talk-with-shorter-title"
     }
 }

src/download.py

@@ -19,6 +19,9 @@
     "speakers?questions=all",
 ]
 
+if not Config.raw_path.exists():
+    Config.raw_path.mkdir(parents=True)
+
 for resource in resources:
     url = base_url + f"{resource}"
 

src/transform.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 import json
+import sys
 from datetime import datetime
 
 from pydantic import BaseModel, Field, model_validator
@@ -30,9 +33,9 @@ class SubmissionState:
 class PretalxAnswer(BaseModel):
     question_text: str
     answer_text: str
-    answer_file: str | None
-    submission_id: str | None
-    speaker_id: str | None
+    answer_file: str | None = None
+    submission_id: str | None = None
+    speaker_id: str | None = None
 
     @model_validator(mode="before")
     @classmethod
@@ -48,8 +51,8 @@ def extract(cls, values):
 class PretalxSpeaker(BaseModel):
     code: str
     name: str
-    biography: str | None
-    avatar: str | None
+    biography: str | None = None
+    avatar: str | None = None
     slug: str
     answers: list[PretalxAnswer] = Field(..., exclude=True)
     submissions: list[str]
@@ -93,7 +96,7 @@ class PretalxSubmission(BaseModel):
     speakers: list[str]  # We only want the code, not the full info
     submission_type: str
     slug: str
-    track: str | None
+    track: str | None = None
     state: str
     abstract: str
     answers: list[PretalxAnswer] = Field(..., exclude=True)
@@ -105,14 +108,16 @@ class PretalxSubmission(BaseModel):
 
     # This is embedding a slot inside a submission for easier lookup later
     room: str | None = None
-    start: datetime | None = None
-    end: datetime | None = None
+    start: datetime | str | None = None
+    end: datetime | str | None = None
 
     # TODO: once we have schedule data then we can prefill those in the code here
+    # These are added after the model is created
     talks_in_parallel: list[str] | None = None
     talks_after: list[str] | None = None
-    next_talk_code: str | None = None
-    prev_talk_code: str | None = None
+    talks_before: list[str] | None = None
+    next_talk: str | None = None
+    prev_talk: str | None = None
 
     website_url: str | None = None
 
@@ -153,9 +158,21 @@ def extract(cls, values):
         if isinstance(values["duration"], int):
             values["duration"] = str(values["duration"])
 
+        if cls.is_publishable and values["slot"]:
+            slot = values["slot"]
+
+            if isinstance(slot["room"], dict):
+                values["room"] = slot["room"]["en"]
+
+            if slot["start"]:
+                values["start"] = datetime.fromisoformat(slot["start"])
+                values["end"] = datetime.fromisoformat(slot["end"])
+
         slug = slugify(values["title"])
         values["slug"] = slug
-        values["website_url"] = f"https://ep2024.europython.eu/session/{slug}"
+        values["website_url"] = (
+            f"https://ep{Config.event.split('-')[1]}.europython.eu/session/{slug}"
+        )
 
         return values
 
@@ -171,6 +188,114 @@ def is_confirmed(self):
     def is_publishable(self):
         return self.is_accepted or self.is_confirmed
 
+    @staticmethod
+    def set_talks_in_parallel(
+        submission: PretalxSubmission, all_sessions: dict[str, PretalxSubmission]
+    ):
+        parallel = []
+        for session in all_sessions.values():
+            if (
+                session.code == submission.code
+                or session.start is None
+                or submission.start is None
+            ):
+                continue
+
+            # If they intersect, they are in parallel
+            if session.start < submission.end and session.end > submission.start:
+                parallel.append(session.code)
+
+        submission.talks_in_parallel = parallel
+
+    @staticmethod
+    def set_talks_after(
+        submission: PretalxSubmission, all_sessions: dict[str, PretalxSubmission]
+    ):
+        # Sort sessions based on start time, early first
+        all_sessions_sorted = sorted(
+            all_sessions.values(), key=lambda x: (x.start is None, x.start)
+        )
+
+        # Filter out sessions
+        remaining_sessions = [
+            session
+            for session in all_sessions_sorted
+            if session.start is not None
+            and session.start >= submission.end
+            and session.code not in submission.talks_in_parallel
+            and session.code != submission.code
+            and submission.start.day == session.start.day
+            and not submission.submission_type
+            == session.submission_type
+            == "Announcements"
+        ]
+
+        # Add sessions to the list if they are in different rooms
+        seen_rooms = set()
+        unique_sessions = []
+
+        for session in remaining_sessions:
+            if session.room not in seen_rooms:
+                unique_sessions.append(session)
+                seen_rooms.add(session.room)
+
+        # If there is a keynote next, only show that
+        if any(s.submission_type == "Keynote" for s in unique_sessions):
+            unique_sessions = [
+                s for s in unique_sessions if s.submission_type == "Keynote"
+            ]
+
+        # Set the next talks in all rooms
+        submission.talks_after = [session.code for session in unique_sessions]
+
+        # Set the next talk in the same room
+        for session in unique_sessions:
+            if session.room == submission.room:
+                submission.next_talk = session.code
+                break
+
+    @staticmethod
+    def set_talks_before(
+        submission: PretalxSubmission, all_sessions: dict[str, PretalxSubmission]
+    ):
+        # Sort sessions based on start time, late first
+        all_sessions_sorted = sorted(
+            all_sessions.values(),
+            key=lambda x: (x.start is None, x.start),
+            reverse=True,
+        )
+
+        remaining_sessions = [
+            session
+            for session in all_sessions_sorted
+            if session.start is not None
+            and session.code not in submission.talks_in_parallel
+            and session.start <= submission.start
+            and session.code != submission.code
+            and submission.start.day == session.start.day
+            and session.submission_type != "Announcements"
+        ]
+
+        seen_rooms = set()
+        unique_sessions = []
+
+        for session in remaining_sessions:
+            if session.room not in seen_rooms:
+                unique_sessions.append(session)
+                seen_rooms.add(session.room)
+
+        submission.talks_before = [session.code for session in unique_sessions]
+
+        for session in unique_sessions:
+            if session.room == submission.room:
+                submission.prev_talk = session.code
+                break
+
+    def model_dump(self):
+        self.start = self.start.isoformat() if self.start else None
+        self.end = self.end.isoformat() if self.end else None
+        return super().model_dump()
+
 
 def parse_submissions() -> list[PretalxSubmission]:
     """
@@ -209,33 +334,73 @@ def publishable_speakers(accepted_proposals: set[str]) -> dict[str, PretalxSpeak
     return output
 
 
-def save_publishable_sessions():
+def save_publishable_sessions(publishable: dict[str, PretalxSubmission]):
     path = Config.public_path / "sessions.json"
 
-    publishable = publishable_submissions()
+    for sub in publishable.values():
+        if sub.start is not None:
+            PretalxSubmission.set_talks_in_parallel(sub, publishable)
+            PretalxSubmission.set_talks_after(sub, publishable)
+            PretalxSubmission.set_talks_before(sub, publishable)
 
     data = {k: v.model_dump() for k, v in publishable.items()}
     with open(path, "w") as fd:
         json.dump(data, fd, indent=2)
 
 
-def save_publishable_speakers():
+def save_publishable_speakers(publishable: dict[str, PretalxSubmission]):
     path = Config.public_path / "speakers.json"
 
-    publishable = publishable_submissions()
     speakers = publishable_speakers(publishable.keys())
 
     data = {k: v.model_dump() for k, v in speakers.items()}
     with open(path, "w") as fd:
         json.dump(data, fd, indent=2)
 
 
+def save_all(all_sessions: dict[str, PretalxSubmission]):
+    if not Config.public_path.exists():
+        Config.public_path.mkdir(parents=True)
+
+    save_publishable_sessions(all_sessions)
+    save_publishable_speakers(all_sessions)
+
+
+def check_duplicate_slugs(all_sessions: dict[str, PretalxSubmission]) -> bool:
+    all_speakers = publishable_speakers(all_sessions.keys())
+
+    session_slugs = [s.slug for s in all_sessions.values()]
+    speaker_slugs = [s.slug for s in all_speakers.values()]
+
+    session_duplicates = [
+        slug for slug in set(session_slugs) if session_slugs.count(slug) > 1
+    ]
+    speaker_duplicates = [
+        slug for slug in set(speaker_slugs) if speaker_slugs.count(slug) > 1
+    ]
+
+    if session_duplicates or speaker_duplicates:
+        print("Found duplicate slugs:")
+        for slug in session_duplicates:
+            print(f"Session: {slug}")
+        for slug in speaker_duplicates:
+            print(f"Speaker: {slug}")
+        return False
+    return True
+
+
 if __name__ == "__main__":
+    print(f"Transforming {Config.event} data...")
     print("Checking for duplicate slugs...")
-    assert len(set(s.slug for s in publishable_submissions().values())) == len(
-        publishable_submissions()
-    )
+
+    all_sessions = publishable_submissions()
+
+    if not check_duplicate_slugs(all_sessions) and (
+        len(sys.argv) <= 1 or sys.argv[1] != "--allow-dupes"
+    ):
+        print("Exiting. Use ``make transform ALLOW_DUPES=true`` to continue.")
+        sys.exit(1)
+
     print("Saving publishable data...")
-    save_publishable_sessions()
-    save_publishable_speakers()
+    save_all(all_sessions)
     print("Done")