Extractions, calculations and optimizations + some documentation

Makefile

@@ -14,8 +14,11 @@ download:
 	python -m src.download
 
 transform:
+ifeq ($(ALLOW_DUPES), true)
+	python -m src.transform --allow-dupes
+else
 	python -m src.transform
-
+endif
 
 all: download transform
 

README.md

@@ -1,2 +1,37 @@
 # programapi
-Program API
+
+This project downloads, processes, saves, and serves the static JSON files containing details of accepted speakers and submissions via an API.
+
+What this project does step-by-step:
+
+1. Downloads the Pretalx speaker and submission data, and saves it as JSON files.
+2. Transforms the JSON files into a format that is easier to work with and OK to serve publicly. This includes removing unnecessary/private fields, and adding new fields.
+3. Serves the JSON files via an API.
+
+## Installation
+
+1. Clone the repository.
+2. Install the dependency management tool: ``make deps/pre``
+3. Install the dependencies: ``make deps/install``
+4. Set up ``pre-commit``: ``make pre-commit``
+
+## Configuration
+
+You can change the event in the [``config.py``](src/config.py) file. It is set to ``europython-2024`` right now.
+
+## Usage
+
+- Run the whole process: ``make all``
+- Run only the download process: ``make download``
+- Run only the transformation process: ``make transform``
+
+**Note:** Don't forget to set the ``PRETALX_TOKEN`` environment variable before running the download process. And please don't make too many requests to the Pretalx API, it might get angry 🤪
+
+## API
+
+The API is served at ``programapi24.europython.eu/2024``. It has two endpoints (for now):
+
+- ``/speakers.json``: Returns the list of confirmed speakers.
+- ``/sessions.json``: Returns the list of confirmed sessions.
+
+**Note:** See [this page](data/examples/README.md) for the explanations of the fields in the returned JSON files.

data/examples/README.md

@@ -0,0 +1,123 @@
+# Explaining the output data
+
+**Note:** Some of the fields may be `null` or empty (`""`).
+
+## `sessions.json`
+
+<details>
+<summary>Example session data JSON</summary>
+
+```json
+{
+    "A1B2C3": {
+        "code": "A1B2C3",
+        "title": "Example talk",
+        "speakers": [
+        "B4D5E6",
+        ...
+        ],
+        "submission_type": "Talk",
+        "slug": "example-talk",
+        "track": "Some Track",
+        "state": "confirmed",
+        "abstract": "This is an example talk. It is a great talk.",
+        "tweet": "This is an example talk.",
+        "duration": "60",
+        "level": "intermediate",
+        "delivery": "in-person",
+        "room": "South Hall 2A",
+        "start": "2024-07-10T14:00:00+02:00",
+        "end": "2024-07-10T15:00:00+02:00",
+        "talks_in_parallel": [
+        "F7G8H9",
+        ...
+        ],
+        "talks_after": [
+        "I0J1K2",
+        ...
+        ],
+        "talks_before": [
+        "L3M4N5",
+        ...
+        ],
+        "next_talk": "O6P7Q8",
+        "prev_talk": "R9S0T1",
+        "website_url": "https://ep2024.europython.eu/session/example-talk/"
+    },
+}
+```
+</details>
+
+&nbsp;
+
+The fields are as follows:
+
+| Key                | Type                              | Notes                                                         |
+|--------------------|-----------------------------------|---------------------------------------------------------------|
+| `code`             | `string`                          | Unique identifier for the session                             |
+| `title`            | `string`                          | Title of the session                                          |
+| `speakers`         | `list[string]`                    | List of codes of the speakers                                 |
+| `submission_type`  | `string`                          | Type of the session (e.g. Talk, Workshop, Poster, etc.)       |
+| `slug`             | `string`                          | URL-friendly version of the title                             |
+| `track`            | `string` \| `null`                | Track of the session (e.g. PyData, Web, etc.)                 |
+| `state`            | `string`                          | State of the session (e.g. confirmed, canceled, etc.)         |
+| `abstract`         | `string`                          | Abstract of the session                                       |
+| `tweet`            | `string`                          | Tweet-length description of the session                       |
+| `duration`         | `string`                          | Duration of the session in minutes                            |
+| `level`            | `string`                          | Level of the session (e.g. beginner, intermediate, advanced)  |
+| `delivery`         | `string`                          | Delivery mode of the session (e.g. in-person, remote)         |
+| `room`             | `string` \| `null`                | Room where the session will be held                           |
+| `start`            | `datetime (ISO format)` \| `null` | Start time of the session                                     |
+| `end`              | `datetime (ISO format)` \| `null` | End time of the session                                       |
+| `talks_in_parallel`| `list[string]` \| `null`          | List of codes of sessions happening in parallel               |
+| `talks_after`      | `list[string]` \| `null`          | List of codes of sessions happening after this session        |
+| `talks_before`     | `list[string]` \| `null`          | List of codes of sessions happening before this session       |
+| `next_talk`        | `string` \| `null`                | Code of the next session in the same room                     |
+| `prev_talk`        | `string` \| `null`                | Code of the previous session in the same room                 |
+| `website_url`      | `string`                          | URL of the session on the conference website                  |
+
+&nbsp;
+
+## `speakers.json`
+
+<details>
+<summary>Example speaker data JSON</summary>
+
+```json
+{
+  "B4D5E6": {
+    "code": "B4D5E6",
+    "name": "A Speaker",
+    "biography": "Some bio",
+    "avatar": "https://pretalx.com/media/avatars/picture.jpg",
+    "slug": "a-speaker",
+    "submissions": [
+      "A1B2C3",
+      ...
+    ],
+    "affiliation": "A Company",
+    "homepage": "https://example.com",
+    "twitter": "example",
+    "mastodon": "example"
+  },
+  ...
+}
+```
+</details>
+
+&nbsp;
+
+The fields are as follows:
+
+| Key            | Type               | Notes                                                                 |
+|----------------|--------------------|-----------------------------------------------------------------------|
+| `code`         | `string`           | Unique identifier for the speaker                                     |
+| `name`         | `string`           | Name of the speaker                                                   |
+| `biography`    | `string` \| `null` | Biography of the speaker                                              |
+| `avatar`       | `string` \| `null` | URL of the speaker's avatar                                           |
+| `slug`         | `string`           | URL-friendly version of the name                                      |
+| `submissions`  | `list[string]`     | List of codes of the sessions the speaker is speaking at              |
+| `affiliation`  | `string` \| `null` | Affiliation of the speaker                                            |
+| `homepage`     | `string` \| `null` | URL of the speaker's homepage                                         |
+| `twitter`      | `string` \| `null` | Twitter handle of the speaker                                         |
+| `mastodon`     | `string` \| `null` | Mastodon handle of the speaker                                        |

data/examples/output/sessions.json

@@ -19,8 +19,9 @@
         "end": null,
         "talks_in_parallel": null,
         "talks_after": null,
-        "next_talk_code": null,
-        "prev_talk_code": null,
+        "talks_before": null,
+        "next_talk": null,
+        "prev_talk": null,
         "website_url": "https://ep2024.europython.eu/session/this-is-a-test-talk-from-a-test-speaker-about-a-test-topic"
     },
     "B8CD4F": {
@@ -43,8 +44,9 @@
         "end": null,
         "talks_in_parallel": null,
         "talks_after": null,
-        "next_talk_code": null,
-        "prev_talk_code": null,
+        "talks_before": null,
+        "next_talk": null,
+        "prev_talk": null,
         "website_url": "https://ep2024.europython.eu/session/a-talk-with-shorter-title"
     }
 }

src/download.py

@@ -19,6 +19,9 @@
     "speakers?questions=all",
 ]
 
+if not Config.raw_path.exists():
+    Config.raw_path.mkdir(parents=True)
+
 for resource in resources:
     url = base_url + f"{resource}"