airflow/Readme.md

@@ -1,9 +1,5 @@
 # Apache Airflow Investigation
 
-## How to run
-in airflow directory:
-`docker-compose up -d`
-
 login:
 airflow:airflow
 
@@ -26,8 +22,3 @@ https://airflow.apache.org/docs/apache-airflow-providers-openfaas/stable/index.h
 
 https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/index.html
 
-
-Lessons learned
-Airflow stack takes A LOT of RAM: ~ 9GB
-
-

airflow/dags/aoe_draft_dag.py

@@ -12,7 +12,7 @@ def extract_draft_data():
     return drafts.download_draft_json(marcel_vs_canuck)
 
 def transform_draft(ti):
-    draft_json=ti.xcom_pull(task_ids='extract_draft1', key='return_value') 
+    draft_json=ti.xcom_pull(task_ids=['extract_draft1'])
     if not draft_json:
         raise ValueError("No value stored in XComs")
     draft_picks=drafts.get_picked_maps(draft_json)
@@ -32,16 +32,13 @@ with DAG("aoe_dag", start_date=datetime(2023, 1, 1), schedule_interval="@daily",
     #T
     transform_data = PythonOperator(
         task_id="transform_draft1",
-        python_callable=transform_draft,
-        do_xcom_push=True
+        python_callable=transform_draft
     )
 
     #L
-    string_to_print = "{{ task_instance.xcom_pull(task_ids='transform_draft1', key='return_value') }}"
     print_output = BashOperator(
         task_id="print1",
-        bash_command="echo ${STRING_TO_PRINT}",
-        env={"STRING_TO_PRINT": string_to_print}
+        bash_command="echo 'bla'"
     )
 
 extract_data >> transform_data >> print_output

jupyter/README.md

@@ -14,23 +14,10 @@ docker run --name jupyter -d --rm \
     -e NB_USER="marcelo" \
     -e CHOWN_HOME=yes \
     -w "/home/marcelo" \
-    -v /mnt/c/Users/Marcelo/git/aoe2-data/jupyter/src:/home/marcelo/src \
+    -v /mnt/c/Users/dechert/git/aoe2-data/jupyter/src:/home/marcelo/src \
     jupyter/scipy-notebook:2023-03-27
 ```
 
-Username agnostic command (does not work):
-```bash
-PATH_TO_JUPYTER_FOLDER="/mnt/c/Users/$(echo whoami)/git/aoe2-data/jupyter/src" docker run --name jupyter -d --rm \
-    -p 8888:8888 \
-    --user root \
-    -e NB_USER="marcelo" \
-    -e CHOWN_HOME=yes \
-    -w "/home/marcelo" \
-    -v ${PATH_TO_JUPYTER_FOLDER}:/home/marcelo/src \
-    jupyter/scipy-notebook:2023-03-27
-```
-
-
 `docker logs -f jupyter` to see the url with secret
 
 configuration:

jupyter/src/AoE-data.ipynb

@@ -9,7 +9,7 @@
    },
    "outputs": [],
    "source": [
-    "%load_ext autoreload"
+    "%load_ext autoreload\n"
    ]
   },
   {
@@ -26,7 +26,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "id": "a121cdfe-114a-4aae-b107-df891338dfcc",
    "metadata": {
     "tags": []
@@ -109,97 +109,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "id": "7b337190-7887-40b7-9d10-b8039f5f2dfd",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th>index</th>\n",
-       "      <th>player</th>\n",
-       "      <th>executingPlayer</th>\n",
-       "      <th>actionType</th>\n",
-       "      <th>chosenOptionId</th>\n",
-       "      <th>isRandomlyChosen</th>\n",
-       "      <th>offset</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>0</th>\n",
-       "      <td>2</td>\n",
-       "      <td>HOST</td>\n",
-       "      <td>HOST</td>\n",
-       "      <td>pick</td>\n",
-       "      <td>Koala</td>\n",
-       "      <td>False</td>\n",
-       "      <td>13973</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1</th>\n",
-       "      <td>3</td>\n",
-       "      <td>GUEST</td>\n",
-       "      <td>GUEST</td>\n",
-       "      <td>pick</td>\n",
-       "      <td>Nordinseln</td>\n",
-       "      <td>False</td>\n",
-       "      <td>26925</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2</th>\n",
-       "      <td>8</td>\n",
-       "      <td>HOST</td>\n",
-       "      <td>HOST</td>\n",
-       "      <td>pick</td>\n",
-       "      <td>Vier Seen</td>\n",
-       "      <td>False</td>\n",
-       "      <td>54947</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "   index player executingPlayer actionType chosenOptionId  isRandomlyChosen  \\\n",
-       "0      2   HOST            HOST       pick          Koala             False   \n",
-       "1      3  GUEST           GUEST       pick     Nordinseln             False   \n",
-       "2      8   HOST            HOST       pick      Vier Seen             False   \n",
-       "\n",
-       "   offset  \n",
-       "0   13973  \n",
-       "1   26925  \n",
-       "2   54947  "
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "draft_picks.reset_index()"
-   ]
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

jupyter/src/drafts.py

@@ -23,5 +23,5 @@ def get_picked_maps(draft_json):
 
 # the last pick is the neutral map
 def neutral_map_as_string(draft_picks):
-    neutral_map = draft_picks.tail(1)['chosenOptionId'].values[0]
-    return neutral_map
+    # print(draft_picks.tail(1)['chosenOptionId'])
+    return str(draft_picks.tail(1)['chosenOptionId'][8])