From 38e4d086cff3d09b9f947b3f2eb3cd732c95a4b2 Mon Sep 17 00:00:00 2001 From: Marcel D Date: Thu, 30 Mar 2023 16:54:11 +0200 Subject: [PATCH 1/2] fix xcom --- airflow/Readme.md | 9 +++++++++ airflow/dags/aoe_draft_dag.py | 9 ++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/airflow/Readme.md b/airflow/Readme.md index 34cf526..061a0d7 100644 --- a/airflow/Readme.md +++ b/airflow/Readme.md @@ -1,5 +1,9 @@ # Apache Airflow Investigation +## How to run +in airflow directory: +`docker-compose up -d` + login: airflow:airflow @@ -22,3 +26,8 @@ https://airflow.apache.org/docs/apache-airflow-providers-openfaas/stable/index.h https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/index.html + +Lessons learned +Airflow stack takes A LOT of RAM: ~ 9GB + + diff --git a/airflow/dags/aoe_draft_dag.py b/airflow/dags/aoe_draft_dag.py index 8ed2953..72ba0f6 100644 --- a/airflow/dags/aoe_draft_dag.py +++ b/airflow/dags/aoe_draft_dag.py @@ -12,7 +12,7 @@ def extract_draft_data(): return drafts.download_draft_json(marcel_vs_canuck) def transform_draft(ti): - draft_json=ti.xcom_pull(task_ids=['extract_draft1']) + draft_json=ti.xcom_pull(task_ids='extract_draft1', key='return_value') if not draft_json: raise ValueError("No value stored in XComs") draft_picks=drafts.get_picked_maps(draft_json) @@ -32,13 +32,16 @@ with DAG("aoe_dag", start_date=datetime(2023, 1, 1), schedule_interval="@daily", #T transform_data = PythonOperator( task_id="transform_draft1", - python_callable=transform_draft + python_callable=transform_draft, + do_xcom_push=True ) #L + string_to_print = "{{ task_instance.xcom_pull(task_ids='transform_draft1', key='return_value') }}" print_output = BashOperator( task_id="print1", - bash_command="echo 'bla'" + bash_command="echo ${STRING_TO_PRINT}", + env={"STRING_TO_PRINT": string_to_print} ) extract_data >> transform_data >> print_output \ No newline at end of file From d1d7d1b58d303ed219f1ca623f94f120dcbd6868 Mon Sep 17 00:00:00 2001 From: Marcel D Date: Thu, 30 Mar 2023 20:24:03 +0200 Subject: [PATCH 2/2] improve neutral map algo --- jupyter/README.md | 15 +++++- jupyter/src/AoE-data.ipynb | 98 +++++++++++++++++++++++++++++++++++--- jupyter/src/drafts.py | 4 +- 3 files changed, 108 insertions(+), 9 deletions(-) diff --git a/jupyter/README.md b/jupyter/README.md index 3e38aba..639260d 100644 --- a/jupyter/README.md +++ b/jupyter/README.md @@ -14,10 +14,23 @@ docker run --name jupyter -d --rm \ -e NB_USER="marcelo" \ -e CHOWN_HOME=yes \ -w "/home/marcelo" \ - -v /mnt/c/Users/dechert/git/aoe2-data/jupyter/src:/home/marcelo/src \ + -v /mnt/c/Users/Marcelo/git/aoe2-data/jupyter/src:/home/marcelo/src \ jupyter/scipy-notebook:2023-03-27 ``` +Username agnostic command (does not work): +```bash +PATH_TO_JUPYTER_FOLDER="/mnt/c/Users/$(echo whoami)/git/aoe2-data/jupyter/src" docker run --name jupyter -d --rm \ + -p 8888:8888 \ + --user root \ + -e NB_USER="marcelo" \ + -e CHOWN_HOME=yes \ + -w "/home/marcelo" \ + -v ${PATH_TO_JUPYTER_FOLDER}:/home/marcelo/src \ + jupyter/scipy-notebook:2023-03-27 +``` + + `docker logs -f jupyter` to see the url with secret configuration: diff --git a/jupyter/src/AoE-data.ipynb b/jupyter/src/AoE-data.ipynb index ce570fa..cb202c3 100644 --- a/jupyter/src/AoE-data.ipynb +++ b/jupyter/src/AoE-data.ipynb @@ -9,7 +9,7 @@ }, "outputs": [], "source": [ - "%load_ext autoreload\n" + "%load_ext autoreload" ] }, { @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "a121cdfe-114a-4aae-b107-df891338dfcc", "metadata": { "tags": [] @@ -109,11 +109,97 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "7b337190-7887-40b7-9d10-b8039f5f2dfd", - "metadata": {}, - "outputs": [], - "source": [] + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
indexplayerexecutingPlayeractionTypechosenOptionIdisRandomlyChosenoffset
02HOSTHOSTpickKoalaFalse13973
13GUESTGUESTpickNordinselnFalse26925
28HOSTHOSTpickVier SeenFalse54947
\n", + "
" + ], + "text/plain": [ + " index player executingPlayer actionType chosenOptionId isRandomlyChosen \\\n", + "0 2 HOST HOST pick Koala False \n", + "1 3 GUEST GUEST pick Nordinseln False \n", + "2 8 HOST HOST pick Vier Seen False \n", + "\n", + " offset \n", + "0 13973 \n", + "1 26925 \n", + "2 54947 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "draft_picks.reset_index()" + ] } ], "metadata": { diff --git a/jupyter/src/drafts.py b/jupyter/src/drafts.py index a6cec3c..9b73952 100644 --- a/jupyter/src/drafts.py +++ b/jupyter/src/drafts.py @@ -23,5 +23,5 @@ def get_picked_maps(draft_json): # the last pick is the neutral map def neutral_map_as_string(draft_picks): - # print(draft_picks.tail(1)['chosenOptionId']) - return str(draft_picks.tail(1)['chosenOptionId'][8]) \ No newline at end of file + neutral_map = draft_picks.tail(1)['chosenOptionId'].values[0] + return neutral_map \ No newline at end of file