Bug: fix aztk cluster submit paths, imports (#464)

jafreck · web-flow · commit c1f43c73c1e7 · 2018-03-27T16:05:54.000-07:00
* fix cluster submit

* add export pythonpath to docker_main
diff --git a/aztk/node_scripts/docker_main.sh b/aztk/node_scripts/docker_main.sh
@@ -29,6 +29,7 @@ echo "Starting setup using Docker"
 
 $(pyenv root)/versions/$AZTK_PYTHON_VERSION/bin/pip install -r $(dirname $0)/requirements.txt
 export PYTHONPATH=$PYTHONPATH:$DOCKER_WORKING_DIR
+echo 'export PYTHONPATH=$PYTHONPATH:$DOCKER_WORKING_DIR' >> ~/.bashrc
 
 echo "Running main.py script"
 $(pyenv root)/versions/$AZTK_PYTHON_VERSION/bin/python $(dirname $0)/main.py install
diff --git a/aztk/node_scripts/submit.py b/aztk/node_scripts/submit.py
@@ -7,7 +7,7 @@
 from typing import List
 import azure.storage.blob as blob
 import azure.batch.models as batch_models
-from command_builder import CommandBuilder
+from aztk.utils.command_builder import CommandBuilder
 from core import config
 
 # limit azure.storage logging
diff --git a/aztk/spark/helpers/submit.py b/aztk/spark/helpers/submit.py
@@ -81,9 +81,10 @@ def generate_task(spark_client, container_id, application):
     task_cmd.add_option('-e', 'AZ_BATCH_TASK_WORKING_DIR=$AZ_BATCH_TASK_WORKING_DIR')
     task_cmd.add_option('-e', 'STORAGE_LOGS_CONTAINER={0}'.format(container_id))
     task_cmd.add_argument('spark /bin/bash >> output.log 2>&1')
-    task_cmd.add_argument('-c "cd $AZ_BATCH_TASK_WORKING_DIR; ' \
+    task_cmd.add_argument('-c "source ~/.bashrc; '\
+                          'cd $AZ_BATCH_TASK_WORKING_DIR; ' \
                           '\$(pyenv root)/versions/\$AZTK_PYTHON_VERSION/bin/python ' \
-                          '\$DOCKER_WORKING_DIR/submit.py"')
+                          '\$DOCKER_WORKING_DIR/aztk/node_scripts/submit.py"')
 
     # Create task
     task = batch_models.TaskAddParameter(
