python -m venv ./venv
. ./venv/bin/activate
- Run
deactivatein the command line to deactivate the virtual env. - To show the path to the virtual env run:
echo $VIRTUAL_ENV
from pyspark import SparkContext, SparkConf, SQLContext
from pyspark.sql import SparkSession
sparkConf = SparkConf()
sparkConf.setAppName("app_name")
sparkConf.set("spark.executor.instances", "4")
sparkConf.set("spark.executor.cores", "4")
sparkConf.set("spark.executor.memory", "5000m") # 5GB exec memory
sparkConf.set("spark.driver.memory", "5120m") # 5GB driver memory
spark = SparkSession.builder.config(conf=sparkConf).enableHiveSupport().getOrCreate()
dept = [("Finance",10),
("Marketing",20),
("Sales",30),
("IT",40)
]
deptColumns = ["dept_name","dept_id"]
deptDF = spark.createDataFrame(data=dept, schema = deptColumns)
deptDF.printSchema()
deptDF.show(truncate=False)
#define schema
from pyspark.sql.types import StructType,StructField, StringType, IntegerType, DoubleType
schema = StructType([ \
StructField("id", IntegerType(), True), \
StructField("name",StringType(),True), \
StructField("grade", DoubleType(), True) \
])
data2 = [(100, "James", 80.0),
(101, "Michael", 75.5),
(102, "Robert", 90.2)
]
df = spark.createDataFrame(data=data2, schema=schema)
mvn archetype:generate -DarchetypeGroupId=net.alchim31.maven -DarchetypeArtifactId=scala-archetype-simple
mvn clean test
mvn package -DskipTests=true
- Change directory to the folder where
Dockerfileresides. - Run
docker build -t {your_image_name} .-ttags your image as a human-readable name for the final image.- The
.at the end tells docker to look for Dockerfile inside the current folder.
docker run --name {countainer_name} -p 8081:8081 {your_image_name}--nameassigns a name to the container. If not specified, a random name would be set.-pcreates a port mapping between the host and the container.
# by docker image Id
docker run \
-p 8080:8080 \
-v /local/path/:/path/in/docker/ \
bf3c64c34b80 \
serve
-vmaps local file/folder to a file/folder in container.
- Install
ffmpegandgifsicleif not installed yet.
$brew install ffmpeg
$brew install gifsicle
- Convert your file to gif:
$ffmpeg -i in.mov -pix_fmt rgb8 -r 10 output.gif && gifsicle -O3 output.gif -o output.gif
- Arguments:
- input path argument
-i - pixel format argument
-pix_fmt - removing some frames using framerate argument
-r - end
ffmpegwith new path/to/filename
- input path argument