# Clone
git clone https://github.com/disq/crawler.git
cd crawler

# Fetch dependencies
dep ensure

# Build
go build ./cmd/crawler

Usage: ./crawler [options] [start url] [additional hosts to include...]
  -l string
        Log level (default "info")
  -t duration
        HTTP timeout (default 5s)
  -w int
        Number of worker goroutines. Negative numbers mean multiples of the CPU core count. (default 256)