m3db · robskillington · Jul 16, 2018 · Jul 16, 2018 · Jul 16, 2018
diff --git a/docs/how_to/cluster_hard_way.md b/docs/how_to/cluster_hard_way.md
@@ -1,7 +1,8 @@
 # M3DB Cluster Deployment, Manually (The Hard Way)
 
 ## Introduction
-This document lists the manual steps involved in deploying a M3DB cluster. In practice, you wouldn’t be doing this by hand, you’d be automating it using Terraform or Kubernetes. If you’re interested in how you could do so, we have some other guides you might be interested under the How-To section on https://m3db.github.io/m3db/.
+
+This document lists the manual steps involved in deploying a M3DB cluster. In practice, you'd be automating this using Terraform or using Kubernetes rather than doing this by hand; guides for doing so are available under the How-To section.
 
 ## Primer Architecture
 A quick primer on M3DB architecture. Here’s what a typical deployment looks like:
@@ -14,7 +15,7 @@ A few different things to highlight about the diagram:
 
 There are three ‘role types’ for a m3db deployment -
 
-- m3coordinator: m3coordinator serves to coordinate reads and writes across all hosts in the cluster. It’s a lightweight process, and does not store any data. This role would typically be run alongside a Prometheus instance, or be baked into a collector agent.
+- Coordinator: `m3coordinator` serves to coordinate reads and writes across all hosts in the cluster. It’s a lightweight process, and does not store any data. This role would typically be run alongside a Prometheus instance, or be baked into a collector agent.
 
 - Storage Node: `m3dbnode` processes running on these hosts are the workhorses of the database, they store data; and serve reads and writes.
 
@@ -113,7 +114,7 @@ Note: Isolation group specifies how the cluster places shards to avoid more than
 
 ```json
 curl -X POST localhost:7201/api/v1/placement/init -d '{
-    "num_shards": 256,
+    "num_shards": 1024,
     "replication_factor": 3,
     "instances": [
         {
@@ -236,6 +237,6 @@ curl -sSf -X POST http://localhost:9003/query -d '{
 }' | jq .
 ```
 
-## Use integrations
+## Integrations
 
-Checkout the integrations documentation to integrate with our software, such as Prometheus as a long term storage remote read/write endpoint.
+[Prometheus as a long term storage remote read/write endpoint](../integrations/prometheus.md).
diff --git a/docs/how_to/kubernetes.md b/docs/how_to/kubernetes.md
@@ -72,7 +72,7 @@ Forwarding from [::1]:7201 -> 7201
 ```json
 # Create an initial cluster topology
 curl -sSf -X POST localhost:7201/api/v1/placement/init -d '{
-    "num_shards": 256,
+    "num_shards": 1024,
     "replication_factor": 3,
     "instances": [
         {
@@ -109,7 +109,7 @@ curl -sSf -X POST localhost:7201/api/v1/placement/init -d '{
 ```json
 # Create a namespace to hold your metrics
 curl -X POST localhost:7201/api/v1/namespace -d '{
-  "name": "metrics",
+  "name": "default",
   "options": {
     "bootstrapEnabled": true,
     "flushEnabled": true,
@@ -161,7 +161,7 @@ Forwarding from [::1]:9003 -> 9003
 
 ```json
 curl -sSf -X POST localhost:9003/writetagged -d '{
-  "namespace": "metrics",
+  "namespace": "default",
   "id": "foo",
   "tags": [
     {
@@ -182,7 +182,7 @@ curl -sSf -X POST localhost:9003/writetagged -d '{
 
 ```json
 $ curl -sSf -X POST http://localhost:9003/query -d '{
-  "namespace": "metrics",
+  "namespace": "default",
   "query": {
     "regexp": {
       "field": "city",
@@ -255,6 +255,7 @@ curl -sSf -X POST localhost:7201/api/v1/placement/add -d '{
 
 As mentioned in our integrations [guide](../integrations/prometheus.md), M3DB can be used as a [remote read/write
 endpoint](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_write%3E) for Prometheus.
+
 If you run Prometheus on your Kubernetes cluster you can easily point it at M3DB in your Prometheus server config:
 
 ```

diff --git a/docs/how_to/single_node.md b/docs/how_to/single_node.md
@@ -15,6 +15,7 @@ and port `9003` (used to read and write metrics) exposed. We recommend you creat
 directory on your host for durability:
 
 ```
+docker pull quay.io/m3db/m3db:latest
 docker run -p 7201:7201 -p 9003:9003 --name m3db -v $(pwd)/m3db_data:/var/lib/m3db quay.io/m3db/m3db:latest
 ```
 
@@ -24,53 +25,15 @@ placements, etc. are -->
 <!-- TODO: add something about how this is in no way a recommended production deployment guide,
 and write a guide for what is considered a production-ready deployment (this is in the works) -->
 
-Next, create an initial namespace for your metrics:
+Next, create an initial namespace for your metrics in the database:
 
 <!-- TODO: link to config reference docs once available -->
 
 ```json
-curl -X POST localhost:7201/api/v1/namespace -d '{
-  "name": "default",
-  "options": {
-    "bootstrapEnabled": true,
-    "flushEnabled": true,
-    "writesToCommitLog": true,
-    "cleanupEnabled": true,
-    "snapshotEnabled": false,
-    "repairEnabled": false,
-    "retentionOptions": {
-      "retentionPeriodDuration": "720h",
-      "blockSizeDuration": "12h",
-      "bufferFutureDuration": "1h",
-      "bufferPastDuration": "1h",
-      "blockDataExpiry": true,
-      "blockDataExpiryAfterNotAccessPeriodDuration": "5m"
-    },
-    "indexOptions": {
-      "enabled": true,
-      "blockSizeDuration": "12h"
-    }
-  }
-}'
-```
-
-With a namespace to hold your metrics created, you can initialize your first placement:
-
-```json
-curl -X POST localhost:7201/api/v1/placement/init -d '{
-    "num_shards": 64,
-    "replication_factor": 1,
-    "instances": [
-        {
-            "id": "m3db_local",
-            "isolation_group": "rack-a",
-            "zone": "embedded",
-            "weight": 1024,
-            "endpoint": "127.0.0.1:9000",
-            "hostname": "127.0.0.1",
-            "port": 9000
-        }
-    ]
+curl -X POST http://localhost:7201/api/v1/database/create -d '{
+  "type": "local",
+  "namespaceName": "default",
+  "retentionTime": "48h"
 }'
 ```
 
@@ -80,16 +43,13 @@ errors related to a local cache file, such as `[W] could not load cache from fil
 warn-level errors (prefixed with `[W]`) should not block bootstrapping.
 
 ```
-20:10:12.911218[I] updating database namespaces [{adds [default]} {updates []} {removals []}]
-20:10:13.462798[I] node tchannelthrift: listening on 0.0.0.0:9000
-20:10:13.463107[I] cluster tchannelthrift: listening on 0.0.0.0:9001
-20:10:13.747173[I] node httpjson: listening on 0.0.0.0:9002
-20:10:13.747506[I] cluster httpjson: listening on 0.0.0.0:9003
-20:10:13.747763[I] bootstrapping shards for range starting ...
-...
-20:10:13.757834[I] bootstrap finished [{namespace default} {duration 10.1261ms}]
-20:10:13.758001[I] bootstrapped
-20:10:14.764771[I] successfully updated topology to 1 hosts
+02:28:30.008072[I] updating database namespaces [{adds [default]} {updates []} {removals []}]
+02:28:30.270681[I] node tchannelthrift: listening on 0.0.0.0:9000
+02:28:30.271909[I] cluster tchannelthrift: listening on 0.0.0.0:9001
+02:28:30.519468[I] node httpjson: listening on 0.0.0.0:9002
+02:28:30.520061[I] cluster httpjson: listening on 0.0.0.0:9003
+02:28:30.520652[I] bootstrap finished [{namespace metrics} {duration 55.4µs}]
+02:28:30.520909[I] bootstrapped
 ```
 
 The node also self-hosts its OpenAPI docs, outlining available endpoints. You can access this by
@@ -159,3 +119,7 @@ curl -sSf -X POST http://localhost:9003/query -d '{
   "exhaustive": true
 }
 ```
+
+## Integrations
+
+[Prometheus as a long term storage remote read/write endpoint](../integrations/prometheus.md).
diff --git a/docs/integrations/prometheus.md b/docs/integrations/prometheus.md
@@ -6,28 +6,63 @@ This document is a getting started guide to integrating M3DB with Prometheus.
 
 To write to a remote M3DB cluster the simplest configuration is to run `m3coordinator` as a sidecar alongside Prometheus.
 
-Start by downloading the [config template](https://github.com/m3db/m3db/blob/master/src/coordinator/config/m3coordinator-cluster-template.yml). Update the config ‘service’ and 'seedNodes' sections to match your cluster's configuration, something like:
-
-```
-config:
-  service:
-    env: default_env
-    zone: embedded
-    service: m3db
-    cacheDir: /var/lib/m3kv
-    etcdClusters:
-      - zone: embedded
-        endpoints:
-          - SEED_HOST_1_STATIC_IP:2379
-          - SEED_HOST_2_STATIC_IP:2379
-          ...
-  seedNodes:
-    initialCluster:
-      - hostID: SEED_HOST_1_NAME
-        endpoint: http://SEED_HOST_1_STATIC_IP:2380
-      - hostID: SEED_HOST_2_NAME
-        endpoint: http://SEED_HOST_2_STATIC_IP:2380
-      ...
+Start by downloading the [config template](https://github.com/m3db/m3db/blob/master/src/coordinator/config/m3coordinator-cluster-template.yml). Update the `namespaces` and the `client` section for a new cluster to match your cluster's configuration.
+
+You'll need to specify the static IPs or hostnames of your M3DB seed nodes, and the name and retention values of the namespace you set up.  You can leave the namespace storage metrics type as `unaggregated` since it's required by default to have a cluster that receives all Prometheus metrics unaggregated.  In the future you might also want to aggregate and downsample metrics for longer retention, and you can come back and update the config once you've setup those clusters.
+
+It should look something like:
+
+```
+listenAddress: 0.0.0.0:7201
+
+metrics:
+  scope:
+    prefix: "coordinator"
+  prometheus:
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3db/issues/682 is resolved
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+clusters:
+   - namespaces:
+# We created a namespace called "default" and had set it to retention "48h".
+       - namespace: default
+         retention: 48h
+         storageMetricsType: unaggregated
+     client:
+       config:
+         service:
+           env: default_env
+           zone: embedded
+           service: m3db
+           cacheDir: /var/lib/m3kv
+           etcdClusters:
+             - zone: embedded
+               endpoints:
+# We have five M3DB nodes but only three are seed nodes, they are listed here.
+                 - M3DB_NODE_01_STATIC_IP_ADDRESS:2379
+                 - M3DB_NODE_02_STATIC_IP_ADDRESS:2379
+                 - M3DB_NODE_03_STATIC_IP_ADDRESS:2379
+       writeConsistencyLevel: majority
+       readConsistencyLevel: unstrict_majority
+       writeTimeout: 10s
+       fetchTimeout: 15s
+       connectTimeout: 20s
+       writeRetry:
+         initialBackoff: 500ms
+         backoffFactor: 3
+         maxRetries: 2
+         jitter: true
+       fetchRetry:
+         initialBackoff: 500ms
+         backoffFactor: 2
+         maxRetries: 3
+         jitter: true
+       backgroundHealthCheckFailLimit: 4
+       backgroundHealthCheckFailThrottleFactor: 0.5
+
 ```
 
 Now start the process up:
@@ -36,6 +71,13 @@ Now start the process up:
 m3coordinator -f <config-name.yml>
 ```
 
+Or, use the docker container:
+
+```
+docker pull quay.io/m3db/m3coordinator:latest
+docker run -p 7201:7201 --name m3coordinator -v <config-name.yml>:/etc/m3coordinator/m3coordinator.yml quay.io/m3db/m3coordinator:latest
+```
+
 ## Prometheus configuration
 
 Add to your Prometheus configuration the `m3coordinator` sidecar remote read/write endpoints, something like:

diff --git a/src/coordinator/api/v1/handler/database/create.go b/src/coordinator/api/v1/handler/database/create.go
@@ -53,6 +53,9 @@ const (
 	// CreateHTTPMethod is the HTTP method used with this resource.
 	CreateHTTPMethod = http.MethodPost
 
+	// DefaultLocalHostID is the default local host ID when creating a database.
+	DefaultLocalHostID = "m3db_local"
+
 	idealDatapointsPerBlock           = 720
 	blockSizeFromExpectedSeriesScalar = idealDatapointsPerBlock * int64(time.Hour)
 	shardMultiplier                   = 64
@@ -221,18 +224,27 @@ func defaultedNamespaceAddRequest(r *admin.DatabaseCreateRequest) (*admin.Namesp
 		opts = opts.SetRepairEnabled(false)
 		retentionOpts := opts.RetentionOptions()
 
-		if r.RetentionPeriodNanos <= 0 {
+		if r.RetentionTime == "" {
 			retentionOpts = retentionOpts.SetRetentionPeriod(defaultLocalRetentionPeriod)
 		} else {
-			retentionOpts = retentionOpts.SetRetentionPeriod(time.Duration(r.RetentionPeriodNanos))
+			value, err := time.ParseDuration(r.RetentionTime)
+			if err != nil {
+				return nil, fmt.Errorf("invalid retention time: %v", err)
+			}
+			retentionOpts = retentionOpts.SetRetentionPeriod(value)
 		}
 
 		retentionPeriod := retentionOpts.RetentionPeriod()
 
 		var blockSize time.Duration
 		switch {
-		case r.BlockSize != nil && r.BlockSize.Nanos > 0:
-			blockSize = time.Duration(r.BlockSize.Nanos)
+		case r.BlockSize != nil && r.BlockSize.Time != "":
+			value, err := time.ParseDuration(r.BlockSize.Time)
+			if err != nil {
+				return nil, fmt.Errorf("invalid block size time: %v", err)
+			}
+			blockSize = value
+
 		case r.BlockSize != nil && r.BlockSize.ExpectedSeriesDatapointsPerHour > 0:
 			value := r.BlockSize.ExpectedSeriesDatapointsPerHour
 			blockSize = time.Duration(blockSizeFromExpectedSeriesScalar / value)
@@ -254,6 +266,7 @@ func defaultedNamespaceAddRequest(r *admin.DatabaseCreateRequest) (*admin.Namesp
 			} else if blockSize > maxRecommendCalculateBlockSize {
 				blockSize = maxRecommendCalculateBlockSize
 			}
+
 		default:
 			// Use the maximum block size if we don't find a way to
 			// recommended one based on request parameters
@@ -265,6 +278,7 @@ func defaultedNamespaceAddRequest(r *admin.DatabaseCreateRequest) (*admin.Namesp
 					break
 				}
 			}
+
 		}
 
 		retentionOpts = retentionOpts.SetBlockSize(blockSize)
@@ -310,7 +324,7 @@ func defaultedPlacementInitRequest(
 		replicationFactor = 1
 		instances = []*placementpb.Instance{
 			&placementpb.Instance{
-				Id:             "localhost",
+				Id:             DefaultLocalHostID,
 				IsolationGroup: "local",
 				Zone:           "embedded",
 				Weight:         1,