From 2718c75f654abb96d7fb10d07272a1c041f7d95b Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Sun, 15 Jul 2018 23:37:53 -0400 Subject: [PATCH 1/2] Fix documentation for single node walkthrough --- docs/how_to/cluster_hard_way.md | 9 +- docs/how_to/kubernetes.md | 9 +- docs/how_to/single_node.md | 70 ++------ docs/integrations/prometheus.md | 88 ++++++++--- .../api/v1/handler/database/create.go | 21 ++- .../api/v1/handler/database/create_test.go | 24 +-- .../config/m3coordinator-cluster-template.yml | 77 +++++---- .../generated/proto/admin/database.pb.go | 149 ++++++++++-------- .../generated/proto/admin/database.proto | 8 +- 9 files changed, 249 insertions(+), 206 deletions(-) diff --git a/docs/how_to/cluster_hard_way.md b/docs/how_to/cluster_hard_way.md index 8407d71362..1861973455 100644 --- a/docs/how_to/cluster_hard_way.md +++ b/docs/how_to/cluster_hard_way.md @@ -1,7 +1,8 @@ # M3DB Cluster Deployment, Manually (The Hard Way) ## Introduction -This document lists the manual steps involved in deploying a M3DB cluster. In practice, you wouldn’t be doing this by hand, you’d be automating it using Terraform or Kubernetes. If you’re interested in how you could do so, we have some other guides you might be interested under the How-To section on https://m3db.github.io/m3db/. + +This document lists the manual steps involved in deploying a M3DB cluster. In practice, you wouldn’t be doing this by hand, you’d be automating it using Terraform or Kubernetes. If you’re interested in how you could do so, we have some other guides you might be interested under the How-To section. ## Primer Architecture A quick primer on M3DB architecture. Here’s what a typical deployment looks like: @@ -14,7 +15,7 @@ A few different things to highlight about the diagram: There are three ‘role types’ for a m3db deployment - -- m3coordinator: m3coordinator serves to coordinate reads and writes across all hosts in the cluster. It’s a lightweight process, and does not store any data. This role would typically be run alongside a Prometheus instance, or be baked into a collector agent. +- Coordinator: `m3coordinator` serves to coordinate reads and writes across all hosts in the cluster. It’s a lightweight process, and does not store any data. This role would typically be run alongside a Prometheus instance, or be baked into a collector agent. - Storage Node: `m3dbnode` processes running on these hosts are the workhorses of the database, they store data; and serve reads and writes. @@ -113,7 +114,7 @@ Note: Isolation group specifies how the cluster places shards to avoid more than ```json curl -X POST localhost:7201/api/v1/placement/init -d '{ - "num_shards": 256, + "num_shards": 1024, "replication_factor": 3, "instances": [ { @@ -238,4 +239,4 @@ curl -sSf -X POST http://localhost:9003/query -d '{ ## Use integrations -Checkout the integrations documentation to integrate with our software, such as Prometheus as a long term storage remote read/write endpoint. +Checkout the integrations documentation to integrate with our software, such as [Prometheus as a long term storage remote read/write endpoint](../integrations/prometheus.md). diff --git a/docs/how_to/kubernetes.md b/docs/how_to/kubernetes.md index 5492a9a0ff..cd9208315a 100644 --- a/docs/how_to/kubernetes.md +++ b/docs/how_to/kubernetes.md @@ -72,7 +72,7 @@ Forwarding from [::1]:7201 -> 7201 ```json # Create an initial cluster topology curl -sSf -X POST localhost:7201/api/v1/placement/init -d '{ - "num_shards": 256, + "num_shards": 1024, "replication_factor": 3, "instances": [ { @@ -109,7 +109,7 @@ curl -sSf -X POST localhost:7201/api/v1/placement/init -d '{ ```json # Create a namespace to hold your metrics curl -X POST localhost:7201/api/v1/namespace -d '{ - "name": "metrics", + "name": "default", "options": { "bootstrapEnabled": true, "flushEnabled": true, @@ -161,7 +161,7 @@ Forwarding from [::1]:9003 -> 9003 ```json curl -sSf -X POST localhost:9003/writetagged -d '{ - "namespace": "metrics", + "namespace": "default", "id": "foo", "tags": [ { @@ -182,7 +182,7 @@ curl -sSf -X POST localhost:9003/writetagged -d '{ ```json $ curl -sSf -X POST http://localhost:9003/query -d '{ - "namespace": "metrics", + "namespace": "default", "query": { "regexp": { "field": "city", @@ -255,6 +255,7 @@ curl -sSf -X POST localhost:7201/api/v1/placement/add -d '{ As mentioned in our integrations [guide](../integrations/prometheus.md), M3DB can be used as a [remote read/write endpoint](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_write%3E) for Prometheus. + If you run Prometheus on your Kubernetes cluster you can easily point it at M3DB in your Prometheus server config: ``` diff --git a/docs/how_to/single_node.md b/docs/how_to/single_node.md index c2130755c3..090f7f056d 100644 --- a/docs/how_to/single_node.md +++ b/docs/how_to/single_node.md @@ -15,6 +15,7 @@ and port `9003` (used to read and write metrics) exposed. We recommend you creat directory on your host for durability: ``` +docker pull quay.io/m3db/m3db:latest docker run -p 7201:7201 -p 9003:9003 --name m3db -v $(pwd)/m3db_data:/var/lib/m3db quay.io/m3db/m3db:latest ``` @@ -24,53 +25,15 @@ placements, etc. are --> -Next, create an initial namespace for your metrics: +Next, create an initial namespace for your metrics in the database: ```json -curl -X POST localhost:7201/api/v1/namespace -d '{ - "name": "default", - "options": { - "bootstrapEnabled": true, - "flushEnabled": true, - "writesToCommitLog": true, - "cleanupEnabled": true, - "snapshotEnabled": false, - "repairEnabled": false, - "retentionOptions": { - "retentionPeriodDuration": "720h", - "blockSizeDuration": "12h", - "bufferFutureDuration": "1h", - "bufferPastDuration": "1h", - "blockDataExpiry": true, - "blockDataExpiryAfterNotAccessPeriodDuration": "5m" - }, - "indexOptions": { - "enabled": true, - "blockSizeDuration": "12h" - } - } -}' -``` - -With a namespace to hold your metrics created, you can initialize your first placement: - -```json -curl -X POST localhost:7201/api/v1/placement/init -d '{ - "num_shards": 64, - "replication_factor": 1, - "instances": [ - { - "id": "m3db_local", - "isolation_group": "rack-a", - "zone": "embedded", - "weight": 1024, - "endpoint": "127.0.0.1:9000", - "hostname": "127.0.0.1", - "port": 9000 - } - ] +curl -X POST http://localhost:7201/api/v1/database/create -d '{ + "type": "local", + "namespaceName": "default", + "retentionTime": "48h" }' ``` @@ -80,16 +43,13 @@ errors related to a local cache file, such as `[W] could not load cache from fil warn-level errors (prefixed with `[W]`) should not block bootstrapping. ``` -20:10:12.911218[I] updating database namespaces [{adds [default]} {updates []} {removals []}] -20:10:13.462798[I] node tchannelthrift: listening on 0.0.0.0:9000 -20:10:13.463107[I] cluster tchannelthrift: listening on 0.0.0.0:9001 -20:10:13.747173[I] node httpjson: listening on 0.0.0.0:9002 -20:10:13.747506[I] cluster httpjson: listening on 0.0.0.0:9003 -20:10:13.747763[I] bootstrapping shards for range starting ... -... -20:10:13.757834[I] bootstrap finished [{namespace default} {duration 10.1261ms}] -20:10:13.758001[I] bootstrapped -20:10:14.764771[I] successfully updated topology to 1 hosts +02:28:30.008072[I] updating database namespaces [{adds [default]} {updates []} {removals []}] +02:28:30.270681[I] node tchannelthrift: listening on 0.0.0.0:9000 +02:28:30.271909[I] cluster tchannelthrift: listening on 0.0.0.0:9001 +02:28:30.519468[I] node httpjson: listening on 0.0.0.0:9002 +02:28:30.520061[I] cluster httpjson: listening on 0.0.0.0:9003 +02:28:30.520652[I] bootstrap finished [{namespace metrics} {duration 55.4µs}] +02:28:30.520909[I] bootstrapped ``` The node also self-hosts its OpenAPI docs, outlining available endpoints. You can access this by @@ -159,3 +119,7 @@ curl -sSf -X POST http://localhost:9003/query -d '{ "exhaustive": true } ``` + +## Use integrations + +Checkout the integrations documentation to integrate with our software, such as [Prometheus as a long term storage remote read/write endpoint](../integrations/prometheus.md). diff --git a/docs/integrations/prometheus.md b/docs/integrations/prometheus.md index 35f46c27d6..ed70ee8d84 100644 --- a/docs/integrations/prometheus.md +++ b/docs/integrations/prometheus.md @@ -6,28 +6,65 @@ This document is a getting started guide to integrating M3DB with Prometheus. To write to a remote M3DB cluster the simplest configuration is to run `m3coordinator` as a sidecar alongside Prometheus. -Start by downloading the [config template](https://github.com/m3db/m3db/blob/master/src/coordinator/config/m3coordinator-cluster-template.yml). Update the config ‘service’ and 'seedNodes' sections to match your cluster's configuration, something like: - -``` -config: - service: - env: default_env - zone: embedded - service: m3db - cacheDir: /var/lib/m3kv - etcdClusters: - - zone: embedded - endpoints: - - SEED_HOST_1_STATIC_IP:2379 - - SEED_HOST_2_STATIC_IP:2379 - ... - seedNodes: - initialCluster: - - hostID: SEED_HOST_1_NAME - endpoint: http://SEED_HOST_1_STATIC_IP:2380 - - hostID: SEED_HOST_2_NAME - endpoint: http://SEED_HOST_2_STATIC_IP:2380 - ... +Start by downloading the [config template](https://github.com/m3db/m3db/blob/master/src/coordinator/config/m3coordinator-cluster-template.yml). Update the `namespaces` and the `client` section for a new cluster to match your cluster's configuration. + +At the very least you'll need to specify the name of your namespace that you setup, the retention that it has. You can leave the metrics type as `unaggregated` since it's required by default to have a cluster that receives all Prometheus metrics unaggregated. In the future you might also want to aggregate and downsample metrics for longer retention, and you can come back and update the config once you've setup those clusters. + +You'll also need to set the static IPs or hostnames of your M3DB seed nodes + +It should look something like: + +``` +listenAddress: 0.0.0.0:7201 + +metrics: + scope: + prefix: "coordinator" + prometheus: + handlerPath: /metrics + listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3db/issues/682 is resolved + sanitization: prometheus + samplingRate: 1.0 + extended: none + +clusters: + - namespaces: +# We created a namespace called "default" and had set it to retention "48h". + - namespace: default + retention: 48h + storageMetricsType: unaggregated + client: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: +# We have five M3DB nodes but only three are seed nodes, they are listed here. + - M3DB_NODE_01_STATIC_IP_ADDRESS:2379 + - M3DB_NODE_02_STATIC_IP_ADDRESS:2379 + - M3DB_NODE_03_STATIC_IP_ADDRESS:2379 + writeConsistencyLevel: majority + readConsistencyLevel: unstrict_majority + writeTimeout: 10s + fetchTimeout: 15s + connectTimeout: 20s + writeRetry: + initialBackoff: 500ms + backoffFactor: 3 + maxRetries: 2 + jitter: true + fetchRetry: + initialBackoff: 500ms + backoffFactor: 2 + maxRetries: 3 + jitter: true + backgroundHealthCheckFailLimit: 4 + backgroundHealthCheckFailThrottleFactor: 0.5 + ``` Now start the process up: @@ -36,6 +73,13 @@ Now start the process up: m3coordinator -f ``` +Or use the docker container: + +``` +docker pull quay.io/m3db/m3coordinator:latest +docker run -p 7201:7201 --name m3coordinator -v :/etc/m3coordinator/m3coordinator.yml quay.io/m3db/m3coordinator:latest +``` + ## Prometheus configuration Add to your Prometheus configuration the `m3coordinator` sidecar remote read/write endpoints, something like: diff --git a/src/coordinator/api/v1/handler/database/create.go b/src/coordinator/api/v1/handler/database/create.go index 322f1ac40e..cb1601fdb3 100644 --- a/src/coordinator/api/v1/handler/database/create.go +++ b/src/coordinator/api/v1/handler/database/create.go @@ -221,18 +221,27 @@ func defaultedNamespaceAddRequest(r *admin.DatabaseCreateRequest) (*admin.Namesp opts = opts.SetRepairEnabled(false) retentionOpts := opts.RetentionOptions() - if r.RetentionPeriodNanos <= 0 { + if r.RetentionTime == "" { retentionOpts = retentionOpts.SetRetentionPeriod(defaultLocalRetentionPeriod) } else { - retentionOpts = retentionOpts.SetRetentionPeriod(time.Duration(r.RetentionPeriodNanos)) + value, err := time.ParseDuration(r.RetentionTime) + if err != nil { + return nil, fmt.Errorf("invalid retention time: %v", err) + } + retentionOpts = retentionOpts.SetRetentionPeriod(value) } retentionPeriod := retentionOpts.RetentionPeriod() var blockSize time.Duration switch { - case r.BlockSize != nil && r.BlockSize.Nanos > 0: - blockSize = time.Duration(r.BlockSize.Nanos) + case r.BlockSize != nil && r.BlockSize.Time != "": + value, err := time.ParseDuration(r.BlockSize.Time) + if err != nil { + return nil, fmt.Errorf("invalid block size time: %v", err) + } + blockSize = value + case r.BlockSize != nil && r.BlockSize.ExpectedSeriesDatapointsPerHour > 0: value := r.BlockSize.ExpectedSeriesDatapointsPerHour blockSize = time.Duration(blockSizeFromExpectedSeriesScalar / value) @@ -254,6 +263,7 @@ func defaultedNamespaceAddRequest(r *admin.DatabaseCreateRequest) (*admin.Namesp } else if blockSize > maxRecommendCalculateBlockSize { blockSize = maxRecommendCalculateBlockSize } + default: // Use the maximum block size if we don't find a way to // recommended one based on request parameters @@ -265,6 +275,7 @@ func defaultedNamespaceAddRequest(r *admin.DatabaseCreateRequest) (*admin.Namesp break } } + } retentionOpts = retentionOpts.SetBlockSize(blockSize) @@ -310,7 +321,7 @@ func defaultedPlacementInitRequest( replicationFactor = 1 instances = []*placementpb.Instance{ &placementpb.Instance{ - Id: "localhost", + Id: "m3db_local", IsolationGroup: "local", Zone: "embedded", Weight: 1, diff --git a/src/coordinator/api/v1/handler/database/create_test.go b/src/coordinator/api/v1/handler/database/create_test.go index 75ae4fb190..a0034b7099 100644 --- a/src/coordinator/api/v1/handler/database/create_test.go +++ b/src/coordinator/api/v1/handler/database/create_test.go @@ -96,7 +96,7 @@ func TestLocalType(t *testing.T) { placementProto := &placementpb.Placement{ Instances: map[string]*placementpb.Instance{ "localhost": &placementpb.Instance{ - Id: "localhost", + Id: "m3db_local", IsolationGroup: "local", Zone: "embedded", Weight: 1, @@ -148,8 +148,8 @@ func TestLocalType(t *testing.T) { "placement": { "placement": { "instances": { - "localhost": { - "id": "localhost", + "m3db_local": { + "id": "m3db_local", "isolationGroup": "local", "zone": "embedded", "weight": 1, @@ -180,13 +180,13 @@ func TestLocalWithBlockSizeNanos(t *testing.T) { createHandler := NewCreateHandler(mockClient, config.Configuration{}, testDBCfg) w := httptest.NewRecorder() - jsonInput := fmt.Sprintf(` + jsonInput := ` { "namespaceName": "testNamespace", "type": "local", - "blockSize": {"nanos": %d} + "blockSize": {"time": "3h"} } - `, int64(3*time.Hour)) + ` req := httptest.NewRequest("POST", "/database/create", strings.NewReader(jsonInput)) require.NotNil(t, req) @@ -197,7 +197,7 @@ func TestLocalWithBlockSizeNanos(t *testing.T) { placementProto := &placementpb.Placement{ Instances: map[string]*placementpb.Instance{ "localhost": &placementpb.Instance{ - Id: "localhost", + Id: "m3db_local", IsolationGroup: "local", Zone: "embedded", Weight: 1, @@ -249,8 +249,8 @@ func TestLocalWithBlockSizeNanos(t *testing.T) { "placement": { "placement": { "instances": { - "localhost": { - "id": "localhost", + "m3db_local": { + "id": "m3db_local", "isolationGroup": "local", "zone": "embedded", "weight": 1, @@ -301,7 +301,7 @@ func TestLocalWithBlockSizeExpectedSeriesDatapointsPerHour(t *testing.T) { placementProto := &placementpb.Placement{ Instances: map[string]*placementpb.Instance{ "localhost": &placementpb.Instance{ - Id: "localhost", + Id: "m3db_local", IsolationGroup: "local", Zone: "embedded", Weight: 1, @@ -353,8 +353,8 @@ func TestLocalWithBlockSizeExpectedSeriesDatapointsPerHour(t *testing.T) { "placement": { "placement": { "instances": { - "localhost": { - "id": "localhost", + "m3db_local": { + "id": "m3db_local", "isolationGroup": "local", "zone": "embedded", "weight": 1, diff --git a/src/coordinator/config/m3coordinator-cluster-template.yml b/src/coordinator/config/m3coordinator-cluster-template.yml index ff13052b2e..e728c3382c 100644 --- a/src/coordinator/config/m3coordinator-cluster-template.yml +++ b/src/coordinator/config/m3coordinator-cluster-template.yml @@ -10,42 +10,41 @@ metrics: samplingRate: 1.0 extended: none -dbClient: -# Fill-out the following and un-comment before using. -# config: -# service: -# env: default_env -# zone: embedded -# service: m3db -# cacheDir: /var/lib/m3kv -# etcdClusters: -# - zone: embedded -# endpoints: -# - HOST1_STATIC_IP_ADDRESS:2379 -# - HOST2_STATIC_IP_ADDRESS:2379 -# - HOST3_STATIC_IP_ADDRESS:2379 -# seedNodes: -# initialCluster: -# - hostID: host1 -# endpoint: http://HOST1_STATIC_IP_ADDRESS:2380 -# - hostID: host2 -# endpoint: http://HOST2_STATIC_IP_ADDRESS:2380 -# - hostID: host3 -# endpoint: http://HOST3_STATIC_IP_ADDRESS:2380 - writeConsistencyLevel: majority - readConsistencyLevel: unstrict_majority - writeTimeout: 10s - fetchTimeout: 15s - connectTimeout: 20s - writeRetry: - initialBackoff: 500ms - backoffFactor: 3 - maxRetries: 2 - jitter: true - fetchRetry: - initialBackoff: 500ms - backoffFactor: 2 - maxRetries: 3 - jitter: true - backgroundHealthCheckFailLimit: 4 - backgroundHealthCheckFailThrottleFactor: 0.5 +clusters: +## Fill-out the following and un-comment before using, and +## make sure indent by two spaces is applied. +# - namespaces: +# - namespace: default +# retention: 48h +# storageMetricsType: unaggregated +# client: +# config: +# service: +# env: default_env +# zone: embedded +# service: m3db +# cacheDir: /var/lib/m3kv +# etcdClusters: +# - zone: embedded +# endpoints: +# - M3DB_NODE_01_STATIC_IP_ADDRESS:2379 +# - M3DB_NODE_02_STATIC_IP_ADDRESS:2379 +# - M3DB_NODE_03_STATIC_IP_ADDRESS:2379 +## ... etc, list only M3DB seed nodes +# writeConsistencyLevel: majority +# readConsistencyLevel: unstrict_majority +# writeTimeout: 10s +# fetchTimeout: 15s +# connectTimeout: 20s +# writeRetry: +# initialBackoff: 500ms +# backoffFactor: 3 +# maxRetries: 2 +# jitter: true +# fetchRetry: +# initialBackoff: 500ms +# backoffFactor: 2 +# maxRetries: 3 +# jitter: true +# backgroundHealthCheckFailLimit: 4 +# backgroundHealthCheckFailThrottleFactor: 0.5 diff --git a/src/coordinator/generated/proto/admin/database.pb.go b/src/coordinator/generated/proto/admin/database.pb.go index 426060f96c..b3099729f8 100644 --- a/src/coordinator/generated/proto/admin/database.pb.go +++ b/src/coordinator/generated/proto/admin/database.pb.go @@ -66,8 +66,8 @@ type DatabaseCreateRequest struct { // Optional fields that may be inferred depending on database type NumShards int32 `protobuf:"varint,3,opt,name=num_shards,json=numShards,proto3" json:"num_shards,omitempty"` ReplicationFactor int32 `protobuf:"varint,4,opt,name=replication_factor,json=replicationFactor,proto3" json:"replication_factor,omitempty"` - // The below two options are used to default retention options - RetentionPeriodNanos int64 `protobuf:"varint,5,opt,name=retention_period_nanos,json=retentionPeriodNanos,proto3" json:"retention_period_nanos,omitempty"` + // Explicit retention time using time shorthand, e.g. "48h" + RetentionTime string `protobuf:"bytes,5,opt,name=retention_time,json=retentionTime,proto3" json:"retention_time,omitempty"` // If no block size fields are set then the block size is // derived from the length of the retention period BlockSize *BlockSize `protobuf:"bytes,6,opt,name=block_size,json=blockSize" json:"block_size,omitempty"` @@ -108,11 +108,11 @@ func (m *DatabaseCreateRequest) GetReplicationFactor() int32 { return 0 } -func (m *DatabaseCreateRequest) GetRetentionPeriodNanos() int64 { +func (m *DatabaseCreateRequest) GetRetentionTime() string { if m != nil { - return m.RetentionPeriodNanos + return m.RetentionTime } - return 0 + return "" } func (m *DatabaseCreateRequest) GetBlockSize() *BlockSize { @@ -130,8 +130,8 @@ func (m *DatabaseCreateRequest) GetHosts() []*Host { } type BlockSize struct { - // Explicit block size nanoseconds - Nanos int64 `protobuf:"varint,1,opt,name=nanos,proto3" json:"nanos,omitempty"` + // Explicit block size using time shorthand, e.g. "2h" + Time string `protobuf:"bytes,1,opt,name=time,proto3" json:"time,omitempty"` // With the expected series datapoints per hour, use a recommended block size ExpectedSeriesDatapointsPerHour int64 `protobuf:"varint,2,opt,name=expected_series_datapoints_per_hour,json=expectedSeriesDatapointsPerHour,proto3" json:"expected_series_datapoints_per_hour,omitempty"` } @@ -141,11 +141,11 @@ func (m *BlockSize) String() string { return proto.CompactTextString( func (*BlockSize) ProtoMessage() {} func (*BlockSize) Descriptor() ([]byte, []int) { return fileDescriptorDatabase, []int{1} } -func (m *BlockSize) GetNanos() int64 { +func (m *BlockSize) GetTime() string { if m != nil { - return m.Nanos + return m.Time } - return 0 + return "" } func (m *BlockSize) GetExpectedSeriesDatapointsPerHour() int64 { @@ -287,10 +287,11 @@ func (m *DatabaseCreateRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintDatabase(dAtA, i, uint64(m.ReplicationFactor)) } - if m.RetentionPeriodNanos != 0 { - dAtA[i] = 0x28 + if len(m.RetentionTime) > 0 { + dAtA[i] = 0x2a i++ - i = encodeVarintDatabase(dAtA, i, uint64(m.RetentionPeriodNanos)) + i = encodeVarintDatabase(dAtA, i, uint64(len(m.RetentionTime))) + i += copy(dAtA[i:], m.RetentionTime) } if m.BlockSize != nil { dAtA[i] = 0x32 @@ -332,10 +333,11 @@ func (m *BlockSize) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.Nanos != 0 { - dAtA[i] = 0x8 + if len(m.Time) > 0 { + dAtA[i] = 0xa i++ - i = encodeVarintDatabase(dAtA, i, uint64(m.Nanos)) + i = encodeVarintDatabase(dAtA, i, uint64(len(m.Time))) + i += copy(dAtA[i:], m.Time) } if m.ExpectedSeriesDatapointsPerHour != 0 { dAtA[i] = 0x10 @@ -461,8 +463,9 @@ func (m *DatabaseCreateRequest) Size() (n int) { if m.ReplicationFactor != 0 { n += 1 + sovDatabase(uint64(m.ReplicationFactor)) } - if m.RetentionPeriodNanos != 0 { - n += 1 + sovDatabase(uint64(m.RetentionPeriodNanos)) + l = len(m.RetentionTime) + if l > 0 { + n += 1 + l + sovDatabase(uint64(l)) } if m.BlockSize != nil { l = m.BlockSize.Size() @@ -480,8 +483,9 @@ func (m *DatabaseCreateRequest) Size() (n int) { func (m *BlockSize) Size() (n int) { var l int _ = l - if m.Nanos != 0 { - n += 1 + sovDatabase(uint64(m.Nanos)) + l = len(m.Time) + if l > 0 { + n += 1 + l + sovDatabase(uint64(l)) } if m.ExpectedSeriesDatapointsPerHour != 0 { n += 1 + sovDatabase(uint64(m.ExpectedSeriesDatapointsPerHour)) @@ -670,10 +674,10 @@ func (m *DatabaseCreateRequest) Unmarshal(dAtA []byte) error { } } case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RetentionPeriodNanos", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RetentionTime", wireType) } - m.RetentionPeriodNanos = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowDatabase @@ -683,11 +687,21 @@ func (m *DatabaseCreateRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.RetentionPeriodNanos |= (int64(b) & 0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDatabase + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RetentionTime = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex case 6: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field BlockSize", wireType) @@ -803,10 +817,10 @@ func (m *BlockSize) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Nanos", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType) } - m.Nanos = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowDatabase @@ -816,11 +830,21 @@ func (m *BlockSize) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Nanos |= (int64(b) & 0x7F) << shift + stringLen |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDatabase + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Time = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex case 2: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ExpectedSeriesDatapointsPerHour", wireType) @@ -1291,39 +1315,38 @@ func init() { } var fileDescriptorDatabase = []byte{ - // 532 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x52, 0xcd, 0x6e, 0x13, 0x31, - 0x10, 0x66, 0x93, 0xa6, 0xd5, 0x3a, 0x6a, 0x28, 0x56, 0x89, 0x56, 0x20, 0x42, 0x08, 0x42, 0xe4, - 0x42, 0x56, 0x4a, 0xb9, 0x70, 0x2d, 0x15, 0x2d, 0x08, 0x45, 0x91, 0xf3, 0x00, 0x2b, 0xef, 0x7a, - 0x48, 0x2c, 0xb2, 0xb6, 0xb1, 0xbd, 0x02, 0xf2, 0x10, 0x88, 0x2b, 0xaf, 0xc0, 0x93, 0x70, 0xe4, - 0x11, 0x50, 0x78, 0x11, 0xb4, 0xb3, 0x3f, 0x05, 0xae, 0xbd, 0x44, 0x33, 0xdf, 0xf7, 0xcd, 0xe4, - 0xdb, 0xcf, 0x43, 0x5e, 0xaf, 0xa5, 0xdf, 0x14, 0xe9, 0x2c, 0xd3, 0x79, 0x9c, 0x9f, 0x89, 0xb4, - 0xfa, 0x71, 0x36, 0x8b, 0x33, 0xad, 0xad, 0x90, 0x8a, 0x7b, 0x6d, 0xe3, 0x35, 0x28, 0xb0, 0xdc, - 0x83, 0x88, 0x8d, 0xd5, 0x5e, 0xc7, 0x5c, 0xe4, 0x52, 0xc5, 0x82, 0x7b, 0x9e, 0x72, 0x07, 0x33, - 0x04, 0x69, 0x0f, 0xd1, 0x7b, 0x6f, 0x6e, 0xb0, 0x51, 0xf1, 0x1c, 0x9c, 0xe1, 0x59, 0xbd, 0xf2, - 0x46, 0xbb, 0xcc, 0x96, 0x67, 0x90, 0x83, 0xf2, 0xd5, 0xae, 0xc9, 0xf7, 0x0e, 0xb9, 0x7b, 0x51, - 0x3b, 0x7e, 0x69, 0x81, 0x7b, 0x60, 0xf0, 0xa1, 0x00, 0xe7, 0xe9, 0x13, 0x32, 0x68, 0xff, 0x38, - 0x29, 0xab, 0x28, 0x18, 0x07, 0xd3, 0x90, 0x1d, 0xb7, 0xe8, 0x82, 0xe7, 0x40, 0x29, 0x39, 0xf0, - 0x9f, 0x0d, 0x44, 0x1d, 0x24, 0xb1, 0xa6, 0x0f, 0x08, 0x51, 0x45, 0x9e, 0xb8, 0x0d, 0xb7, 0xc2, - 0x45, 0xdd, 0x71, 0x30, 0xed, 0xb1, 0x50, 0x15, 0xf9, 0x0a, 0x01, 0xfa, 0x8c, 0x50, 0x0b, 0x66, - 0x2b, 0x33, 0xee, 0xa5, 0x56, 0xc9, 0x3b, 0x9e, 0x79, 0x6d, 0xa3, 0x03, 0x94, 0xdd, 0xf9, 0x8b, - 0x79, 0x85, 0x04, 0x7d, 0x4e, 0x86, 0x16, 0x3c, 0x28, 0x14, 0x1b, 0xb0, 0x52, 0x8b, 0x44, 0x71, - 0xa5, 0x5d, 0xd4, 0x1b, 0x07, 0xd3, 0x2e, 0x3b, 0x6d, 0xd9, 0x25, 0x92, 0x8b, 0x92, 0xa3, 0x31, - 0x21, 0xe9, 0x56, 0x67, 0xef, 0x13, 0x27, 0x77, 0x10, 0x1d, 0x8e, 0x83, 0x69, 0x7f, 0x7e, 0x32, - 0xc3, 0x10, 0x66, 0xe7, 0x25, 0xb1, 0x92, 0x3b, 0x60, 0x61, 0xda, 0x94, 0xf4, 0x11, 0xe9, 0x6d, - 0xb4, 0xf3, 0x2e, 0x3a, 0x1a, 0x77, 0xa7, 0xfd, 0x79, 0xbf, 0xd6, 0x5e, 0x69, 0xe7, 0x59, 0xc5, - 0x4c, 0x34, 0x09, 0xdb, 0x51, 0x7a, 0x4a, 0x7a, 0x95, 0x8b, 0x00, 0x5d, 0x54, 0x0d, 0x7d, 0x4b, - 0x1e, 0xc3, 0x27, 0x03, 0x99, 0x07, 0x91, 0x38, 0xb0, 0x12, 0x5c, 0x52, 0x1e, 0x84, 0xd1, 0x52, - 0x79, 0x57, 0xba, 0x4f, 0x36, 0xba, 0xb0, 0x98, 0x56, 0x97, 0x3d, 0x6c, 0xa4, 0x2b, 0x54, 0x5e, - 0xb4, 0xc2, 0x25, 0xd8, 0x2b, 0x5d, 0xd8, 0xc9, 0xb7, 0x80, 0x1c, 0x94, 0x06, 0xe8, 0x80, 0x74, - 0xa4, 0xa8, 0x1f, 0xa0, 0x23, 0x05, 0x8d, 0xc8, 0x11, 0x17, 0xc2, 0x82, 0x73, 0x75, 0xf0, 0x4d, - 0x5b, 0xbe, 0x87, 0xd1, 0xd6, 0x63, 0xea, 0xc7, 0x0c, 0x6b, 0xfa, 0x94, 0xdc, 0x96, 0x4e, 0x6f, - 0xab, 0xb8, 0xd7, 0x56, 0x17, 0x06, 0xd3, 0x0e, 0xd9, 0xa0, 0x85, 0x2f, 0x4b, 0xb4, 0x1c, 0xde, - 0x69, 0x05, 0x18, 0x6c, 0xc8, 0xb0, 0xa6, 0x43, 0x72, 0xf8, 0x11, 0xe4, 0x7a, 0xe3, 0x31, 0xc4, - 0x63, 0x56, 0x77, 0x93, 0x2f, 0x01, 0x19, 0xfe, 0x7f, 0x39, 0xce, 0x68, 0xe5, 0x80, 0xbe, 0x20, - 0x61, 0x7b, 0x24, 0x68, 0xba, 0x3f, 0xbf, 0x5f, 0xc7, 0xb9, 0x68, 0xf0, 0x4b, 0xf0, 0x8d, 0x9e, - 0x5d, 0xab, 0xcb, 0xd1, 0xf6, 0x44, 0xf1, 0xd3, 0xae, 0x47, 0x97, 0x0d, 0xfe, 0xcf, 0x68, 0xab, - 0x3e, 0x3f, 0xf9, 0xb1, 0x1f, 0x05, 0x3f, 0xf7, 0xa3, 0xe0, 0xd7, 0x7e, 0x14, 0x7c, 0xfd, 0x3d, - 0xba, 0x95, 0x1e, 0xe2, 0x8d, 0x9f, 0xfd, 0x09, 0x00, 0x00, 0xff, 0xff, 0xa8, 0x52, 0x89, 0xf4, - 0xcf, 0x03, 0x00, 0x00, + // 513 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x52, 0xcb, 0x6e, 0xd3, 0x40, + 0x14, 0xc5, 0x69, 0x92, 0xca, 0x13, 0x25, 0x94, 0x91, 0xa8, 0x2c, 0x10, 0x21, 0x04, 0x21, 0xb2, + 0x21, 0x96, 0xd2, 0x15, 0xdb, 0x52, 0xd1, 0x82, 0x50, 0x55, 0x39, 0xec, 0xad, 0xb1, 0xe7, 0x92, + 0x8c, 0xc8, 0x3c, 0x98, 0x19, 0x0b, 0xc8, 0x47, 0x20, 0xb6, 0x88, 0x1f, 0x62, 0xc9, 0x27, 0xa0, + 0xf0, 0x23, 0xc8, 0x37, 0xf6, 0x14, 0xd8, 0x76, 0x63, 0x9d, 0x39, 0xf7, 0xdc, 0xd7, 0xf1, 0x25, + 0xaf, 0x56, 0xc2, 0xaf, 0xab, 0x62, 0x5e, 0x6a, 0x99, 0xca, 0x13, 0x5e, 0xec, 0x3f, 0xce, 0x96, + 0x69, 0xa9, 0xb5, 0xe5, 0x42, 0x31, 0xaf, 0x6d, 0xba, 0x02, 0x05, 0x96, 0x79, 0xe0, 0xa9, 0xb1, + 0xda, 0xeb, 0x94, 0x71, 0x29, 0x54, 0xca, 0x99, 0x67, 0x05, 0x73, 0x30, 0x47, 0x92, 0xf6, 0x90, + 0xbd, 0xf7, 0xfa, 0x06, 0x15, 0x15, 0x93, 0xe0, 0x0c, 0x2b, 0x9b, 0x92, 0x37, 0xaa, 0x65, 0x36, + 0xac, 0x04, 0x09, 0xca, 0xef, 0x6b, 0x4d, 0xbf, 0x77, 0xc8, 0xdd, 0xb3, 0x66, 0xe2, 0x17, 0x16, + 0x98, 0x87, 0x0c, 0x3e, 0x54, 0xe0, 0x3c, 0x7d, 0x42, 0x46, 0xa1, 0x71, 0x5e, 0xa3, 0x24, 0x9a, + 0x44, 0xb3, 0x38, 0x1b, 0x06, 0xf6, 0x92, 0x49, 0xa0, 0x94, 0x74, 0xfd, 0x67, 0x03, 0x49, 0x07, + 0x83, 0x88, 0xe9, 0x03, 0x42, 0x54, 0x25, 0x73, 0xb7, 0x66, 0x96, 0xbb, 0xe4, 0x60, 0x12, 0xcd, + 0x7a, 0x59, 0xac, 0x2a, 0xb9, 0x44, 0x82, 0x3e, 0x23, 0xd4, 0x82, 0xd9, 0x88, 0x92, 0x79, 0xa1, + 0x55, 0xfe, 0x8e, 0x95, 0x5e, 0xdb, 0xa4, 0x8b, 0xb2, 0x3b, 0x7f, 0x45, 0x5e, 0x62, 0xa0, 0x1e, + 0xc4, 0x82, 0x07, 0x85, 0x62, 0x2f, 0x24, 0x24, 0xbd, 0xfd, 0x20, 0x81, 0x7d, 0x2b, 0x24, 0xd0, + 0x94, 0x90, 0x62, 0xa3, 0xcb, 0xf7, 0xb9, 0x13, 0x5b, 0x48, 0xfa, 0x93, 0x68, 0x36, 0x58, 0x1c, + 0xcd, 0x71, 0xeb, 0xf9, 0x69, 0x1d, 0x58, 0x8a, 0x2d, 0x64, 0x71, 0xd1, 0x42, 0xfa, 0x88, 0xf4, + 0xd6, 0xda, 0x79, 0x97, 0x1c, 0x4e, 0x0e, 0x66, 0x83, 0xc5, 0xa0, 0xd1, 0x5e, 0x68, 0xe7, 0xb3, + 0x7d, 0x64, 0x2a, 0x49, 0x1c, 0x52, 0x71, 0x53, 0x11, 0x6c, 0x40, 0x4c, 0xdf, 0x90, 0xc7, 0xf0, + 0xc9, 0x40, 0xe9, 0x81, 0xe7, 0x0e, 0xac, 0x00, 0x97, 0xd7, 0xff, 0xdf, 0x68, 0xa1, 0xbc, 0xcb, + 0x0d, 0xd8, 0x7c, 0xad, 0x2b, 0x8b, 0xe6, 0x1c, 0x64, 0x0f, 0x5b, 0xe9, 0x12, 0x95, 0x67, 0x41, + 0x78, 0x05, 0xf6, 0x42, 0x57, 0x76, 0xfa, 0x2d, 0x22, 0xdd, 0xba, 0x3d, 0x1d, 0x91, 0x8e, 0xe0, + 0x4d, 0xa3, 0x8e, 0xe0, 0x34, 0x21, 0x87, 0x8c, 0x73, 0x0b, 0xce, 0x35, 0x3e, 0xb7, 0xcf, 0x7a, + 0x28, 0xa3, 0xad, 0x47, 0x93, 0x87, 0x19, 0x62, 0xfa, 0x94, 0xdc, 0x16, 0x4e, 0x6f, 0xf6, 0xee, + 0xae, 0xac, 0xae, 0x0c, 0x9a, 0x1b, 0x67, 0xa3, 0x40, 0x9f, 0xd7, 0x6c, 0x9d, 0xbc, 0xd5, 0xaa, + 0xf5, 0x13, 0x31, 0x3d, 0x26, 0xfd, 0x8f, 0x20, 0x56, 0x6b, 0x8f, 0x16, 0x0e, 0xb3, 0xe6, 0x35, + 0xfd, 0x12, 0x91, 0xe3, 0xff, 0x0f, 0xc5, 0x19, 0xad, 0x1c, 0xd0, 0xe7, 0x24, 0x0e, 0x37, 0x81, + 0x43, 0x0f, 0x16, 0xf7, 0x1b, 0x33, 0x2f, 0x5b, 0xfe, 0x1c, 0x7c, 0xab, 0xcf, 0xae, 0xd5, 0x75, + 0x6a, 0xb8, 0x48, 0x5c, 0xed, 0x3a, 0xf5, 0xaa, 0xe5, 0xff, 0x49, 0x0d, 0xea, 0xd3, 0xa3, 0x1f, + 0xbb, 0x71, 0xf4, 0x73, 0x37, 0x8e, 0x7e, 0xed, 0xc6, 0xd1, 0xd7, 0xdf, 0xe3, 0x5b, 0x45, 0x1f, + 0x4f, 0xfa, 0xe4, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4a, 0xb6, 0xa9, 0xdc, 0xbe, 0x03, 0x00, + 0x00, } diff --git a/src/coordinator/generated/proto/admin/database.proto b/src/coordinator/generated/proto/admin/database.proto index 4a5d4b89dd..ca690e8248 100644 --- a/src/coordinator/generated/proto/admin/database.proto +++ b/src/coordinator/generated/proto/admin/database.proto @@ -14,8 +14,8 @@ message DatabaseCreateRequest { int32 num_shards = 3; int32 replication_factor = 4; - // The below two options are used to default retention options - int64 retention_period_nanos = 5; + // Explicit retention time using time shorthand, e.g. "48h" + string retention_time = 5; // If no block size fields are set then the block size is // derived from the length of the retention period @@ -26,8 +26,8 @@ message DatabaseCreateRequest { } message BlockSize { - // Explicit block size nanoseconds - int64 nanos = 1; + // Explicit block size using time shorthand, e.g. "2h" + string time = 1; // With the expected series datapoints per hour, use a recommended block size int64 expected_series_datapoints_per_hour = 2; } From 23598424098bebba62bf7b5826b61dce7690db70 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Mon, 16 Jul 2018 09:49:17 -0400 Subject: [PATCH 2/2] Address feedback --- docs/how_to/cluster_hard_way.md | 6 +++--- docs/how_to/single_node.md | 4 ++-- docs/integrations/prometheus.md | 6 ++---- src/coordinator/api/v1/handler/database/create.go | 5 ++++- src/coordinator/api/v1/handler/database/create_test.go | 4 ++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/how_to/cluster_hard_way.md b/docs/how_to/cluster_hard_way.md index 1861973455..4f0ef37c9e 100644 --- a/docs/how_to/cluster_hard_way.md +++ b/docs/how_to/cluster_hard_way.md @@ -2,7 +2,7 @@ ## Introduction -This document lists the manual steps involved in deploying a M3DB cluster. In practice, you wouldn’t be doing this by hand, you’d be automating it using Terraform or Kubernetes. If you’re interested in how you could do so, we have some other guides you might be interested under the How-To section. +This document lists the manual steps involved in deploying a M3DB cluster. In practice, you'd be automating this using Terraform or using Kubernetes rather than doing this by hand; guides for doing so are available under the How-To section. ## Primer Architecture A quick primer on M3DB architecture. Here’s what a typical deployment looks like: @@ -237,6 +237,6 @@ curl -sSf -X POST http://localhost:9003/query -d '{ }' | jq . ``` -## Use integrations +## Integrations -Checkout the integrations documentation to integrate with our software, such as [Prometheus as a long term storage remote read/write endpoint](../integrations/prometheus.md). +[Prometheus as a long term storage remote read/write endpoint](../integrations/prometheus.md). diff --git a/docs/how_to/single_node.md b/docs/how_to/single_node.md index 090f7f056d..9b77c01649 100644 --- a/docs/how_to/single_node.md +++ b/docs/how_to/single_node.md @@ -120,6 +120,6 @@ curl -sSf -X POST http://localhost:9003/query -d '{ } ``` -## Use integrations +## Integrations -Checkout the integrations documentation to integrate with our software, such as [Prometheus as a long term storage remote read/write endpoint](../integrations/prometheus.md). +[Prometheus as a long term storage remote read/write endpoint](../integrations/prometheus.md). diff --git a/docs/integrations/prometheus.md b/docs/integrations/prometheus.md index ed70ee8d84..b2e6a57193 100644 --- a/docs/integrations/prometheus.md +++ b/docs/integrations/prometheus.md @@ -8,9 +8,7 @@ To write to a remote M3DB cluster the simplest configuration is to run `m3coordi Start by downloading the [config template](https://github.com/m3db/m3db/blob/master/src/coordinator/config/m3coordinator-cluster-template.yml). Update the `namespaces` and the `client` section for a new cluster to match your cluster's configuration. -At the very least you'll need to specify the name of your namespace that you setup, the retention that it has. You can leave the metrics type as `unaggregated` since it's required by default to have a cluster that receives all Prometheus metrics unaggregated. In the future you might also want to aggregate and downsample metrics for longer retention, and you can come back and update the config once you've setup those clusters. - -You'll also need to set the static IPs or hostnames of your M3DB seed nodes +You'll need to specify the static IPs or hostnames of your M3DB seed nodes, and the name and retention values of the namespace you set up. You can leave the namespace storage metrics type as `unaggregated` since it's required by default to have a cluster that receives all Prometheus metrics unaggregated. In the future you might also want to aggregate and downsample metrics for longer retention, and you can come back and update the config once you've setup those clusters. It should look something like: @@ -73,7 +71,7 @@ Now start the process up: m3coordinator -f ``` -Or use the docker container: +Or, use the docker container: ``` docker pull quay.io/m3db/m3coordinator:latest diff --git a/src/coordinator/api/v1/handler/database/create.go b/src/coordinator/api/v1/handler/database/create.go index cb1601fdb3..614cf2a85e 100644 --- a/src/coordinator/api/v1/handler/database/create.go +++ b/src/coordinator/api/v1/handler/database/create.go @@ -53,6 +53,9 @@ const ( // CreateHTTPMethod is the HTTP method used with this resource. CreateHTTPMethod = http.MethodPost + // DefaultLocalHostID is the default local host ID when creating a database. + DefaultLocalHostID = "m3db_local" + idealDatapointsPerBlock = 720 blockSizeFromExpectedSeriesScalar = idealDatapointsPerBlock * int64(time.Hour) shardMultiplier = 64 @@ -321,7 +324,7 @@ func defaultedPlacementInitRequest( replicationFactor = 1 instances = []*placementpb.Instance{ &placementpb.Instance{ - Id: "m3db_local", + Id: DefaultLocalHostID, IsolationGroup: "local", Zone: "embedded", Weight: 1, diff --git a/src/coordinator/api/v1/handler/database/create_test.go b/src/coordinator/api/v1/handler/database/create_test.go index a0034b7099..81808c27f6 100644 --- a/src/coordinator/api/v1/handler/database/create_test.go +++ b/src/coordinator/api/v1/handler/database/create_test.go @@ -197,7 +197,7 @@ func TestLocalWithBlockSizeNanos(t *testing.T) { placementProto := &placementpb.Placement{ Instances: map[string]*placementpb.Instance{ "localhost": &placementpb.Instance{ - Id: "m3db_local", + Id: DefaultLocalHostID, IsolationGroup: "local", Zone: "embedded", Weight: 1, @@ -301,7 +301,7 @@ func TestLocalWithBlockSizeExpectedSeriesDatapointsPerHour(t *testing.T) { placementProto := &placementpb.Placement{ Instances: map[string]*placementpb.Instance{ "localhost": &placementpb.Instance{ - Id: "m3db_local", + Id: DefaultLocalHostID, IsolationGroup: "local", Zone: "embedded", Weight: 1,