.
├── app/
│   ├── config.py                 # Configuração centralizada com detecção de ambiente
│   ├── jobs/
│   │   └── sentiment.py          # PySpark job principal (Kafka/Pub-Sub adaptativo)
│   ├── producers/
│   │   └── kafka_producer.py     # Gerador de dados sintéticos
│   └── schemas/
│       ├── input_schema.py       # Schema mensagem Kafka/Pub-Sub
│       └── bigquery_schema.json  # Schema tabela BigQuery
├── scripts/
│   ├── deploy.sh                 # Orquestrador de deploy completo
│   ├── submit_job.sh             # Submissão job Dataproc
│   ├── cleanup.sh                # Destruição de recursos GCP
│   └── bootstrap/
│       ├── install_kafka.sh      # Setup Kafka no cluster (local)
│       └── install_nltk.sh       # Download NLTK VADER data
├── terraform/
│   ├── environments/
│   │   └── dev/
│   │       ├── main.tf           # Root module
│   │       ├── variables.tf      # Input variables
│   │       ├── outputs.tf        # Output values
│   │       └── terraform.tfvars  # Environment config (e2-standard-4)
│   └── modules/
│       ├── dataproc/             # Cluster + IAM + auto-delete
│       ├── network/              # VPC + Firewall
│       ├── storage/              # GCS buckets (checkpoints)
│       └── bigquery/             # Dataset + Tables (partitioned)
├── docs/
│   ├── architecture.md           # ADRs (Architecture Decision Records)
│   ├── runbook.md                # Procedimentos operacionais
│   └── img/                      # Screenshots e diagramas
├── tests/
│   ├── unit/                     # Testes unitários (pytest)
│   └── integration/              # Testes E2E
├── .gitignore
├── requirements.txt              # Dependências Python
└── README.md

# Ferramentas necessárias
- gcloud CLI (>= 400.0.0)
- Terraform (>= 1.6.0)
- Python (>= 3.10)
- Docker + Docker Compose (para ambiente local)

# Permissões GCP necessárias
- roles/editor (ou granular: dataproc.admin, storage.admin, bigquery.admin, pubsub.admin)

# 1. Criar projeto no GCP
gcloud projects create spark-streaming-gcp-terraform --name="Sentiment Pipeline"
gcloud config set project spark-streaming-gcp-terraform

# 2. Habilitar faturamento (obrigatório para Dataproc)
# Acesse: https://console.cloud.google.com/billing/linkedaccount?project=spark-streaming-gcp-terraform

# 3. Habilitar APIs necessárias
gcloud services enable \
  dataproc.googleapis.com \
  storage.googleapis.com \
  bigquery.googleapis.com \
  pubsub.googleapis.com \
  compute.googleapis.com

# 4. Exportar variáveis de ambiente
export GCP_PROJECT_ID="spark-streaming-gcp-terraform"
export GCP_REGION="us-central1"
export TF_VAR_project_id=$GCP_PROJECT_ID
export TF_VAR_region=$GCP_REGION

# 1. Clone o repositório
git clone https://github.com/seu-usuario/spark-streaming-gcp-terraform.git
cd spark-streaming-gcp-terraform

# 2. Autentique no GCP
gcloud auth application-default login

# 3. Inicialize o Terraform
cd terraform/environments/dev
terraform init

# 4. Execute o deploy automatizado
cd ../../..
chmod +x scripts/deploy.sh
./scripts/deploy.sh

# 5. Verifique o status do cluster
gcloud dataproc clusters describe spark-sentiment-dev \
  --region=$GCP_REGION \
  --format="value(status.state,config.masterConfig.machineTypeUri)"

# Método 1: Script automatizado (recomendado)
./scripts/submit_job.sh

# Método 2: Comando direto com configurações otimizadas
gcloud dataproc jobs submit pyspark \
  gs://spark-sentiment-dev-code/jobs/sentiment.py \
  --cluster=spark-sentiment-dev \
  --region=us-central1 \
  --properties=spark.executor.memory=4g,spark.executor.cores=2,spark.sql.adaptive.enabled=true \
  --jars=gs://spark-lib/bigquery/spark-bigquery-with-dependencies_2.12-0.32.2.jar \
  --files=gs://spark-sentiment-dev-code/config/nltk_data.zip

# Logs do Spark Streaming
gcloud dataproc jobs list \
  --cluster=spark-sentiment-dev \
  --region=us-central1 \
  --state-filter=ACTIVE

# Métricas do Pub/Sub
gcloud pubsub subscriptions describe sentiment-sub \
  --format="value(pushConfig.pushEndpoint,ackDeadlineSeconds)"

# Query BigQuery para verificar dados recentes
bq query --use_legacy_sql=false \
  'SELECT 
    COUNT(*) as total_records,
    COUNTIF(sentiment_score > 0) as positive,
    COUNTIF(sentiment_score < 0) as negative,
    MAX(processed_at) as last_processed
   FROM `'"$GCP_PROJECT_ID"'.sentiment_analysis.tweets`
   WHERE DATE(timestamp) = CURRENT_DATE()'

# Verificar lag de processamento
bq query --use_legacy_sql=false \
  'SELECT 
    TIMESTAMP_DIFF(processed_at, timestamp, SECOND) as lag_seconds
   FROM `'"$GCP_PROJECT_ID"'.sentiment_analysis.tweets`
   ORDER BY processed_at DESC
   LIMIT 10'

# Diagnóstico
gcloud dataproc clusters describe spark-sentiment-dev \
  --region=us-central1 \
  --format="value(status.state,status.stateStartTime)"

# Verificar logs de inicialização
gcloud logging read "resource.type=cloud_dataproc_cluster AND resource.labels.cluster_name=spark-sentiment-dev" \
  --limit=50 \
  --format=json \
  --freshness=1h

# Obter logs detalhados do job
gcloud dataproc jobs describe <JOB_ID> \
  --region=us-central1 \
  --format="value(status.state,driverOutputResourceUri)"

# Verificar checkpoint no GCS
gsutil ls -lh gs://spark-sentiment-dev-checkpoints/sentiment_job_v1/

# Limpar checkpoint corrompido (CUIDADO: perde estado)
gsutil -m rm -r gs://spark-sentiment-dev-checkpoints/sentiment_job_v1/

# Recriar bucket com lifecycle
gsutil mb -p $GCP_PROJECT_ID -l $GCP_REGION gs://spark-sentiment-dev-checkpoints
gsutil lifecycle set terraform/modules/storage/lifecycle.json gs://spark-sentiment-dev-checkpoints

# Verificar lag de mensagens
gcloud pubsub subscriptions describe sentiment-sub \
  --format="value(name,numUndeliveredMessages,oldestUnackedMessageAge)"

# Purgar mensagens antigas (reset)
gcloud pubsub subscriptions seek sentiment-sub --time=$(date -u +%Y-%m-%dT%H:%M:%SZ)