Data Modernization
& Integrated Reporting Platform

flowchart TB
  classDef st fill:#171D28,stroke:#2A3340,color:#C9D1DC
  classDef ic fill:#1a1320,stroke:#075863,color:#fff
  classDef cm fill:#11161F,stroke:#0E97A8,color:#fff
  subgraph STORAGE[Storage Layer · HDFS / Object Store]
    P[Parquet / ORC files]:::st
  end
  subgraph TABLE[Table Format · Apache Iceberg]
    MD[Metadata + Manifest]:::ic
    SN[Snapshots · Time Travel]:::ic
    PR[Partition + Hidden Partitioning]:::ic
    SE[Schema Evolution]:::ic
  end
  subgraph COMPUTE[Compute Engines]
    SP[Spark · ETL/ELT]:::cm
    TRN[Trino · Ad-hoc SQL]:::cm
    HV[Hive · Batch]:::cm
  end
  P --> MD
  MD --> SN & PR & SE
  SP --> MD
  TRN --> MD
  HV --> MD
        

flowchart LR
  classDef lz fill:#171D28,stroke:#2A3340,color:#C9D1DC
  classDef bz fill:#2a1a10,stroke:#b5743a,color:#fff
  classDef sv fill:#1d2128,stroke:#9aa6b5,color:#fff
  classDef gd fill:#2a230f,stroke:#46BE6A,color:#fff
  LZ[Landing
raw as-is]:::lz --> BZ[Bronze
raw + audit cols]:::bz
  BZ -->|DQ + clean + dedupe| SV[Silver
conformed]:::sv
  SV -->|business rules + agg| GD[Gold
Fact & Dimension]:::gd
  GD --> DM[Data Mart Tematik
entitas Peserta]:::gd
  DQ1{{DQ Gate}} -.-> BZ
  DQ2{{DQ Gate}} -.-> SV
  DQ3{{DQ Gate}} -.-> GD
        

sequenceDiagram
  participant S as Source DB
  participant L as Tx Log / Redo
  participant C as CDC Connector
  participant K as Kafka
  participant Sp as Spark Streaming
  participant I as Iceberg (Bronze→Silver)
  S->>L: commit (insert/update/delete)
  C->>L: read log (low impact)
  C->>K: emit change event (op, before/after)
  K->>Sp: consume topic
  Sp->>I: MERGE / UPSERT (idempotent)
  Note over Sp,I: dedupe by PK + event ts
  I-->>Sp: ACK snapshot
        

flowchart LR
  classDef s fill:#171D28,stroke:#2A3340,color:#C9D1DC
  classDef t fill:#1a1320,stroke:#075863,color:#fff
  classDef g fill:#2a230f,stroke:#46BE6A,color:#fff
  T1[(src.pelayanan)]:::s --> B1[bronze.pelayanan]:::t
  T2[(src.peserta)]:::s --> B2[bronze.peserta]:::t
  T3[(src.klaim)]:::s --> B3[bronze.klaim]:::t
  B1 --> S1[silver.visit_clean]:::t
  B2 --> S2[silver.peserta_conf]:::t
  B3 --> S3[silver.klaim_clean]:::t
  S1 & S2 --> F1[gold.fact_pelayanan]:::g
  S3 & S2 --> F2[gold.fact_klaim]:::g
  S2 --> D1[gold.dim_peserta]:::g
  F1 & F2 & D1 --> M[mart.peserta_360]:::g
        

flowchart LR
  classDef a fill:#171D28,stroke:#2A3340,color:#C9D1DC
  classDef b fill:#1a1320,stroke:#075863,color:#fff
  classDef c fill:#2a230f,stroke:#46BE6A,color:#fff
  PL[Pipeline Jobs
Spark/Airflow]:::a --> MX[Metrics
latency/throughput]:::b
  PL --> LG[Logs]:::b
  PL --> DQ[DQ Results]:::b
  MX & LG & DQ --> COL[Collector / TSDB]:::b
  COL --> DASH[Monitoring Dashboard]:::c
  COL --> AL[Alerting Proaktif]:::c
  AL --> OPS[Tim Ops / On-call]:::a