{
  "title": "Splunk Admin - Daily Health Check",
  "description": "Daily operational checks: platform health, ingestion, license, search/scheduler, infrastructure, clustering, errors and security.",
  "inputs": {
    "input_global_trp": {
      "type": "input.timerange",
      "title": "Time range",
      "options": {
        "token": "global_time",
        "defaultValue": "-24h@h,now"
      }
    },
    "input_silent": {
      "type": "input.number",
      "title": "Silent host threshold (hours)",
      "options": {
        "token": "silent_hours",
        "defaultValue": 4
      }
    }
  },
  "defaults": {
    "dataSources": {
      "ds.search": {
        "options": {
          "queryParameters": {
            "earliest": "$global_time.earliest$",
            "latest": "$global_time.latest$"
          },
          "refresh": "5m",
          "refreshType": "delay"
        }
      }
    }
  },
  "dataSources": {
    "ds_kpi_instances": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_server",
        "query": "| stats count as instances"
      }
    },
    "ds_kpi_license": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal source=*license_usage.log* type=Usage | stats sum(b) as used | appendcols [| rest splunk_server=local /services/licenser/stacks | search type=enterprise | stats sum(quota) as quota] | eval pct=round(used/quota*100,1) | fields pct",
        "queryParameters": {
          "earliest": "@d",
          "latest": "now"
        }
      }
    },
    "ds_kpi_ingest": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_license_today",
        "query": "| stats sum(b) as b | eval GB=round(b/1073741824,2) | fields GB"
      }
    },
    "ds_kpi_blocked": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_queue",
        "query": "| stats sum(blocked_n) as blocked_events"
      }
    },
    "ds_kpi_skipped": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_sched",
        "query": "| where status=\"skipped\" | stats sum(n) as skipped"
      }
    },
    "ds_kpi_errors": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_splunkd",
        "query": "| where log_level=\"ERROR\" | stats sum(n) as errors"
      }
    },
    "ds_kpi_disk": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_disk",
        "query": "| eval pct_free=round(free/capacity*100,1) | stats min(pct_free) as min_free_pct"
      }
    },
    "ds_kpi_silent": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_silent",
        "query": "| eval hours_silent=round((now()-last_seen)/3600,1) | where hours_silent>$silent_hours$ | stats count as silent_hosts"
      }
    },
    "ds_ingest_throughput": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_index_thruput",
        "query": "| timechart span=15m sum(eval(kb/1024)) by series limit=8"
      }
    },
    "ds_queue_fill": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_queue",
        "query": "| search name=parsingqueue OR name=aggqueue OR name=typingqueue OR name=indexqueue | timechart span=5m max(fill_p90) by name"
      }
    },
    "ds_forwarders": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal source=*metrics.log* group=tcpin_connections | timechart span=15m dc(hostname) as \"Connected forwarders\""
      }
    },
    "ds_top_sourcetypes": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_license_today",
        "query": "| stats sum(b) as b by st | eval GB=round(b/1073741824,2) | sort - GB | head 10 | fields st GB"
      }
    },
    "ds_silent_hosts": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_silent",
        "query": "| eval hours_silent=round((now()-last_seen)/3600,1) | where hours_silent>$silent_hours$ | eval last_seen=strftime(last_seen,\"%F %T\") | sort - hours_silent | table host last_seen hours_silent"
      }
    },
    "ds_latency": {
      "type": "ds.search",
      "options": {
        "query": "index=* | eval lag=_indextime-_time | where lag>=0 | stats avg(lag) as avg_lag_s perc95(lag) as p95_lag_s max(lag) as max_lag_s count by index sourcetype | eval avg_lag_s=round(avg_lag_s,1), p95_lag_s=round(p95_lag_s,1) | sort - p95_lag_s | head 20",
        "queryParameters": {
          "earliest": "-15m",
          "latest": "now"
        }
      }
    },
    "ds_license_trend": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal source=*license_usage.log* type=RolloverSummary | eval GB=round(b/1073741824,2) | timechart span=1d sum(GB) as \"Daily GB\"",
        "queryParameters": {
          "earliest": "-30d@d",
          "latest": "now"
        }
      }
    },
    "ds_license_by_index": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_license_today",
        "query": "| stats sum(b) as b by idx | eval GB=round(b/1073741824,2) | sort - GB | fields idx GB"
      }
    },
    "ds_license_messages": {
      "type": "ds.search",
      "options": {
        "query": "| rest splunk_server=local /services/licenser/messages | eval created=strftime(create_time,\"%F %T\") | table created severity category description",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    },
    "ds_sched_status": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_sched",
        "query": "| timechart span=30m sum(n) by status"
      }
    },
    "ds_concurrency": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal source=*metrics.log* group=search_concurrency \"system total\" | timechart span=5m max(active_hist_searches) as historical max(active_realtime_searches) as realtime"
      }
    },
    "ds_skipped_table": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_sched",
        "query": "| where status=\"skipped\" | stats sum(n) as count values(reason) as reason by savedsearch_name app user | sort - count | head 15"
      }
    },
    "ds_long_running": {
      "type": "ds.search",
      "options": {
        "query": "| rest /services/search/jobs splunk_server=* count=0 | search isDone=0 | eval runtime_min=round(runDuration/60,1) | where runtime_min>=5 | rename \"eai:acl.app\" as app | eval search=substr(search,1,150) | sort - runtime_min | head 20 | table author app runtime_min dispatchState search",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    },
    "ds_top_users": {
      "type": "ds.search",
      "options": {
        "query": "index=_audit action=search info=completed total_run_time=* | stats count as searches sum(total_run_time) as total_runtime_s avg(total_run_time) as avg_runtime_s by user | eval total_runtime_min=round(total_runtime_s/60,1), avg_runtime_s=round(avg_runtime_s,1) | sort - total_runtime_min | head 10 | fields user searches total_runtime_min avg_runtime_s"
      }
    },
    "ds_slow_scheduled": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_sched",
        "query": "| where status=\"success\" | stats sum(n) as runs sum(rt_sum) as total_rt max(rt_max) as max_s by savedsearch_name app | eval avg_s=round(total_rt/runs,1), max_s=round(max_s,1) | sort - avg_s | head 15 | fields savedsearch_name app runs avg_s max_s"
      }
    },
    "ds_disk": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_disk",
        "query": "| eval capacity_GB=round(capacity/1024,1), free_GB=round(free/1024,1), pct_used=round((capacity-free)/capacity*100,1) | eval status=case(pct_used>=90,\"CRITICAL\",pct_used>=80,\"WARNING\",true(),\"OK\") | sort - pct_used | table splunk_server mount_point capacity_GB free_GB pct_used status"
      }
    },
    "ds_server_status": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_server",
        "query": "| eval started=strftime(startup_time,\"%F %T\"), uptime_hours=round((now()-startup_time)/3600,1) | table splunk_server version server_roles os_name started uptime_hours"
      }
    },
    "ds_cpu": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_resource",
        "query": "| eval cpu_pct=sys+usr | timechart span=5m avg(cpu_pct) by host limit=10"
      }
    },
    "ds_mem": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_resource",
        "query": "| eval mem_pct=round(mem_used/mem_total*100,1) | timechart span=5m avg(mem_pct) by host limit=10"
      }
    },
    "ds_idx_peers": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_cm_peers",
        "query": "| eval health=if(status=\"Up\",\"OK\",\"CHECK\") | table label status health site bucket_count"
      }
    },
    "ds_cm_info": {
      "type": "ds.search",
      "options": {
        "query": "| rest splunk_server=local /services/cluster/manager/info | table replication_factor search_factor maintenance_mode indexing_ready_flag service_ready_flag rolling_restart_flag",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    },
    "ds_fixup": {
      "type": "ds.search",
      "options": {
        "query": "| rest splunk_server=local /services/cluster/manager/fixup level=search_factor | stats count as fixup_tasks",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    },
    "ds_peers_down": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_cm_peers",
        "query": "| search status!=Up | stats count as peers_down"
      }
    },
    "ds_shc": {
      "type": "ds.search",
      "options": {
        "query": "| rest splunk_server=local /services/shcluster/captain/members | eval last_heartbeat=strftime(last_heartbeat,\"%F %T\") | table label status last_heartbeat replication_count mgmt_uri",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    },
    "ds_errors_time": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_splunkd",
        "query": "| timechart span=1h sum(n) by log_level"
      }
    },
    "ds_errors_top": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_splunkd",
        "query": "| stats sum(n) as count by component log_level | sort - count | head 15"
      }
    },
    "ds_errors_recent": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal sourcetype=splunkd log_level=ERROR | eval msg=substr(_raw,1,200) | table _time host component msg | head 50"
      }
    },
    "ds_failed_logins": {
      "type": "ds.search",
      "options": {
        "query": "index=_audit action=\"login attempt\" info=failed | stats count latest(_time) as last_attempt by user | eval last_attempt=strftime(last_attempt,\"%F %T\") | sort - count | head 20"
      }
    },
    "ds_bundle_failed": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_dist_peers",
        "query": "| search replicationStatus!=Successful | stats count as failed_peers"
      }
    },
    "ds_dist_down": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_dist_peers",
        "query": "| search status!=Up | stats count as peers_down"
      }
    },
    "ds_bundle_issues": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_splunkd",
        "query": "| where component=\"DistributedBundleReplicationManager\" | stats sum(n) as bundle_issues"
      }
    },
    "ds_dist_peers": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_dist_peers",
        "query": "| eval health=if(status=\"Up\" AND replicationStatus=\"Successful\",\"OK\",\"CHECK\") | sort - health | table peerName status replicationStatus health version"
      }
    },
    "ds_bundle_time": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_splunkd",
        "query": "| where component=\"DistributedBundleReplicationManager\" | timechart span=1h sum(n) by log_level"
      }
    },
    "ds_bundle_recent": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal sourcetype=splunkd component=DistributedBundleReplicationManager (log_level=ERROR OR log_level=WARN) | eval msg=substr(_raw,1,220) | table _time host log_level msg | head 50"
      }
    },
    "ds_conf_repl": {
      "type": "ds.chain",
      "options": {
        "extend": "ds_base_splunkd",
        "query": "| search component=ConfReplication* OR component=SHC* OR component=*Raft* | stats sum(n) as count by component log_level | sort - count | head 15"
      }
    },
    "ds_base_license_today": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal source=*license_usage.log* type=Usage | fields b idx st",
        "queryParameters": {
          "earliest": "@d",
          "latest": "now"
        }
      }
    },
    "ds_base_index_thruput": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal source=*metrics.log* group=per_index_thruput series!=_* | bin _time span=15m | stats sum(kb) as kb by _time series"
      }
    },
    "ds_base_queue": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal source=*metrics.log* group=queue | eval fill_pct=round(current_size_kb/max_size_kb*100,1), blocked_n=if(blocked=\"true\",1,0) | bin _time span=5m | stats perc90(fill_pct) as fill_p90 sum(blocked_n) as blocked_n by _time name"
      }
    },
    "ds_base_sched": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal sourcetype=scheduler | eval rt=tonumber(run_time), reason=coalesce(reason,\"none\") | bin _time span=30m | stats count as n sum(rt) as rt_sum max(rt) as rt_max by _time status savedsearch_name app user reason"
      }
    },
    "ds_base_splunkd": {
      "type": "ds.search",
      "options": {
        "query": "index=_internal sourcetype=splunkd (log_level=ERROR OR log_level=WARN) | bin _time span=1h | stats count as n by _time log_level component"
      }
    },
    "ds_base_resource": {
      "type": "ds.search",
      "options": {
        "query": "index=_introspection sourcetype=splunk_resource_usage component=Hostwide | bin _time span=5m | stats avg('data.cpu_system_pct') as sys avg('data.cpu_user_pct') as usr avg('data.mem_used') as mem_used avg('data.mem') as mem_total by _time host"
      }
    },
    "ds_base_silent": {
      "type": "ds.search",
      "options": {
        "query": "| tstats latest(_time) as last_seen where index=* by host",
        "queryParameters": {
          "earliest": "-7d",
          "latest": "now"
        }
      }
    },
    "ds_base_server": {
      "type": "ds.search",
      "options": {
        "query": "| rest /services/server/info splunk_server=*",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    },
    "ds_base_disk": {
      "type": "ds.search",
      "options": {
        "query": "| rest /services/server/status/partitions-space splunk_server=*",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    },
    "ds_base_cm_peers": {
      "type": "ds.search",
      "options": {
        "query": "| rest splunk_server=local /services/cluster/manager/peers",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    },
    "ds_base_dist_peers": {
      "type": "ds.search",
      "options": {
        "query": "| rest splunk_server=local /services/search/distributed/peers",
        "queryParameters": {
          "earliest": "-5m",
          "latest": "now"
        }
      }
    }
  },
  "visualizations": {
    "viz_h1": {
      "type": "splunk.markdown",
      "options": {
        "markdown": "## 1. Platform health at a glance",
        "fontSize": "large"
      }
    },
    "viz_h2": {
      "type": "splunk.markdown",
      "options": {
        "markdown": "## 2. Indexing and ingestion",
        "fontSize": "large"
      }
    },
    "viz_h3": {
      "type": "splunk.markdown",
      "options": {
        "markdown": "## 3. License",
        "fontSize": "large"
      }
    },
    "viz_h4": {
      "type": "splunk.markdown",
      "options": {
        "markdown": "## 4. Search and scheduler",
        "fontSize": "large"
      }
    },
    "viz_h5": {
      "type": "splunk.markdown",
      "options": {
        "markdown": "## 5. Infrastructure",
        "fontSize": "large"
      }
    },
    "viz_h6": {
      "type": "splunk.markdown",
      "options": {
        "markdown": "## 6. Clustering (run on, or with search access to, the cluster manager / SHC member)",
        "fontSize": "large"
      }
    },
    "viz_h7": {
      "type": "splunk.markdown",
      "options": {
        "markdown": "## 7. Errors and security",
        "fontSize": "large"
      }
    },
    "viz_kpi_instances": {
      "type": "splunk.singlevalue",
      "title": "Instances reachable",
      "dataSources": {
        "primary": "ds_kpi_instances"
      },
      "options": {
        "majorValue": "> primary | seriesByName('instances') | lastPoint()",
        "trendDisplay": "off",
        "underLabel": "compare with expected count"
      }
    },
    "viz_kpi_license": {
      "type": "splunk.singlevalue",
      "title": "License used today",
      "dataSources": {
        "primary": "ds_kpi_license"
      },
      "options": {
        "majorValue": "> primary | seriesByName('pct') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "unit": "%",
        "trendDisplay": "off",
        "underLabel": "of daily quota"
      },
      "context": {
        "colorConfig": [
          {
            "to": 70,
            "value": "#118832"
          },
          {
            "from": 70,
            "to": 90,
            "value": "#CBA700"
          },
          {
            "from": 90,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_kpi_ingest": {
      "type": "splunk.singlevalue",
      "title": "Indexed today",
      "dataSources": {
        "primary": "ds_kpi_ingest"
      },
      "options": {
        "majorValue": "> primary | seriesByName('GB') | lastPoint()",
        "unit": "GB",
        "trendDisplay": "off",
        "underLabel": "since midnight"
      }
    },
    "viz_kpi_blocked": {
      "type": "splunk.singlevalue",
      "title": "Blocked queue events",
      "dataSources": {
        "primary": "ds_kpi_blocked"
      },
      "options": {
        "majorValue": "> primary | seriesByName('blocked_events') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "selected time range"
      },
      "context": {
        "colorConfig": [
          {
            "to": 1,
            "value": "#118832"
          },
          {
            "from": 1,
            "to": 10,
            "value": "#CBA700"
          },
          {
            "from": 10,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_kpi_skipped": {
      "type": "splunk.singlevalue",
      "title": "Skipped searches",
      "dataSources": {
        "primary": "ds_kpi_skipped"
      },
      "options": {
        "majorValue": "> primary | seriesByName('skipped') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "selected time range"
      },
      "context": {
        "colorConfig": [
          {
            "to": 1,
            "value": "#118832"
          },
          {
            "from": 1,
            "to": 20,
            "value": "#CBA700"
          },
          {
            "from": 20,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_kpi_errors": {
      "type": "splunk.singlevalue",
      "title": "splunkd errors",
      "dataSources": {
        "primary": "ds_kpi_errors"
      },
      "options": {
        "majorValue": "> primary | seriesByName('errors') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "selected time range"
      },
      "context": {
        "colorConfig": [
          {
            "to": 50,
            "value": "#118832"
          },
          {
            "from": 50,
            "to": 500,
            "value": "#CBA700"
          },
          {
            "from": 500,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_kpi_disk": {
      "type": "splunk.singlevalue",
      "title": "Lowest free disk",
      "dataSources": {
        "primary": "ds_kpi_disk"
      },
      "options": {
        "majorValue": "> primary | seriesByName('min_free_pct') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "unit": "%",
        "trendDisplay": "off",
        "underLabel": "worst partition"
      },
      "context": {
        "colorConfig": [
          {
            "to": 10,
            "value": "#D41F1F"
          },
          {
            "from": 10,
            "to": 20,
            "value": "#CBA700"
          },
          {
            "from": 20,
            "value": "#118832"
          }
        ]
      }
    },
    "viz_kpi_silent": {
      "type": "splunk.singlevalue",
      "title": "Silent hosts",
      "dataSources": {
        "primary": "ds_kpi_silent"
      },
      "options": {
        "majorValue": "> primary | seriesByName('silent_hosts') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "no data beyond threshold"
      },
      "context": {
        "colorConfig": [
          {
            "to": 1,
            "value": "#118832"
          },
          {
            "from": 1,
            "to": 10,
            "value": "#CBA700"
          },
          {
            "from": 10,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_ingest_throughput": {
      "type": "splunk.area",
      "title": "Indexing throughput by index (MB per 15 min)",
      "dataSources": {
        "primary": "ds_ingest_throughput"
      },
      "options": {
        "stackMode": "stacked",
        "legendDisplay": "bottom",
        "xAxisTitleVisibility": "hide",
        "yAxisTitleText": "MB"
      }
    },
    "viz_queue_fill": {
      "type": "splunk.line",
      "title": "Indexing queue fill % (90th percentile)",
      "dataSources": {
        "primary": "ds_queue_fill"
      },
      "options": {
        "legendDisplay": "bottom",
        "xAxisTitleVisibility": "hide",
        "yAxisTitleText": "% full",
        "yAxisMax": 100
      }
    },
    "viz_forwarders": {
      "type": "splunk.line",
      "title": "Connected forwarders",
      "dataSources": {
        "primary": "ds_forwarders"
      },
      "options": {
        "legendDisplay": "off",
        "xAxisTitleVisibility": "hide",
        "yAxisTitleText": "Forwarders"
      }
    },
    "viz_top_sourcetypes": {
      "type": "splunk.bar",
      "title": "Top 10 sourcetypes by volume today (GB)",
      "dataSources": {
        "primary": "ds_top_sourcetypes"
      },
      "options": {
        "legendDisplay": "off",
        "xAxisTitleVisibility": "hide",
        "yAxisTitleVisibility": "hide"
      }
    },
    "viz_silent_hosts": {
      "type": "splunk.table",
      "title": "Hosts silent beyond threshold (seen in last 7 days)",
      "dataSources": {
        "primary": "ds_silent_hosts"
      },
      "options": {
        "count": 15,
        "showRowNumbers": false
      }
    },
    "viz_latency": {
      "type": "splunk.table",
      "title": "Indexing latency, last 15 min (seconds, _indextime - _time)",
      "dataSources": {
        "primary": "ds_latency"
      },
      "options": {
        "count": 15,
        "showRowNumbers": false
      }
    },
    "viz_license_trend": {
      "type": "splunk.column",
      "title": "Daily license usage, last 30 days (GB)",
      "dataSources": {
        "primary": "ds_license_trend"
      },
      "options": {
        "legendDisplay": "off",
        "xAxisTitleVisibility": "hide",
        "yAxisTitleText": "GB"
      }
    },
    "viz_license_by_index": {
      "type": "splunk.pie",
      "title": "License usage by index today (GB)",
      "dataSources": {
        "primary": "ds_license_by_index"
      },
      "options": {
        "collapseThreshold": 0.02
      }
    },
    "viz_license_messages": {
      "type": "splunk.table",
      "title": "License messages (warnings / violations)",
      "dataSources": {
        "primary": "ds_license_messages"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_sched_status": {
      "type": "splunk.column",
      "title": "Scheduled search results by status",
      "dataSources": {
        "primary": "ds_sched_status"
      },
      "options": {
        "stackMode": "stacked",
        "legendDisplay": "bottom",
        "xAxisTitleVisibility": "hide"
      }
    },
    "viz_concurrency": {
      "type": "splunk.line",
      "title": "Search concurrency (system total)",
      "dataSources": {
        "primary": "ds_concurrency"
      },
      "options": {
        "legendDisplay": "bottom",
        "xAxisTitleVisibility": "hide",
        "yAxisTitleText": "Active searches"
      }
    },
    "viz_skipped_table": {
      "type": "splunk.table",
      "title": "Skipped scheduled searches",
      "dataSources": {
        "primary": "ds_skipped_table"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_long_running": {
      "type": "splunk.table",
      "title": "Running searches over 5 minutes",
      "dataSources": {
        "primary": "ds_long_running"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_top_users": {
      "type": "splunk.table",
      "title": "Top users by total search runtime",
      "dataSources": {
        "primary": "ds_top_users"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_slow_scheduled": {
      "type": "splunk.table",
      "title": "Slowest scheduled searches (average runtime)",
      "dataSources": {
        "primary": "ds_slow_scheduled"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_disk": {
      "type": "splunk.table",
      "title": "Disk usage by partition (warn 80%, critical 90%)",
      "dataSources": {
        "primary": "ds_disk"
      },
      "options": {
        "count": 15,
        "showRowNumbers": false
      }
    },
    "viz_server_status": {
      "type": "splunk.table",
      "title": "Splunk instances: version, roles, uptime",
      "dataSources": {
        "primary": "ds_server_status"
      },
      "options": {
        "count": 15,
        "showRowNumbers": false
      }
    },
    "viz_cpu": {
      "type": "splunk.line",
      "title": "CPU % by host",
      "dataSources": {
        "primary": "ds_cpu"
      },
      "options": {
        "legendDisplay": "bottom",
        "xAxisTitleVisibility": "hide",
        "yAxisTitleText": "% CPU"
      }
    },
    "viz_mem": {
      "type": "splunk.line",
      "title": "Memory used % by host",
      "dataSources": {
        "primary": "ds_mem"
      },
      "options": {
        "legendDisplay": "bottom",
        "xAxisTitleVisibility": "hide",
        "yAxisTitleText": "% memory"
      }
    },
    "viz_idx_peers": {
      "type": "splunk.table",
      "title": "Indexer cluster peers",
      "dataSources": {
        "primary": "ds_idx_peers"
      },
      "options": {
        "count": 15,
        "showRowNumbers": false
      }
    },
    "viz_cm_info": {
      "type": "splunk.table",
      "title": "Cluster manager: RF / SF, maintenance and readiness",
      "dataSources": {
        "primary": "ds_cm_info"
      },
      "options": {
        "count": 5,
        "showRowNumbers": false
      }
    },
    "viz_shc": {
      "type": "splunk.table",
      "title": "Search head cluster members",
      "dataSources": {
        "primary": "ds_shc"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_fixup": {
      "type": "splunk.singlevalue",
      "title": "Pending fixup tasks",
      "dataSources": {
        "primary": "ds_fixup"
      },
      "options": {
        "majorValue": "> primary | seriesByName('fixup_tasks') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "search factor level"
      },
      "context": {
        "colorConfig": [
          {
            "to": 1,
            "value": "#118832"
          },
          {
            "from": 1,
            "to": 50,
            "value": "#CBA700"
          },
          {
            "from": 50,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_peers_down": {
      "type": "splunk.singlevalue",
      "title": "Indexer peers not Up",
      "dataSources": {
        "primary": "ds_peers_down"
      },
      "options": {
        "majorValue": "> primary | seriesByName('peers_down') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "should be 0"
      },
      "context": {
        "colorConfig": [
          {
            "to": 1,
            "value": "#118832"
          },
          {
            "from": 1,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_errors_time": {
      "type": "splunk.column",
      "title": "splunkd errors and warnings over time",
      "dataSources": {
        "primary": "ds_errors_time"
      },
      "options": {
        "stackMode": "stacked",
        "legendDisplay": "bottom",
        "xAxisTitleVisibility": "hide"
      }
    },
    "viz_errors_top": {
      "type": "splunk.table",
      "title": "Top error and warning components",
      "dataSources": {
        "primary": "ds_errors_top"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_errors_recent": {
      "type": "splunk.table",
      "title": "Most recent splunkd errors",
      "dataSources": {
        "primary": "ds_errors_recent"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_failed_logins": {
      "type": "splunk.table",
      "title": "Failed logins by user",
      "dataSources": {
        "primary": "ds_failed_logins"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_h8": {
      "type": "splunk.markdown",
      "options": {
        "markdown": "## 8. Search head cluster and bundle replication (run on an SHC member)",
        "fontSize": "large"
      }
    },
    "viz_kpi_bundle_failed": {
      "type": "splunk.singlevalue",
      "title": "Peers with failed bundle replication",
      "dataSources": {
        "primary": "ds_bundle_failed"
      },
      "options": {
        "majorValue": "> primary | seriesByName('failed_peers') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "should be 0"
      },
      "context": {
        "colorConfig": [
          {
            "to": 1,
            "value": "#118832"
          },
          {
            "from": 1,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_kpi_dist_down": {
      "type": "splunk.singlevalue",
      "title": "Search peers not Up",
      "dataSources": {
        "primary": "ds_dist_down"
      },
      "options": {
        "majorValue": "> primary | seriesByName('peers_down') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "should be 0"
      },
      "context": {
        "colorConfig": [
          {
            "to": 1,
            "value": "#118832"
          },
          {
            "from": 1,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_kpi_bundle_issues": {
      "type": "splunk.singlevalue",
      "title": "Bundle replication warnings and errors",
      "dataSources": {
        "primary": "ds_bundle_issues"
      },
      "options": {
        "majorValue": "> primary | seriesByName('bundle_issues') | lastPoint()",
        "majorColor": "> majorValue | rangeValue(colorConfig)",
        "trendDisplay": "off",
        "underLabel": "selected time range"
      },
      "context": {
        "colorConfig": [
          {
            "to": 1,
            "value": "#118832"
          },
          {
            "from": 1,
            "to": 20,
            "value": "#CBA700"
          },
          {
            "from": 20,
            "value": "#D41F1F"
          }
        ]
      }
    },
    "viz_dist_peers": {
      "type": "splunk.table",
      "title": "Search peers: status and bundle replication status",
      "dataSources": {
        "primary": "ds_dist_peers"
      },
      "options": {
        "count": 15,
        "showRowNumbers": false
      }
    },
    "viz_shc_members": {
      "type": "splunk.table",
      "title": "SHC members: status, heartbeat, replication count",
      "dataSources": {
        "primary": "ds_shc"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_bundle_time": {
      "type": "splunk.column",
      "title": "Bundle replication warnings and errors over time",
      "dataSources": {
        "primary": "ds_bundle_time"
      },
      "options": {
        "stackMode": "stacked",
        "legendDisplay": "bottom",
        "xAxisTitleVisibility": "hide"
      }
    },
    "viz_conf_repl": {
      "type": "splunk.table",
      "title": "SHC config replication and RAFT errors by component",
      "dataSources": {
        "primary": "ds_conf_repl"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    },
    "viz_bundle_recent": {
      "type": "splunk.table",
      "title": "Recent bundle replication messages",
      "dataSources": {
        "primary": "ds_bundle_recent"
      },
      "options": {
        "count": 10,
        "showRowNumbers": false
      }
    }
  },
  "layout": {
    "type": "absolute",
    "options": {
      "width": 1440,
      "height": 5220,
      "display": "auto-scale"
    },
    "globalInputs": [
      "input_global_trp",
      "input_silent"
    ],
    "structure": [
      {
        "item": "viz_h1",
        "type": "block",
        "position": {
          "x": 20,
          "y": 10,
          "w": 1400,
          "h": 40
        }
      },
      {
        "item": "viz_kpi_instances",
        "type": "block",
        "position": {
          "x": 25,
          "y": 55,
          "w": 165,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_license",
        "type": "block",
        "position": {
          "x": 200,
          "y": 55,
          "w": 165,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_ingest",
        "type": "block",
        "position": {
          "x": 375,
          "y": 55,
          "w": 165,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_blocked",
        "type": "block",
        "position": {
          "x": 550,
          "y": 55,
          "w": 165,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_skipped",
        "type": "block",
        "position": {
          "x": 725,
          "y": 55,
          "w": 165,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_errors",
        "type": "block",
        "position": {
          "x": 900,
          "y": 55,
          "w": 165,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_disk",
        "type": "block",
        "position": {
          "x": 1075,
          "y": 55,
          "w": 165,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_silent",
        "type": "block",
        "position": {
          "x": 1250,
          "y": 55,
          "w": 165,
          "h": 120
        }
      },
      {
        "item": "viz_h2",
        "type": "block",
        "position": {
          "x": 20,
          "y": 190,
          "w": 1400,
          "h": 40
        }
      },
      {
        "item": "viz_ingest_throughput",
        "type": "block",
        "position": {
          "x": 20,
          "y": 235,
          "w": 700,
          "h": 280
        }
      },
      {
        "item": "viz_queue_fill",
        "type": "block",
        "position": {
          "x": 730,
          "y": 235,
          "w": 690,
          "h": 280
        }
      },
      {
        "item": "viz_forwarders",
        "type": "block",
        "position": {
          "x": 20,
          "y": 525,
          "w": 700,
          "h": 260
        }
      },
      {
        "item": "viz_top_sourcetypes",
        "type": "block",
        "position": {
          "x": 730,
          "y": 525,
          "w": 690,
          "h": 260
        }
      },
      {
        "item": "viz_silent_hosts",
        "type": "block",
        "position": {
          "x": 20,
          "y": 795,
          "w": 700,
          "h": 300
        }
      },
      {
        "item": "viz_latency",
        "type": "block",
        "position": {
          "x": 730,
          "y": 795,
          "w": 690,
          "h": 300
        }
      },
      {
        "item": "viz_h3",
        "type": "block",
        "position": {
          "x": 20,
          "y": 1105,
          "w": 1400,
          "h": 40
        }
      },
      {
        "item": "viz_license_trend",
        "type": "block",
        "position": {
          "x": 20,
          "y": 1150,
          "w": 700,
          "h": 280
        }
      },
      {
        "item": "viz_license_by_index",
        "type": "block",
        "position": {
          "x": 730,
          "y": 1150,
          "w": 330,
          "h": 280
        }
      },
      {
        "item": "viz_license_messages",
        "type": "block",
        "position": {
          "x": 1070,
          "y": 1150,
          "w": 350,
          "h": 280
        }
      },
      {
        "item": "viz_h4",
        "type": "block",
        "position": {
          "x": 20,
          "y": 1440,
          "w": 1400,
          "h": 40
        }
      },
      {
        "item": "viz_sched_status",
        "type": "block",
        "position": {
          "x": 20,
          "y": 1485,
          "w": 700,
          "h": 280
        }
      },
      {
        "item": "viz_concurrency",
        "type": "block",
        "position": {
          "x": 730,
          "y": 1485,
          "w": 690,
          "h": 280
        }
      },
      {
        "item": "viz_skipped_table",
        "type": "block",
        "position": {
          "x": 20,
          "y": 1775,
          "w": 700,
          "h": 300
        }
      },
      {
        "item": "viz_long_running",
        "type": "block",
        "position": {
          "x": 730,
          "y": 1775,
          "w": 690,
          "h": 300
        }
      },
      {
        "item": "viz_top_users",
        "type": "block",
        "position": {
          "x": 20,
          "y": 2085,
          "w": 700,
          "h": 280
        }
      },
      {
        "item": "viz_slow_scheduled",
        "type": "block",
        "position": {
          "x": 730,
          "y": 2085,
          "w": 690,
          "h": 280
        }
      },
      {
        "item": "viz_h5",
        "type": "block",
        "position": {
          "x": 20,
          "y": 2375,
          "w": 1400,
          "h": 40
        }
      },
      {
        "item": "viz_disk",
        "type": "block",
        "position": {
          "x": 20,
          "y": 2420,
          "w": 700,
          "h": 300
        }
      },
      {
        "item": "viz_server_status",
        "type": "block",
        "position": {
          "x": 730,
          "y": 2420,
          "w": 690,
          "h": 300
        }
      },
      {
        "item": "viz_cpu",
        "type": "block",
        "position": {
          "x": 20,
          "y": 2730,
          "w": 700,
          "h": 260
        }
      },
      {
        "item": "viz_mem",
        "type": "block",
        "position": {
          "x": 730,
          "y": 2730,
          "w": 690,
          "h": 260
        }
      },
      {
        "item": "viz_h6",
        "type": "block",
        "position": {
          "x": 20,
          "y": 3000,
          "w": 1400,
          "h": 40
        }
      },
      {
        "item": "viz_idx_peers",
        "type": "block",
        "position": {
          "x": 20,
          "y": 3045,
          "w": 700,
          "h": 260
        }
      },
      {
        "item": "viz_cm_info",
        "type": "block",
        "position": {
          "x": 730,
          "y": 3045,
          "w": 690,
          "h": 260
        }
      },
      {
        "item": "viz_shc",
        "type": "block",
        "position": {
          "x": 20,
          "y": 3315,
          "w": 700,
          "h": 240
        }
      },
      {
        "item": "viz_fixup",
        "type": "block",
        "position": {
          "x": 730,
          "y": 3315,
          "w": 340,
          "h": 240
        }
      },
      {
        "item": "viz_peers_down",
        "type": "block",
        "position": {
          "x": 1080,
          "y": 3315,
          "w": 340,
          "h": 240
        }
      },
      {
        "item": "viz_h7",
        "type": "block",
        "position": {
          "x": 20,
          "y": 3565,
          "w": 1400,
          "h": 40
        }
      },
      {
        "item": "viz_errors_time",
        "type": "block",
        "position": {
          "x": 20,
          "y": 3610,
          "w": 700,
          "h": 260
        }
      },
      {
        "item": "viz_errors_top",
        "type": "block",
        "position": {
          "x": 730,
          "y": 3610,
          "w": 690,
          "h": 260
        }
      },
      {
        "item": "viz_errors_recent",
        "type": "block",
        "position": {
          "x": 20,
          "y": 3880,
          "w": 700,
          "h": 300
        }
      },
      {
        "item": "viz_failed_logins",
        "type": "block",
        "position": {
          "x": 730,
          "y": 3880,
          "w": 690,
          "h": 300
        }
      },
      {
        "item": "viz_h8",
        "type": "block",
        "position": {
          "x": 20,
          "y": 4200,
          "w": 1400,
          "h": 40
        }
      },
      {
        "item": "viz_kpi_bundle_failed",
        "type": "block",
        "position": {
          "x": 20,
          "y": 4245,
          "w": 225,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_dist_down",
        "type": "block",
        "position": {
          "x": 255,
          "y": 4245,
          "w": 225,
          "h": 120
        }
      },
      {
        "item": "viz_kpi_bundle_issues",
        "type": "block",
        "position": {
          "x": 490,
          "y": 4245,
          "w": 225,
          "h": 120
        }
      },
      {
        "item": "viz_dist_peers",
        "type": "block",
        "position": {
          "x": 730,
          "y": 4245,
          "w": 690,
          "h": 390
        }
      },
      {
        "item": "viz_shc_members",
        "type": "block",
        "position": {
          "x": 20,
          "y": 4375,
          "w": 700,
          "h": 260
        }
      },
      {
        "item": "viz_bundle_time",
        "type": "block",
        "position": {
          "x": 20,
          "y": 4645,
          "w": 700,
          "h": 260
        }
      },
      {
        "item": "viz_conf_repl",
        "type": "block",
        "position": {
          "x": 730,
          "y": 4645,
          "w": 690,
          "h": 260
        }
      },
      {
        "item": "viz_bundle_recent",
        "type": "block",
        "position": {
          "x": 20,
          "y": 4915,
          "w": 1400,
          "h": 280
        }
      }
    ]
  }
}