{ "cluster":{ "layers":{ "_valid":true, "_error":"some error description" }, "processes":{ "$map_key=zoneid":{ "version":"3.0.0", "machine_id":"0ccb4e0feddb5583010f6b77d9d10ece", "locality":{ "$map_key=localityName":"value" }, "class_source":{ "$enum":[ "command_line", "configure_auto", "set_class" ] }, "class_type":{ "$enum":[ "unset", "storage", "transaction", "resolution", "proxy", "master", "test" ] }, "degraded":true, "roles":[ { "query_queue_max":0, "local_rate":0, "input_bytes":{ "hz":0.0, "counter":0, "roughness":0.0 }, "stored_bytes":12341234, "kvstore_used_bytes":12341234, "kvstore_available_bytes":12341234, "kvstore_free_bytes":12341234, "kvstore_total_bytes":12341234, "durable_bytes":{ "hz":0.0, "counter":0, "roughness":0.0 }, "queue_disk_used_bytes":12341234, "queue_disk_available_bytes":12341234, "queue_disk_free_bytes":12341234, "queue_disk_total_bytes":12341234, "role":{ "$enum":[ "master", "proxy", "log", "storage", "resolver", "cluster_controller", "data_distributor", "ratekeeper", "router", "coordinator" ] }, "data_version":12341234, "durable_version":12341234, "data_lag": { "seconds":5.0, "versions":12341234 }, "durability_lag": { "seconds":5.0, "versions":12341234 }, "id":"eb84471d68c12d1d26f692a50000003f", "total_queries":{ "hz":0.0, "counter":0, "roughness":0.0 }, "finished_queries":{ "hz":0.0, "counter":0, "roughness":0.0 }, "bytes_queried":{ "hz":0.0, "counter":0, "roughness":0.0 }, "keys_queried":{ "hz":0.0, "counter":0, "roughness":0.0 }, "mutation_bytes":{ "hz":0.0, "counter":0, "roughness":0.0 }, "mutations":{ "hz":0.0, "counter":0, "roughness":0.0 }, "grv_latency_bands":{ "$map_key=upperBoundOfBand": 1 }, "read_latency_bands":{ "$map_key=upperBoundOfBand": 1 }, "commit_latency_bands":{ "$map_key=upperBoundOfBand": 1 } } ], "command_line":"-r simulation", "memory":{ "available_bytes":0, "limit_bytes":0, "unused_allocated_memory":0, "used_bytes":0 }, "messages":[ { "time":12345.12312, "type":"x", "name":{ "$enum":[ "file_open_error", "incorrect_cluster_file_contents", "process_error", "io_error", "io_timeout", "platform_error", "storage_server_lagging", "(other FDB error messages)" ] }, "raw_log_message":"", "description":"abc" } ], "fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece", "under_maintenance":true, "excluded":false, "address":"1.2.3.4:1234", "disk":{ "free_bytes":3451233456234, "reads":{ "hz":0.0, "counter":0, "sectors":0 }, "busy":0.0, "writes":{ "hz":0.0, "counter":0, "sectors":0 }, "total_bytes":123412341234 }, "uptime_seconds":1234.2345, "cpu":{ "usage_cores":0.0 }, "network":{ "current_connections":0, "connections_established":{ "hz":0.0 }, "connections_closed":{ "hz":0.0 }, "connection_errors":{ "hz":0.0 }, "megabits_sent":{ "hz":0.0 }, "megabits_received":{ "hz":0.0 }, "tls_policy_failures":{ "hz":0.0 } }, "run_loop_busy":0.2 } }, "old_logs":[ { "logs":[ { "id":"7f8d623d0cb9966e", "healthy":true, "address":"1.2.3.4:1234" } ], "log_replication_factor":3, "log_write_anti_quorum":0, "log_fault_tolerance":2, "remote_log_replication_factor":3, "remote_log_fault_tolerance":2, "satellite_log_replication_factor":3, "satellite_log_write_anti_quorum":0, "satellite_log_fault_tolerance":2 } ], "fault_tolerance":{ "max_zone_failures_without_losing_availability":0, "max_zone_failures_without_losing_data":0 }, "qos":{ "worst_queue_bytes_log_server":460, "batch_performance_limited_by":{ "reason_server_id":"7f8d623d0cb9966e", "reason_id":0, "name":{ "$enum":[ "workload", "storage_server_write_queue_size", "storage_server_write_bandwidth_mvcc", "storage_server_readable_behind", "log_server_mvcc_write_bandwidth", "log_server_write_queue", "storage_server_min_free_space", "storage_server_min_free_space_ratio", "log_server_min_free_space", "log_server_min_free_space_ratio", "storage_server_durability_lag" ] }, "description":"The database is not being saturated by the workload." }, "performance_limited_by":{ "reason_server_id":"7f8d623d0cb9966e", "reason_id":0, "name":{ "$enum":[ "workload", "storage_server_write_queue_size", "storage_server_write_bandwidth_mvcc", "storage_server_readable_behind", "log_server_mvcc_write_bandwidth", "log_server_write_queue", "storage_server_min_free_space", "storage_server_min_free_space_ratio", "log_server_min_free_space", "log_server_min_free_space_ratio", "storage_server_durability_lag" ] }, "description":"The database is not being saturated by the workload." }, "batch_transactions_per_second_limit":0, "transactions_per_second_limit":0, "batch_released_transactions_per_second":0, "released_transactions_per_second":0, "limiting_queue_bytes_storage_server":0, "worst_queue_bytes_storage_server":0, "limiting_version_lag_storage_server":0, "worst_version_lag_storage_server":0, "limiting_data_lag_storage_server":{ "versions":0, "seconds":0.0 }, "worst_data_lag_storage_server":{ "versions":0, "seconds":0.0 }, "limiting_durability_lag_storage_server":{ "versions":0, "seconds":0.0 }, "worst_durability_lag_storage_server":{ "versions":0, "seconds":0.0 } }, "incompatible_connections":[ ], "datacenter_lag":{ "seconds":1.0, "versions":1000000 }, "degraded_processes":0, "database_available":true, "database_locked":false, "generation":2, "latency_probe":{ "read_seconds":7, "immediate_priority_transaction_start_seconds":0.0, "batch_priority_transaction_start_seconds":0.0, "transaction_start_seconds":0.0, "commit_seconds":0.02 }, "clients":{ "count":1, "supported_versions":[ { "client_version":"3.0.0", "connected_clients":[ { "address":"127.0.0.1:9898", "log_group":"default" } ], "max_protocol_clients":[ { "address":"127.0.0.1:9898", "log_group":"default" } ], "count" : 1, "max_protocol_count" : 1, "protocol_version" : "fdb00a400050001", "source_version" : "9430e1127b4991cbc5ab2b17f41cfffa5de07e9d" } ] }, "page_cache":{ "log_hit_rate":0.5, "storage_hit_rate":0.5 }, "messages":[ { "reasons":[ { "description":"Blah." } ], "unreachable_processes":[ { "address":"1.2.3.4:1234" } ], "name":{ "$enum":[ "unreachable_master_worker", "unreachable_dataDistributor_worker", "unreachable_ratekeeper_worker", "unreadable_configuration", "full_replication_timeout", "client_issues", "unreachable_processes", "immediate_priority_transaction_start_probe_timeout", "batch_priority_transaction_start_probe_timeout", "transaction_start_probe_timeout", "read_probe_timeout", "commit_probe_timeout", "storage_servers_error", "status_incomplete", "layer_status_incomplete", "database_availability_timeout", "consistencycheck_suspendkey_fetch_timeout", "consistencycheck_disabled" ] }, "issues":[ { "name":{ "$enum":[ "incorrect_cluster_file_contents" ] }, "description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally." } ], "description":"abc" } ], "recovery_state":{ "required_resolvers":1, "required_proxies":1, "name":{ "$enum":[ "reading_coordinated_state", "locking_coordinated_state", "locking_old_transaction_servers", "reading_transaction_system_state", "configuration_missing", "configuration_never_created", "configuration_invalid", "recruiting_transaction_servers", "initializing_transaction_servers", "recovery_transaction", "writing_coordinated_state", "accepting_commits", "all_logs_recruited", "storage_recovered", "fully_recovered" ] }, "required_logs":3, "missing_logs":"7f8d623d0cb9966e", "active_generations":1, "description":"Recovery complete." }, "workload":{ "operations":{ "writes":{ "hz":0.0, "counter":0, "roughness":0.0 }, "reads":{ "hz":0.0, "counter":0, "roughness":0.0 }, "read_requests":{ "hz":0.0, "counter":0, "roughness":0.0 }, "location_requests":{ "hz":0.0, "counter":0, "roughness":0.0 }, "memory_errors":{ "hz":0.0, "counter":0, "roughness":0.0 } }, "bytes":{ "written":{ "hz":0.0, "counter":0, "roughness":0.0 }, "read":{ "hz":0.0, "counter":0, "roughness":0.0 } }, "keys":{ "read":{ "hz":0.0, "counter":0, "roughness":0.0 } }, "transactions":{ "started":{ "hz":0.0, "counter":0, "roughness":0.0 }, "started_immediate_priority":{ "hz":0.0, "counter":0, "roughness":0.0 }, "started_default_priority":{ "hz":0.0, "counter":0, "roughness":0.0 }, "started_batch_priority":{ "hz":0.0, "counter":0, "roughness":0.0 }, "conflicted":{ "hz":0.0, "counter":0, "roughness":0.0 }, "committed":{ "hz":0.0, "counter":0, "roughness":0.0 } } }, "cluster_controller_timestamp":1415650089, "protocol_version":"fdb00a400050001", "connection_string":"a:a@127.0.0.1:4000", "full_replication":true, "maintenance_zone":"0ccb4e0fdbdb5583010f6b77d9d10ece", "maintenance_seconds_remaining":1.0, "data_distribution_disabled_for_ss_failures":true, "data_distribution_disabled_for_rebalance":true, "data_distribution_disabled":true, "configuration":{ "log_anti_quorum":0, "log_replicas":2, "log_replication_policy":"(zoneid^3x1)", "redundancy_mode":{ "$enum":[ "single", "double", "triple", "three_datacenter", "three_datacenter_fallback", "three_data_hall", "three_data_hall_fallback" ]}, "regions":[{ "datacenters":[{ "id":"mr", "priority":1, "satellite":1, "satellite_logs":2 }], "satellite_redundancy_mode":{ "$enum":[ "one_satellite_single", "one_satellite_double", "one_satellite_triple", "two_satellite_safe", "two_satellite_fast" ]}, "satellite_log_replicas":1, "satellite_usable_dcs":1, "satellite_anti_quorum":0, "satellite_log_policy":"(zoneid^3x1)", "satellite_logs":2 }], "remote_redundancy_mode":{ "$enum":[ "remote_default", "remote_single", "remote_double", "remote_triple", "remote_three_data_hall" ]}, "remote_log_replicas":3, "remote_logs":5, "log_routers":10, "usable_regions":1, "repopulate_anti_quorum":1, "storage_replicas":1, "resolvers":1, "storage_replication_policy":"(zoneid^3x1)", "logs":2, "log_version":2, "log_engine":1, "log_spill":1, "storage_engine":{ "$enum":[ "ssd", "ssd-1", "ssd-2", "ssd-redwood-experimental", "memory" ]}, "coordinators_count":1, "excluded_servers":[ { "address":"10.0.4.1" } ], "auto_proxies":3, "auto_resolvers":1, "auto_logs":3, "proxies":5 }, "data":{ "least_operating_space_bytes_log_server":0, "average_partition_size_bytes":0, "state":{ "healthy":true, "min_replicas_remaining":0, "name":{ "$enum":[ "initializing", "missing_data", "healing", "optimizing_team_collections", "healthy_populating_region", "healthy_repartitioning", "healthy_removing_server", "healthy_rebalancing", "healthy" ] }, "description":"" }, "least_operating_space_ratio_storage_server":0.1, "max_machine_failures_without_losing_availability":0, "total_disk_used_bytes":0, "total_kv_size_bytes":0, "system_kv_size_bytes":0, "partitions_count":2, "moving_data":{ "total_written_bytes":0, "in_flight_bytes":0, "in_queue_bytes":0, "highest_priority":0 }, "team_trackers":[ { "primary":true, "in_flight_bytes":0, "unhealthy_servers":0, "state":{ "healthy":true, "min_replicas_remaining":0, "name":{ "$enum":[ "initializing", "missing_data", "healing", "optimizing_team_collections", "healthy_populating_region", "healthy_repartitioning", "healthy_removing_server", "healthy_rebalancing", "healthy" ] }, "description":"" } } ], "least_operating_space_bytes_storage_server":0, "max_machine_failures_without_losing_data":0 }, "machines":{ "$map_key=machineid":{ "network":{ "megabits_sent":{ "hz":0.0 }, "megabits_received":{ "hz":0.0 }, "tcp_segments_retransmitted":{ "hz":0.0 } }, "memory":{ "free_bytes":0, "committed_bytes":0, "total_bytes":0 }, "contributing_workers":4, "datacenter_id":"6344abf1813eb05b", "excluded":false, "address":"1.2.3.4", "machine_id":"6344abf1813eb05b", "locality":{ "$map_key=localityName":"value" }, "cpu":{ "logical_core_utilization":0.4 } } } }, "client":{ "coordinators":{ "coordinators":[ { "reachable":true, "address":"127.0.0.1:4701" } ], "quorum_reachable":true }, "database_status":{ "available":true, "healthy":true }, "messages":[ { "name":{ "$enum":[ "inconsistent_cluster_file", "unreachable_cluster_controller", "no_cluster_controller", "status_incomplete_client", "status_incomplete_coordinators", "status_incomplete_error", "status_incomplete_timeout", "status_incomplete_cluster", "quorum_not_reachable" ] }, "description":"The cluster file is not up to date." } ], "timestamp":1415650089, "cluster_file":{ "path":"/etc/foundationdb/fdb.cluster", "up_to_date":true } } }