Thursday, December 18, 2014

Using jq to talk with elasticsearch

I've recently found a new tool, quite useful, if you often talk to json services. It's named jq, please do not confuse it with jQuery. Eg. let's take Elasticsearch. I want to know which JVM it runs within. So I need to ask like this:
curl -XGET es1.grey:9200/_cluster/stats
so here's the output:
{"timestamp":1418913590962,"cluster_name":"gc-pb-grey","status":"green","indices":{"count":31,"shards":{"total":252,"primaries":126,"replication":1.0,"index":{"shards":{"min":4,"max":10,"avg":8.129032258064516},"primaries":{"min":2,"max":5,"avg":4.064516129032258},"replication":{"min":1.0,"max":1.0,"avg":1.0}}},"docs":{"count":162713289,"deleted":188051},"store":{"size":"44.2gb","size_in_bytes":47543046975,"throttle_time":"1.9d","throttle_time_in_millis":172301216},"fielddata":{"memory_size":"2.6gb","memory_size_in_bytes":2858909795,"evictions":0},"filter_cache":{"memory_size":"4.8gb","memory_size_in_bytes":5161692177,"evictions":504563},"id_cache":{"memory_size":"0b","memory_size_in_bytes":0},"completion":{"size":"0b","size_in_bytes":0},"segments":{"count":2000,"memory":"2.9gb","memory_in_bytes":3169080575}},"nodes":{"count":{"total":8,"master_only":0,"data_only":0,"master_data":8,"client":0},"versions":["0.90.12"],"os":{"available_processors":32,"mem":{"total":"188.7gb","total_in_bytes":202616012800},"cpu":[{"vendor":"AMD","model":"Opteron","mhz":2599,"total_cores":4,"total_sockets":4,"cores_per_socket":1,"cache_size":"512b","cache_size_in_bytes":512,"count":8}]},"process":{"cpu":{"percent":115},"open_file_descriptors":{"min":1356,"max":1537,"avg":1429}},"jvm":{"max_uptime":"8d","max_uptime_in_millis":693602733,"versions":[{"version":"1.6.0_30","vm_name":"OpenJDK 64-Bit Server VM","vm_version":"23.25-b01","vm_vendor":"Sun Microsystems Inc.","count":8}],"mem":{"heap_used":"59.5gb","heap_used_in_bytes":63949850024,"heap_max":"127.7gb","heap_max_in_bytes":137160032256},"threads":621},"fs":{"total":"94.4gb","total_in_bytes":101461655552,"free":"48.9gb","free_in_bytes":52530081792,"available":"44.1gb","available_in_bytes":47376134144,"disk_reads":570224,"disk_writes":164405664,"disk_io_op":164975888,"disk_read_size":"16.1gb","disk_read_size_in_bytes":17347735552,"disk_write_size":"612.6gb","disk_write_size_in_bytes":657792794624,"disk_io_size":"628.7gb","disk_io_size_in_bytes":675140530176,"disk_queue":"0","disk_service_time":"0.9"},"plugins":[{"name":"bigdesk","description":"No description found for bigdesk.","url":"/_plugin/bigdesk/","jvm":false,"site":true},{"name":"hq","description":"No description found for hq.","url":"/_plugin/hq/","jvm":false,"site":true},{"name":"head","description":"No description found for head.","url":"/_plugin/head/","jvm":false,"site":true}]}}
Quite human unreadable. Let's try now with jq:
curl -XGET es1.grey:9200/_cluster/stats | jq "."
Seems more elegant now:
{
  "nodes": {
    "plugins": [
      {
        "site": true,
        "jvm": false,
        "url": "/_plugin/bigdesk/",
        "description": "No description found for bigdesk.",
        "name": "bigdesk"
      },
      {
        "site": true,
        "jvm": false,
        "url": "/_plugin/hq/",
        "description": "No description found for hq.",
        "name": "hq"
      },
      {
        "site": true,
        "jvm": false,
        "url": "/_plugin/head/",
        "description": "No description found for head.",
        "name": "head"
      }
    ],
    "fs": {
      "disk_service_time": "0.3",
      "disk_writes": 164465520,
      "disk_reads": 570266,
      "available_in_bytes": 47369240576,
      "available": "44.1gb",
      "free_in_bytes": 52523188224,
      "free": "48.9gb",
      "total_in_bytes": 101461655552,
      "total": "94.4gb",
      "disk_io_op": 165035786,
      "disk_read_size": "16.1gb",
      "disk_read_size_in_bytes": 17348456448,
      "disk_write_size": "612.8gb",
      "disk_write_size_in_bytes": 658024804352,
      "disk_io_size": "628.9gb",
      "disk_io_size_in_bytes": 675373260800,
      "disk_queue": "8.3E-5"
    },
    "jvm": {
      "threads": 621,
      "mem": {
        "heap_max_in_bytes": 137160032256,
        "heap_max": "127.7gb",
        "heap_used_in_bytes": 53713885664,
        "heap_used": "50gb"
      },
      "versions": [
        {
          "count": 8,
          "vm_vendor": "Sun Microsystems Inc.",
          "vm_version": "23.25-b01",
          "vm_name": "OpenJDK 64-Bit Server VM",
          "version": "1.6.0_30"
        }
      ],
      "max_uptime_in_millis": 694176092,
      "max_uptime": "8d"
    },
    "process": {
      "open_file_descriptors": {
        "avg": 1432,
        "max": 1546,
        "min": 1355
      },
      "cpu": {
        "percent": 134
      }
    },
    "os": {
      "cpu": [
        {
          "count": 8,
          "vendor": "AMD",
          "model": "Opteron",
          "mhz": 2599,
          "total_cores": 4,
          "total_sockets": 4,
          "cores_per_socket": 1,
          "cache_size": "512b",
          "cache_size_in_bytes": 512
        }
      ],
      "mem": {
        "total_in_bytes": 202616012800,
        "total": "188.7gb"
      },
      "available_processors": 32
    },
    "versions": [
      "0.90.12"
    ],
    "count": {
      "client": 0,
      "master_data": 8,
      "data_only": 0,
      "master_only": 0,
      "total": 8
    }
  },
  "indices": {
    "segments": {
      "memory_in_bytes": 3169089995,
      "memory": "2.9gb",
      "count": 2023
    },
    "count": 31,
    "shards": {
      "index": {
        "replication": {
          "avg": 1,
          "max": 1,
          "min": 1
        },
        "primaries": {
          "avg": 4.064516129032258,
          "max": 5,
          "min": 2
        },
        "shards": {
          "avg": 8.129032258064516,
          "max": 10,
          "min": 4
        }
      },
      "replication": 1,
      "primaries": 126,
      "total": 252
    },
    "docs": {
      "deleted": 188053,
      "count": 162713334
    },
    "store": {
      "throttle_time_in_millis": 172306131,
      "throttle_time": "1.9d",
      "size_in_bytes": 47552519067,
      "size": "44.2gb"
    },
    "fielddata": {
      "evictions": 0,
      "memory_size_in_bytes": 2858939869,
      "memory_size": "2.6gb"
    },
    "filter_cache": {
      "evictions": 504563,
      "memory_size_in_bytes": 5181806342,
      "memory_size": "4.8gb"
    },
    "id_cache": {
      "memory_size_in_bytes": 0,
      "memory_size": "0b"
    },
    "completion": {
      "size_in_bytes": 0,
      "size": "0b"
    }
  },
  "status": "green",
  "cluster_name": "gc-pb-grey",
  "timestamp": 1418914164326
}

Ok, this does not changes much however, it's easy to get json formatted. But try this: curl -XGET es1.grey:9200/_cluster/stats | jq ".nodes.jvm" and we get:
{
  "threads": 619,
  "mem": {
    "heap_max_in_bytes": 137160032256,
    "heap_max": "127.7gb",
    "heap_used_in_bytes": 54125025648,
    "heap_used": "50.4gb"
  },
  "versions": [
    {
      "count": 8,
      "vm_vendor": "Sun Microsystems Inc.",
      "vm_version": "23.25-b01",
      "vm_name": "OpenJDK 64-Bit Server VM",
      "version": "1.6.0_30"
    }
  ],
  "max_uptime_in_millis": 694408309,
  "max_uptime": "8d"
}
I took the outer object by first dot, 'nodes' property, and from inside it 'jvm' property. It is now trivial to get actual jvm version: curl -XGET es1.grey:9200/_cluster/stats | jq ".nodes.jvm.versions[0].version" Tadam:
"1.6.0_30"