diff --git a/README.md b/README.md index e2e9d62..9f7ab78 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ mctop depends on the [ruby-pcap](https://rubygems.org/gems/ruby-pcap) gem, if yo this installed you'll need to ensure you have the development pcap libraries (libpcap-devel package on most linux distros) to build the native gem. -![](http://etsycodeascraft.files.wordpress.com/2012/12/mctop.jpg) +![](https://raw.github.com/opepin/mctop/master/screen.png) ## How it works @@ -20,9 +20,14 @@ mctop sniffs network traffic collecting memcache `VALUE` responses and calculate traffic statistics for each key seen. It currently reports on the following metrics per key: * **calls** - the number of times the key has been called since mctop started +* **gets** - the number of times on get on that key since mctop started +* **hits** - hit rate on get requests since mctop started +* **sets** - the number of times on set on that key since mctop started +* **deletes** - the number of times on delete on that key since mctop started * **objsize** - the size of the object stored for that key * **req/sec** - the number of requests per second for the key * **bw (kbps)** - the estimated network bandwidth consumed by this key in kilobits-per-second +* **lifetime** - expiration time of the key at the time it was set ## Getting it running @@ -49,7 +54,12 @@ the quickest way to get it running is to: The following key commands are available in the console UI: * `C` - sort by number of calls -* `S` - sort by object size +* `G` - sort by number of gets +* `L` - sort by lifetime +* `H` - sort by number get hit rate +* `S` - sort by number of sets +* `D` - sort by number of delete +* `O` - sort by object size * `R` - sort by requests/sec * `B` - sort by bandwidth * `T` - toggle sorting by ascending / descending order @@ -73,5 +83,7 @@ The following details are displayed in the status bar ### ruby-pcap drops packets at high volume from my testing the ruby-pcap native interface to libpcap struggles to keep up with high packet rates (in what we see on a production memcache instance) you can keep an eye on the packets recv/drop and loss percentage on the status bar at the bottom of the UI to get an idea of the packet +### does not support the full memcache protocol (multi gets, incr, ...) + ### No binary protocol support There is currently no support for the binary protocol. However, if someone is using it and would like to submit a patch, it would be welcome. diff --git a/bin/mctop b/bin/mctop index 20da5f8..f091bbe 100755 --- a/bin/mctop +++ b/bin/mctop @@ -46,7 +46,17 @@ until done do done = true when /[Cc]/ sort_mode = :calls + when /[Dd]/ + sort_mode = :deletes + when /[Gg]/ + sort_mode = :gets + when /[Ll]/ + sort_mode = :lifetime when /[Ss]/ + sort_mode = :sets + when /[Hh]/ + sort_mode = :hits + when /[Oo]/ sort_mode = :objsize when /[Rr]/ sort_mode = :reqsec diff --git a/lib/sniffer.rb b/lib/sniffer.rb index f75fee2..176a15c 100644 --- a/lib/sniffer.rb +++ b/lib/sniffer.rb @@ -1,21 +1,40 @@ require 'pcap' require 'thread' +#require 'logger' class MemcacheSniffer - attr_accessor :metrics, :semaphore + attr_accessor :metrics, :semaphore, :log def initialize(config) @source = config[:nic] @port = config[:port] @metrics = {} + @metrics[:deletes] = {} + @metrics[:sets] = {} + @metrics[:lifetime] = {} + @metrics[:gets] = {} + @metrics[:hits] = {} @metrics[:calls] = {} @metrics[:objsize] = {} @metrics[:reqsec] = {} - @metrics[:bw] = {} + @metrics[:bw] = {} @metrics[:stats] = { :recv => 0, :drop => 0 } @semaphore = Mutex.new + #@log = Logger.new('/tmp/logfile.log') + end + + def command(key) + if ! @metrics[:calls].has_key?(key) + @metrics[:calls][key] = 0 + @metrics[:sets][key] = 0 + @metrics[:lifetime][key] = -1 + @metrics[:gets][key] = 0 + @metrics[:deletes][key] = 0 + @metrics[:hits][key] = 0 + @metrics[:objsize][key] = -1 + end end def start @@ -29,22 +48,61 @@ def start cap.loop do |packet| @metrics[:stats] = cap.stats - # parse key name, and size from VALUE responses + # hit on a get parse key name, and size from VALUE responses + if packet.raw_data =~ /STAT / + next + end + + # hit on a get parse key name, and size from VALUE responses if packet.raw_data =~ /VALUE (\S+) \S+ (\S+)/ - key = $1 - bytes = $2 + key = $1 + bytes = $2 + @semaphore.synchronize do + self.command(key) + @metrics[:hits][key] += 1 + @metrics[:objsize][key] = bytes.to_i + end + end + # parse key name + # gets ? + if packet.raw_data =~ /get (\S+)\r\n/ + key = $1 @semaphore.synchronize do - if @metrics[:calls].has_key?(key) - @metrics[:calls][key] += 1 - else - @metrics[:calls][key] = 1 - end + self.command(key) + @metrics[:gets][key] += 1 + @metrics[:calls][key] += 1 + end + end + - @metrics[:objsize][key] = bytes.to_i + # parse key/ name and size + if packet.raw_data =~ /set (\S+) (\S+) (\S+) (\S+)\r\n/ + key = $1 + ttl = $3; + bytes = $4; + @semaphore.synchronize do + self.command(key) + @metrics[:sets][key] += 1 + # @log.warn(packet.raw_data) + @metrics[:calls][key] += 1 + @metrics[:objsize][key] = bytes.to_i; + @metrics[:lifetime][key] = ttl.to_i; end end + + # parse key name + # delete + if packet.raw_data =~ /delete (\S+)/ + key = $1 + @semaphore.synchronize do + self.command(key) + @metrics[:deletes][key] += 1 + @metrics[:calls][key] += 1 + end + end + break if @done end diff --git a/lib/ui.rb b/lib/ui.rb index 3ca7420..2f55e5f 100644 --- a/lib/ui.rb +++ b/lib/ui.rb @@ -20,18 +20,23 @@ def initialize(config) init_pair(2, COLOR_WHITE, COLOR_RED) end - @stat_cols = %w[ calls objsize req/sec bw(kbps) ] - @stat_col_width = 10 + @stat_cols = %w[ deletes sets gets hits calls objsize req/sec bw(kbps) lifetime ] + @stat_col_width = 10 @key_col_width = 0 @commands = { 'Q' => "quit", + 'D' => "sort by deletes", + 'S' => "sort by sets", + 'L' => "sort by lifetime", + 'G' => "sort by gets", + 'H' => "sort by hits", 'C' => "sort by calls", - 'S' => "sort by size", + 'O' => "sort by size", 'R' => "sort by req/sec", 'B' => "sort by bandwidth", 'T' => "toggle sort order (asc|desc)" - } + } end def header @@ -39,7 +44,7 @@ def header @stat_cols = @stat_cols.map { |c| sprintf("%#{@stat_col_width}s", c) } # key column width is whatever is left over - @key_col_width = cols - (@stat_cols.length * @stat_col_width) + @key_col_width = cols - (@stat_cols.length * @stat_col_width) - 3 attrset(color_pair(1)) setpos(0,0) @@ -70,10 +75,10 @@ def render_stats(sniffer, sort_mode, sort_order = :desc) # construct and render footer stats line setpos(lines-2,0) attrset(color_pair(2)) - header_summary = sprintf "%-28s %-14s %-30s", + header_summary = sprintf "%-28s %-14s %-30s", "sort mode: #{sort_mode.to_s} (#{sort_order.to_s})", "keys: #{sniffer.metrics[:calls].keys.count}", - "packets (recv/dropped): #{sniffer.metrics[:stats][:recv]} / #{sniffer.metrics[:stats][:drop]} (#{loss}%)" + "packets (recv/dropped): #{sniffer.metrics[:stats][:recv]} / #{sniffer.metrics[:stats][:drop]} (#{loss}%)" addstr(sprintf "%-#{cols}s", header_summary) # reset colours for main key display @@ -81,57 +86,66 @@ def render_stats(sniffer, sort_mode, sort_order = :desc) top = [] - sniffer.semaphore.synchronize do - # we may have seen no packets received on the sniffer thread + sniffer.semaphore.synchronize do + # we may have seen no packets received on the sniffer thread return if sniffer.metrics[:start_time].nil? elapsed = Time.now.to_f - sniffer.metrics[:start_time] # iterate over all the keys in the metrics hash and calculate some values sniffer.metrics[:calls].each do |k,v| - reqsec = v / elapsed - - # if req/sec is <= the discard threshold delete those keys from - # the metrics hash - this is a hack to manage the size of the - # metrics hash in high volume environments - if reqsec <= @config[:discard_thresh] + reqsec = v / elapsed + + # if req/sec is <= the discard threshold delete those keys from + # the metrics hash - this is a hack to manage the size of the + # metrics hash in high volume environments + if reqsec <= @config[:discard_thresh] + sniffer.metrics[:sets].delete(k) + sniffer.metrics[:deletes].delete(k) + sniffer.metrics[:gets].delete(k) + sniffer.metrics[:hits].delete(k) sniffer.metrics[:calls].delete(k) sniffer.metrics[:objsize].delete(k) sniffer.metrics[:reqsec].delete(k) sniffer.metrics[:bw].delete(k) - else - sniffer.metrics[:reqsec][k] = v / elapsed - sniffer.metrics[:bw][k] = ((sniffer.metrics[:objsize][k] * sniffer.metrics[:reqsec][k]) * 8) / 1000 - end - end - - top = sniffer.metrics[sort_mode].sort { |a,b| a[1] <=> b[1] } - end - - unless sort_order == :asc - top.reverse! - end - - for i in 0..maxlines-1 - if i < top.length - k = top[i][0] - v = top[i][1] - - # if the key is too wide for the column truncate it and add an ellipsis - if k.length > @key_col_width - display_key = k[0..@key_col_width-4] - display_key = "#{display_key}..." + sniffer.metrics[:lifetime].delete(k) else - display_key = k + sniffer.metrics[:reqsec][k] = v / elapsed + sniffer.metrics[:bw][k] = ((sniffer.metrics[:objsize][k] * sniffer.metrics[:reqsec][k]) * 8) / 1000 end + end + top = sniffer.metrics[sort_mode].sort { |a,b| a[1] <=> b[1] } + end - # render each key - line = sprintf "%-#{@key_col_width}s %9.d %9.d %9.2f %9.2f", - display_key, - sniffer.metrics[:calls][k], - sniffer.metrics[:objsize][k], - sniffer.metrics[:reqsec][k], - sniffer.metrics[:bw][k] + unless sort_order == :asc + top.reverse! + end + + for i in 0..maxlines-1 + if i < top.length + k = top[i][0] + v = top[i][1] + + # if the key is too wide for the column truncate it and add an ellipsis + if k.length > @key_col_width + display_key = k[0..@key_col_width-4] + display_key = "#{display_key}..." + else + display_key = k + end + + # render each key + line = sprintf "%-#{@key_col_width}s %9.d %9.d %9.d %9.2f %% %9.d %9.d %9.2f %9.2f %9.d", + display_key, + sniffer.metrics[:deletes][k], + sniffer.metrics[:sets][k], + sniffer.metrics[:gets][k], + sniffer.metrics[:gets][k] > 0 ? (sniffer.metrics[:hits][k] * 100 / sniffer.metrics[:gets][k]) : 0, + sniffer.metrics[:calls][k], + sniffer.metrics[:objsize][k], + sniffer.metrics[:reqsec][k], + sniffer.metrics[:bw][k], + sniffer.metrics[:lifetime][k] else # we're not clearing the display between renders so erase past # keys with blank lines if there's < maxlines of results @@ -147,6 +161,7 @@ def render_stats(sniffer, sort_mode, sort_order = :desc) attrset(color_pair(2)) setpos(lines-2, cols-18) addstr(sprintf "rt: %8.3f (ms)", runtime) + end def input_handler diff --git a/screen.png b/screen.png new file mode 100644 index 0000000..15ae37d Binary files /dev/null and b/screen.png differ