Retrieving locking information

gfs_tool lockdump :

    gfs_tool lockdump /cluster/pool/cache_stage > /tmp/lockdump

Analysing raw lockdump

Example

flock on inode 958445:

      Glock (6, 958445)
      gl_flags =
      gl_count = 3
      gl_state = 1
      req_gh = no
      req_bh = no
      lvb_count = 0
      object = no
      new_le = no
      incore_le = no
      reclaim = no
      aspace = no
      ail_bufs = no
      Holder
        owner = 7864
        gh_state = 1
        gh_flags = 5 7 10
        error = 0
        gh_iflags = 1 6 7
      Waiter2
        owner = -1
        gh_state = 0
        gh_flags = 0
        error = 0
        gh_iflags = 2 4 5

Glock (ln_type, ln_number)

ln_type: type of lock

gl_flags =

gl_count = [Usage count]

gl_state = [state]

req_gh = [yes|no]

Holder for request being serviced

req_bh = [yes|no]

The bottom half to execute

lvb_count = count

LVB recursive usage (hold/unhold) count

object = [yes|no]

void gl_object; / The protected entity (e.g. a dinode) */

new_le = [yes|no]

gfs_printf(" new_le = %s\n", (gl->gl_new_le.le_trans) ? "yes" : "no");

struct gfs_log_element gl_new_le; / New, incomplete transaction /

incore_le = [yes|no]

gfs_printf(" incore_le = %s\n", (gl->gl_incore_le.le_trans) ? "yes" : "no");

struct gfs_log_element gl_incore_le; / Complete (committed) trans /

reclaim = [yes|no]

gfs_printf(" reclaim = %s\n",(list_empty(&gl-;>gl_reclaim)) ? "no" : "yes");

struct list_head gl_reclaim; / Link to sd_reclaim_list /

aspace =

gfs_printf(" aspace = %lu\n", gl->gl_aspace->i_mapping->nrpages);

struct inode gl_aspace; / The buffers protected by this lock */

ail_bufs = [yes|no]

gfs_printf(" ail_bufs = %s\n", (list_empty(&gl-;>gl_ail_bufs)) ? "no" : "yes");

struct list_head gl_ail_bufs; / AIL buffers protected by us /

Holder|Waiter|Waiter2|Waiter3

owner = [pid|-1]

pid of the local process holding the lock

gh_state = 1

unsigned int gh_state; / LMST... requested lock state /

gh_flags = 5 7 10

error = 0

int gh_error; / GLR_... CANCELLED/TRYFAILED/-errno /

gh_iflags = 1 6 7

Inode

num = 701919/701919

/ Formal inode # and block address / gfs_printf(" num = %" PRIu64 "/%" PRIu64 "\n", ip->i_num.no_formal_ino, ip->i_num.no_addr);

type =

gfs_printf(" type = %u\n", ip->i_di.di_type);

i_count =

/ Usage count / gfs_printf(" i_count = %d\n", atomic_read(&ip-;>i_count));

i_flags =

&ip-;>i_flags

vnode = [yes|no]

/ Linux VFS inode structure / gfs_printf(" vnode = %s\n", (ip->i_vnode) ? "yes" : "no");

Tools for analysis

decipher_lockstate_dump and parse_lockdump are part of the gfs_tool cvs source

decipher_lockstate_dump lockdumpfile1 lockdumpfile2 ...

example:

        node1# gfs_tool lockdump /cluster/pool/cache_stage > /tmp/lockdump.cache_stage.stage1
        node1# decipher_lockstate_dump /tmp/lockdump.cache_stage.stage1 > /tmp/lockdump.cache_stage.stage1.dec

example for flock on inode 958445:

        Glock (flock[6], 958445)
          gl_flags =
          gl_count = 3
          gl_state = exclusive[1]
          req_gh = no
          req_bh = no
          lvb_count = 0
          object = no
          new_le = no
          incore_le = no
          reclaim = no
          aspace = no
          ail_bufs = no
          Holder
            owner = 7864
            gh_state = exclusive[1]
            gh_flags = local_excl[5] exact[7] nocache[10]
            error = 0
            gh_iflags = promote[1] holder[6] first[7]
          Waiter2
            owner = none[-1]
            gh_state = unlocked[0]
            gh_flags = try[0]
            error = 0
            gh_iflags = demote[2] alloced[4] dealloc[5]

parse_lockdump deciphered_dumpfile1 ...

example for flock on inode 958445:

        2 chain:
        /tmp/lockdump.cache_stage.stage1.dec Glock (flock[6], 958445)
          gl_flags =
          gl_count = 3
          gl_state = exclusive[1]
          req_gh = no
          req_bh = no
          lvb_count = 0
          object = no
          new_le = no
          incore_le = no
          reclaim = no
          aspace = no
          ail_bufs = no
          Holder
            owner = 7864
            gh_state = exclusive[1]
            gh_flags = local_excl[5] exact[7] nocache[10]
            error = 0
            gh_iflags = promote[1] holder[6] first[7]        
          Waiter2
            owner = none[-1]
            gh_state = unlocked[0]
            gh_flags = try[0]
            error = 0
            gh_iflags = demote[2] alloced[4] dealloc[5]
        /tmp/lockdump.cache_stage.stage2.dec Glock (flock[6], 958445)
          gl_flags = lock[1]
          gl_count = 2
          gl_state = unlocked[0]
          req_gh = yes
          req_bh = yes
          lvb_count = 0
          object = no
          new_le = no
          incore_le = no
          reclaim = no
          aspace = no
          ail_bufs = no
          Request
            owner = 23122
            gh_state = exclusive[1]
            gh_flags = local_excl[5] exact[7] nocache[10]
            error = 0
            gh_iflags = promote[1]
          Waiter3
            owner = 23122
            gh_state = exclusive[1]
            gh_flags = local_excl[5] exact[7] nocache[10]
            error = 0
            gh_iflags = promote[1]
        1 requests