Tags:
create new tag
view all tags

Arrow left Go to previous page / next page of Tier3 site log MOVED TO...

20. 11. 2014 t3fs06 RPC error

Nagios warning

***** Nagios  *****
Notification Type: PROBLEM
Service: NFS registered in RPC
Host: t3fs06
Address: 192.33.123.46
State: CRITICAL
Date/Time: 20-11-2014 06:49:56
Additional Info:
CRITICAL: RPC program nfs  version 3 tcp is not running

RPC check

[root@t3fs13 ~]#  rpcinfo -s t3fs06
rpcinfo: can't contact rpcbind: : RPC: Unable to receive; errno = Connection refused

[root@t3fs13 ~]#  rpcinfo -s t3fs05
   program version(s) netid(s)                         service     owner
    100000  2,3,4     udp,tcp,ticlts,ticotsord,ticots  portmapper  superuser
    100024  1         ticots,ticotsord,ticlts,tcp,udp  status      superuser
    100133  1         ticots,ticotsord,ticlts,tcp,udp  -           superuser
    100099  4         ticotsord                        -           superuser
    100021  4,3,2,1   tcp,udp                          nlockmgr    1
    100234  1         ticotsord                        -           superuser
    100424  1         ticotsord                        -           superuser
    100011  1         udp,ticlts                       rquotad     superuser
    100134  1         ticotsord                        -           superuser
    100155  1         ticotsord                        -           superuser
    100235  1         ticotsord                        -           superuser
    100231  1         ticots,ticotsord,ticlts          -           superuser
    100005  3,2,1     ticots,ticotsord,tcp,ticlts,udp  mountd      superuser
    100003  4,3,2     tcp,udp                          nfs         1
    100227  3,2       tcp,udp                          nfs_acl     1
    100169  1         ticots,ticotsord,ticlts          -           superuser

NFS check

[root@t3fs13 ~]# rpcinfo -T tcp t3fs06 nfs
rpcinfo: RPC: Port mapper failure - Unable to receive: errno 111 (Connection refused)

[root@t3fs13 ~]# rpcinfo -T tcp t3fs05 nfs
program 100003 version 2 ready and waiting
program 100003 version 3 ready and waiting
program 100003 version 4 ready and waiting

Mounting ZFS filesystems: (1/8)cannot mount '/swshare2': directory is not empty

-bash-3.00# svcs  -l  svc:/network/rpc/bind:default
fmri         svc:/network/rpc/bind:default
name         RPC bindings
enabled      true
state        offline
next_state   none
state_time   Thu Nov 20 06:45:21 2014
restarter    svc:/system/svc/restarter:default
dependency   require_all/none svc:/system/filesystem/minimal (online)
dependency   require_all/refresh svc:/system/identity:node (online)
dependency   require_all/none svc:/system/sysidtool:net (offline)
dependency   optional_all/none svc:/network/routing-setup:default (online) svc:/network/initial:default (online)

-bash-3.00# svcs -l sysidtool:net
fmri         svc:/system/sysidtool:net
name         sysidtool
enabled      true
state        offline
next_state   none
state_time   Thu Nov 20 06:45:21 2014
restarter    svc:/system/svc/restarter:default
dependency   optional_all/none svc:/network/pfil (online)
dependency   require_all/none svc:/milestone/single-user:default (online)
dependency   require_all/none svc:/system/filesystem/local:default (maintenance)
dependency   require_all/none svc:/system/filesystem/minimal:default (online)
dependency   require_all/none svc:/system/identity:node (online) svc:/system/identity:domain (online)

-bash-3.00# svcadm clear svc:/system/filesystem/local:default
-bash-3.00# Reading ZFS config: done.
Mounting ZFS filesystems: (1/8)cannot mount '/swshare2': directory is not empty
(8/8)
svc:/system/filesystem/local:default: WARNING: /usr/sbin/zfs mount -a failed: exit status 1
Nov 20 10:27:41 svc.startd[7]: svc:/system/filesystem/local:default: Method "/lib/svc/method/fs-local" failed with exit status 95.
Nov 20 10:27:41 svc.startd[7]: system/filesystem/local:default failed fatally: transitioned to maintenance (see 'svcs -xv' for details)

zfs list ( to check /swshare2 )

-bash-3.00# zfs list
NAME                                          USED  AVAIL  REFER  MOUNTPOINT
rpool                                        9.95G   447G    34K  /rpool
rpool/ROOT                                   6.95G   447G    21K  legacy
rpool/ROOT/s10x_u8wos_08a                    6.95G   447G  5.66G  /
rpool/ROOT/s10x_u8wos_08a@2011-Feb-18_11-51  46.3M      -  3.70G  -
rpool/ROOT/s10x_u8wos_08a@python-20110303    49.7M      -  3.77G  -
rpool/ROOT/s10x_u8wos_08a@31-May-2012         142M      -  4.11G  -
rpool/ROOT/s10x_u8wos_08a@09-Apr-2013         118M      -  5.35G  -
rpool/ROOT/s10x_u8wos_08a@05-Jun-2013         133M      -  5.43G  -
rpool/ROOT/s10x_u8wos_08a@28-Nov-2013         121M      -  5.53G  -
rpool/ROOT/s10x_u8wos_08a@21-03-2014          131M      -  5.55G  -
rpool/ROOT/s10x_u8wos_08a@24-Jun-2014         144M      -  5.62G  -
rpool/ROOT/s10x_u8wos_08a@11-Sep-2014         155M      -  5.64G  -
rpool/dump                                   1.00G   447G  1.00G  -
rpool/export                                   44K   447G    23K  /export
rpool/export/home                              21K   447G    21K  /export/home
rpool/swap                                      2G   449G   227M  -
shome                                        6.15T   701G  6.13T  /shome
shome@auto2014-11-17_00:20:00                4.47G      -  6.11T  -
shome@auto2014-11-18_00:20:00                86.9M      -  6.10T  -
shome@auto2014-11-19_00:20:00                 610M      -  6.11T  -
swshare2                                      598G   762G  37.5K  /swshare2
swshare2/swsharebup                           598G   762G   574G  /swshare2/swsharebup
swshare2/swsharebup@auto2014-11-12_06:00:00  4.30G      -   574G  -
swshare2/swsharebup@auto2014-11-13_06:00:00  5.47M      -   573G  -
swshare2/swsharebup@auto2014-11-14_06:00:00  4.57M      -   573G  -
swshare2/swsharebup@auto2014-11-15_06:00:00  10.4M      -   574G  -
swshare2/swsharebup@auto2014-11-16_06:00:00  23.0M      -   574G  -
swshare2/swsharebup@auto2014-11-17_06:00:00  11.5M      -   573G  -
swshare2/swsharebup@auto2014-11-18_06:00:00  5.45M      -   573G  -
swshare2/swsharebup@auto2014-11-19_06:00:00      0      -   574G  -

-bash-3.00# zfs umount  /swshare2/swsharebup

-bash-3.00# zfs list
NAME                                          USED  AVAIL  REFER  MOUNTPOINT
rpool                                        9.95G   447G    34K  /rpool
rpool/ROOT                                   6.95G   447G    21K  legacy
rpool/ROOT/s10x_u8wos_08a                    6.95G   447G  5.66G  /
rpool/ROOT/s10x_u8wos_08a@2011-Feb-18_11-51  46.3M      -  3.70G  -
rpool/ROOT/s10x_u8wos_08a@python-20110303    49.7M      -  3.77G  -
rpool/ROOT/s10x_u8wos_08a@31-May-2012         142M      -  4.11G  -
rpool/ROOT/s10x_u8wos_08a@09-Apr-2013         118M      -  5.35G  -
rpool/ROOT/s10x_u8wos_08a@05-Jun-2013         133M      -  5.43G  -
rpool/ROOT/s10x_u8wos_08a@28-Nov-2013         121M      -  5.53G  -
rpool/ROOT/s10x_u8wos_08a@21-03-2014          131M      -  5.55G  -
rpool/ROOT/s10x_u8wos_08a@24-Jun-2014         144M      -  5.62G  -
rpool/ROOT/s10x_u8wos_08a@11-Sep-2014         155M      -  5.64G  -
rpool/dump                                   1.00G   447G  1.00G  -
rpool/export                                   44K   447G    23K  /export
rpool/export/home                              21K   447G    21K  /export/home
rpool/swap                                      2G   449G   227M  -
shome                                        6.15T   701G  6.13T  /shome
shome@auto2014-11-17_00:20:00                4.47G      -  6.11T  -
shome@auto2014-11-18_00:20:00                86.9M      -  6.10T  -
shome@auto2014-11-19_00:20:00                 610M      -  6.11T  -
swshare2                                      598G   762G  37.5K  /swshare2
swshare2/swsharebup                           598G   762G   574G  /swshare2/swsharebup 
swshare2/swsharebup@auto2014-11-12_06:00:00  4.30G      -   574G  -
swshare2/swsharebup@auto2014-11-13_06:00:00  5.47M      -   573G  -
swshare2/swsharebup@auto2014-11-14_06:00:00  4.57M      -   573G  -
swshare2/swsharebup@auto2014-11-15_06:00:00  10.4M      -   574G  -
swshare2/swsharebup@auto2014-11-16_06:00:00  23.0M      -   574G  -
swshare2/swsharebup@auto2014-11-17_06:00:00  11.5M      -   573G  -
swshare2/swsharebup@auto2014-11-18_06:00:00  5.45M      -   573G  -
swshare2/swsharebup@auto2014-11-19_06:00:00      0      -   574G  -

-bash-3.00# zfs mount -a
cannot mount '/swshare2': directory is not empty

zfs properties of /swshare2

-bash-3.00# zfs get all swshare2
NAME      PROPERTY              VALUE                  SOURCE
swshare2  type                  filesystem             -
swshare2  creation              Mon Nov  7 14:32 2011  -
swshare2  used                  598G                   -
swshare2  available             762G                   -
swshare2  referenced            37.5K                  -
swshare2  compressratio         1.00x                  -
swshare2  mounted               no                     - 
swshare2  quota                 none                   default
swshare2  reservation           none                   default
swshare2  recordsize            128K                   default
swshare2  mountpoint            /swshare2              default
swshare2  sharenfs              off                    default
swshare2  checksum              on                     default
swshare2  compression           off                    default
swshare2  atime                 on                     default
swshare2  devices               on                     default
swshare2  exec                  on                     default
swshare2  setuid                on                     default
swshare2  readonly              off                    default
swshare2  zoned                 off                    default
swshare2  snapdir               hidden                 default
swshare2  aclmode               groupmask              default
swshare2  aclinherit            restricted             default
swshare2  canmount              on                     default
swshare2  shareiscsi            off                    default
swshare2  xattr                 on                     default
swshare2  copies                1                      default
swshare2  version               4                      -
swshare2  utf8only              off                    -
swshare2  normalization         none                   -
swshare2  casesensitivity       sensitive              -
swshare2  vscan                 off                    default
swshare2  nbmand                off                    default
swshare2  sharesmb              off                    default
swshare2  refquota              none                   default
swshare2  refreservation        none                   default
swshare2  primarycache          all                    default
swshare2  secondarycache        all                    default
swshare2  usedbysnapshots       0                      -
swshare2  usedbydataset         37.5K                  -
swshare2  usedbychildren        598G                   -
swshare2  usedbyrefreservation  0                      -

Solution

A further 'unmount' worked this time.. I can't say why !
-bash-3.00# zfs unmount /swshare2/swsharebup

-bash-3.00# df -h /swshare2/swsharebup/
Filesystem             size   used  avail capacity  Mounted on
rpool/ROOT/s10x_u8wos_08a
                       457G   5.7G   447G     2%    /

# Moving the offending dir
-bash-3.00# mv /swshare2/swsharebup /tmp

-bash-3.00# svcadm clear svc:/system/filesystem/local:default

-bash-3.00# Reading ZFS config: done.
Mounting ZFS filesystems: (8/8)
Nov 20 10:42:02 svc.startd[7]: network/cswsmartd:default failed repeatedly: transitioned to maintenance (see 'svcs -xv' for details)
Nov 20 10:16:21 t3fs06.psi.ch rpcbind: [ID 362760 daemon.alert] no value for config/enable_tcpwrappers (cannot use unset argument). Using default "false"
Nov 20 10:16:21 t3fs06.psi.ch rpcbind: [ID 362760 daemon.alert] no value for config/local_only (cannot use unset argument). Using default "false"
Nov 20 10:16:21 t3fs06.psi.ch rpcbind: [ID 362760 daemon.alert] no value for config/allow_indirect (cannot use unset argument). Using default "true"
Nov 20 10:16:21 t3fs06.psi.ch rpcbind: [ID 362760 daemon.alert] no value for config/verbose_logging (cannot use unset argument). Using default "false"
Nov 20 10:42:02 t3fs06.psi.ch svc.startd[7]: network/cswsmartd:default failed repeatedly: transitioned to maintenance (see 'svcs -xv' for details)
Nov 20 10:42:02 t3fs06.psi.ch asy: asy1: UART @ 2f8 scratch register: expected 0x5a, got 0xff
Nov 20 10:42:02 t3fs06.psi.ch asy: Cannot identify UART chip at 2f8

Arrow left Go to previous page / next page of Tier3 site log MOVED TO...

Edit | Attach | Watch | Print version | History: r2 < r1 | Backlinks | Raw View | Raw edit | More topic actions
Topic revision: r2 - 2014-11-20 - FabioMartinelli
 
This site is powered by the TWiki collaboration platform Powered by Perl This site is powered by the TWiki collaboration platformCopyright © 2008-2020 by the contributing authors. All material on this collaboration platform is the property of the contributing authors.
Ideas, requests, problems regarding TWiki? Send feedback