# /opt/zfssnap/zfssnap $PERIOD $MAXSNAPS $BACKUPSERVER >>$LOG 2>&1
)
/etc/security/access_users.conf
on the user interfaces by commenting out the lines that allow access for all CMS users #+ : cms : ALL - : ALL : ALL
/opt/icinga-config/tier3/objects/tier3_templates.cfg
comment the line with members like define contactgroup{ contactgroup_name t3-admins alias TIER3 Administrators # members ......................
ssh t3slurm "scontrol update PartitionName=gpu State=DRAIN;scontrol update PartitionName=wn State=DRAIN; scontrol update PartitionName=qgpu State=DRAIN;scontrol update PartitionName=quick State=DRAIN "
pssh -h node-list-t3/slurm-clients -P "umount /pnfs/psi.ch/cms"
for n in $(seq 10 59); do echo t3wn$n; ssh t3wn$n "sed -i 's/\(t3dcachedb03:\/pnfs\/psi.ch\/cms.*\)/# COMMENTED FOR DOWNTIME \1/' /etc/fstab"; done
for n in $(seq 10 59) ; do echo t3wn$n; ssh !root@t3wn$n shutdown -h now ; sleep 1 ; done
for n in $(seq 10 20) 22 23 $(seq 25 47) $(seq 49 59) ; do node="t3wn$n"; echo -n "$node: "; ipmitool -I lanplus -H rmwn$n -U root -f /root/private/.ipmi-pw chassis power status ; done
ssh phedex@t3cmsvobox /home/phedex/config/T3_CH_PSI/PhEDEx/tools/init.d/phedex_Debug stop
ssh t3se01 systemctl stop xrootd@clustered
ssh t3se01 dcache stop dcap-t3se01Domainand stop xrootd door on t3dcachedb03
[root@t3admin02 ~]# pssh -h node-list-t3/dcache-pools -P "dcache stop"
ssh t3se01 umount /pnfs
; ssh t3dcachedb03 umount /pnfs
ssh t3se01 dcache stop
ssh t3dcachedb03 dcache stop
ssh t3dcachedb03 systemctl status/stop postgresql-11
for n in $(seq 1 3); do ssh t3zkpr1$n systemctl stop zookeeper.service; done
ssh root@t3bdii "/etc/init.d/bdii stop"
systemctl status zookeeper
and zkcli -server t3zkpr11
on t3zkpr11
systemctl start postgresql-11
and systemctl status crond
and dcache check-config
. Start all dcache main services dcache start *Domain
besides doors (currently only one xrootd door configured on t3dcachedb03)
dcache status
(like dcache start info-t3se01Domain
currently the same for dcache-t3se01Domain, pinmanager-t3se01Domain, spacemanager-t3se01Domain, transfermanagers-t3se01Domain should be started ); doors (configured at the moment also on t3se01) should be stared after pools
mount /pnfs
on t3se01 and t3dcachedb03
[root@t3admin02 ~]# pssh -h node-list-t3/fs-dalco -P "dcache start"
takes about 15-30' (in a case of hardware issue first switch on JBOD and than server)
[root@t3nfs11 ~]# multipath -ll
dcache status
) like dcache start dcap-t3se01Domain
, etc. and xrootd door on t3dcachedb03
systemctl start cmsd@clustered
; systemctl start xrootd@clustered
pssh -h node-list-t3/slurm-clients -P "mount |grep pnfs"
scontrol update PartitionName=gpu State=UP
and scontrol update PartitionName=wn State=UP
, etc. for all Partitions