RAC start stop maintenance

[root@node1 ~]# /data01/app/11204/grid_11204/bin/crsctl status server -f
NAME=node1
STATE=ONLINE
ACTIVE_POOLS=Generic ora.racdb
STATE_DETAILS=

NAME=node2
STATE=ONLINE
ACTIVE_POOLS=Generic ora.racdb
STATE_DETAILS=

root@node1 ~]# /data01/app/11204/grid_11204/bin/cemutlo -n
node-cluster
[root@node1 ~]#

[oracle@node1 ~]$ srvctl status nodeapps
VIP node1-vip is enabled
VIP node1-vip is running on node: node1
VIP node2-vip is enabled
VIP node2-vip is running on node: node2
Network is enabled
Network is running on node: node1
Network is running on node: node2
GSD is disabled
GSD is not running on node: node1
GSD is not running on node: node2
ONS is enabled
ONS daemon is running on node: node1
ONS daemon is running on node: node2

[oracle@node1 ~]$ srvctl status scan
SCAN VIP scan1 is enabled
SCAN VIP scan1 is running on node node2
SCAN VIP scan2 is enabled
SCAN VIP scan2 is running on node node1
SCAN VIP scan3 is enabled
SCAN VIP scan3 is running on node node1
[oracle@node1 ~]$ srvctl status scan_listener
SCAN Listener LISTENER_SCAN1 is enabled
SCAN listener LISTENER_SCAN1 is running on node node2
SCAN Listener LISTENER_SCAN2 is enabled
SCAN listener LISTENER_SCAN2 is running on node node1
SCAN Listener LISTENER_SCAN3 is enabled
SCAN listener LISTENER_SCAN3 is running on node node1
[oracle@node1 ~]$

How to check the status of CRS on specific node:
===================================================
[grid@node1 ~]$ crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online

[grid@node1 ~]$ crsctl check cluster
===========================================
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[grid@node1 ~]$ crsctl check cluster -all
**************************************************************
node1:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
node2:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************

[oracle@node1 ~]$ srvctl status database -d racdb -v
Instance racdb1 is running on node node1. Instance status: Open.
Instance racdb2 is running on node node2. Instance status: Open.

How to check the status of complete clusterware stack on all nodes:
=======================================================================

[root@node1 ~]# /data01/app/11204/grid_11204/bin/crsctl status res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS      
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.DATA_NEW.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.FRA.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.LISTENER.lsnr
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCRNEW.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCRVOTE.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCR_VOTE.dg
               ONLINE  OFFLINE      node1                                      
               ONLINE  OFFLINE      node2                                      
ora.asm
               ONLINE  ONLINE       node1                    Started            
               ONLINE  ONLINE       node2                    Started            
ora.gsd
               OFFLINE OFFLINE      node1                                      
               OFFLINE OFFLINE      node2                                      
ora.net1.network
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.ons
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.registry.acfs
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       node2                                      
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       node1                                      
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       node1                                      
ora.cvu
      1        ONLINE  ONLINE       node2                                      
ora.node1.vip
      1        ONLINE  ONLINE       node1                                      
ora.node2.vip
      1        ONLINE  ONLINE       node2                                      
ora.oc4j
      1        ONLINE  OFFLINE                                                  
ora.racdb.db
      1        ONLINE  ONLINE       node1                    Open              
      2        ONLINE  ONLINE       node2                    Open              
ora.scan1.vip
      1        ONLINE  ONLINE       node2                                      
ora.scan2.vip
      1        ONLINE  ONLINE       node1                                      
ora.scan3.vip
      1        ONLINE  ONLINE       node1                            


[oracle@node1 ~]$ ps -ef | grep d.bin
root      2579     1  1 02:07 ?        00:00:19 /data01/app/11204/grid_11204/bin/ohasd.bin reboot
grid      2857     1  0 02:07 ?        00:00:00 /data01/app/11204/grid_11204/bin/mdnsd.bin
grid      2868     1  0 02:07 ?        00:00:02 /data01/app/11204/grid_11204/bin/gpnpd.bin
grid      2878     1  0 02:07 ?        00:00:14 /data01/app/11204/grid_11204/bin/gipcd.bin
root      2892     1  2 02:07 ?        00:00:44 /data01/app/11204/grid_11204/bin/osysmond.bin
grid      2948     1  0 02:07 ?        00:00:14 /data01/app/11204/grid_11204/bin/ocssd.bin
root      3145     1  0 02:08 ?        00:00:07 /data01/app/11204/grid_11204/bin/octssd.bin reboot
grid      3177     1  0 02:08 ?        00:00:05 /data01/app/11204/grid_11204/bin/evmd.bin
root      4426     1  1 02:14 ?        00:00:13 /data01/app/11204/grid_11204/bin/crsd.bin reboot
oracle    9034  8298  0 02:37 pts/2    00:00:00 grep d.bin
[oracle@node1 ~]$

How to stop crs on specific node:
===================================
[oracle@node1 ~]$ srvctl stop instance -d racdb -i racdb1

[oracle@node1 ~]$ srvctl status  database -d racdb -v
Instance racdb1 is not running on node node1
Instance racdb2 is running on node node2. Instance status: Open.

[root@node1 ~]# /data01/app/11204/grid_11204/bin/crsctl stop crs
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'node1'
CRS-2673: Attempting to stop 'ora.crsd' on 'node1'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'node1'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN3.lsnr' on 'node1'
CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'node1'
CRS-2673: Attempting to stop 'ora.FRA.dg' on 'node1'
CRS-2673: Attempting to stop 'ora.OCRNEW.dg' on 'node1'
CRS-2673: Attempting to stop 'ora.OCRVOTE.dg' on 'node1'
CRS-2673: Attempting to stop 'ora.registry.acfs' on 'node1'
CRS-2673: Attempting to stop 'ora.DATA_NEW.dg' on 'node1'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN2.lsnr' on 'node1'
CRS-2677: Stop of 'ora.LISTENER_SCAN2.lsnr' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.scan2.vip' on 'node1'
CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.node1.vip' on 'node1'
CRS-2677: Stop of 'ora.LISTENER_SCAN3.lsnr' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.scan3.vip' on 'node1'
CRS-2677: Stop of 'ora.node1.vip' on 'node1' succeeded
CRS-2672: Attempting to start 'ora.node1.vip' on 'node2'
CRS-2677: Stop of 'ora.scan2.vip' on 'node1' succeeded
CRS-2672: Attempting to start 'ora.scan2.vip' on 'node2'
CRS-2677: Stop of 'ora.scan3.vip' on 'node1' succeeded
CRS-2672: Attempting to start 'ora.scan3.vip' on 'node2'
CRS-2676: Start of 'ora.node1.vip' on 'node2' succeeded
CRS-2676: Start of 'ora.scan2.vip' on 'node2' succeeded
CRS-2676: Start of 'ora.scan3.vip' on 'node2' succeeded
CRS-2672: Attempting to start 'ora.LISTENER_SCAN2.lsnr' on 'node2'
CRS-2672: Attempting to start 'ora.LISTENER_SCAN3.lsnr' on 'node2'
CRS-2677: Stop of 'ora.registry.acfs' on 'node1' succeeded
CRS-2677: Stop of 'ora.OCRNEW.dg' on 'node1' succeeded
CRS-2677: Stop of 'ora.DATA_NEW.dg' on 'node1' succeeded
CRS-2677: Stop of 'ora.FRA.dg' on 'node1' succeeded
CRS-2676: Start of 'ora.LISTENER_SCAN2.lsnr' on 'node2' succeeded
CRS-2676: Start of 'ora.LISTENER_SCAN3.lsnr' on 'node2' succeeded
CRS-2677: Stop of 'ora.OCRVOTE.dg' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'node1'
CRS-2677: Stop of 'ora.asm' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.ons' on 'node1'
CRS-2677: Stop of 'ora.ons' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.net1.network' on 'node1'
CRS-2677: Stop of 'ora.net1.network' on 'node1' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'node1' has completed
CRS-2677: Stop of 'ora.crsd' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.crf' on 'node1'
CRS-2673: Attempting to stop 'ora.ctssd' on 'node1'
CRS-2673: Attempting to stop 'ora.evmd' on 'node1'
CRS-2673: Attempting to stop 'ora.asm' on 'node1'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'node1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'node1'
CRS-2677: Stop of 'ora.evmd' on 'node1' succeeded
CRS-2677: Stop of 'ora.crf' on 'node1' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'node1' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'node1' succeeded
CRS-2677: Stop of 'ora.drivers.acfs' on 'node1' succeeded
CRS-2677: Stop of 'ora.asm' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'node1'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'node1'
CRS-2677: Stop of 'ora.cssd' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'node1'
CRS-2677: Stop of 'ora.gipcd' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.gpnpd' on 'node1'
CRS-2677: Stop of 'ora.gpnpd' on 'node1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'node1' has completed
CRS-4133: Oracle High Availability Services has been stopped.

[root@node2 ~]# /data01/app/11204/grid_11204/bin/crsctl status res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS      
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.DATA_NEW.dg
               ONLINE  ONLINE       node2                                      
ora.FRA.dg
               ONLINE  ONLINE       node2                                      
ora.LISTENER.lsnr
               ONLINE  ONLINE       node2                                      
ora.OCRNEW.dg
               ONLINE  ONLINE       node2                                      
ora.OCRVOTE.dg
               ONLINE  ONLINE       node2                                      
ora.OCR_VOTE.dg
               ONLINE  OFFLINE      node2                                      
ora.asm
               ONLINE  ONLINE       node2                    Started            
ora.gsd
               OFFLINE OFFLINE      node2                                      
ora.net1.network
               ONLINE  ONLINE       node2                                      
ora.ons
               ONLINE  ONLINE       node2                                      
ora.registry.acfs
               ONLINE  ONLINE       node2                                      
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       node2                                      
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       node2                                      
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       node2                                      
ora.cvu
      1        ONLINE  ONLINE       node2                                      
ora.node1.vip
      1        ONLINE  INTERMEDIATE node2                    FAILED OVER        
ora.node2.vip
      1        ONLINE  ONLINE       node2                                      
ora.oc4j
      1        ONLINE  ONLINE       node2                                      
ora.racdb.db
      1        OFFLINE OFFLINE                               Instance Shutdown  
      2        ONLINE  ONLINE       node2                    Open              
ora.scan1.vip
      1        ONLINE  ONLINE       node2                                      
ora.scan2.vip
      1        ONLINE  ONLINE       node2                                      
ora.scan3.vip
      1        ONLINE  ONLINE       node2                                

[oracle@node2 ~]$ srvctl status database -d racdb -v
Instance racdb1 is not running on node node1
Instance racdb2 is running on node node2. Instance status: Open.

[oracle@node2 ~]$ srvctl status scan
SCAN VIP scan1 is enabled
SCAN VIP scan1 is running on node node2
SCAN VIP scan2 is enabled
SCAN VIP scan2 is running on node node2
SCAN VIP scan3 is enabled
SCAN VIP scan3 is running on node node2

[oracle@node2 ~]$ srvctl status scan_listener
SCAN Listener LISTENER_SCAN1 is enabled
SCAN listener LISTENER_SCAN1 is running on node node2
SCAN Listener LISTENER_SCAN2 is enabled
SCAN listener LISTENER_SCAN2 is running on node node2
SCAN Listener LISTENER_SCAN3 is enabled
SCAN listener LISTENER_SCAN3 is running on node node2


NODE2:
=======
[root@node1 ~]# /data01/app/11204/grid_11204/bin/crsctl start crs
CRS-4123: Oracle High Availability Services has been started.

[root@node1 ~]# ps -ef | grep d.bin
root     14259     1  1 03:11 ?        00:00:02 /data01/app/11204/grid_11204/bin/ohasd.bin reboot
grid     14430     1  0 03:11 ?        00:00:00 /data01/app/11204/grid_11204/bin/mdnsd.bin
grid     14441     1  0 03:11 ?        00:00:00 /data01/app/11204/grid_11204/bin/gpnpd.bin
grid     14452     1  1 03:11 ?        00:00:01 /data01/app/11204/grid_11204/bin/gipcd.bin
root     14466     1  1 03:11 ?        00:00:01 /data01/app/11204/grid_11204/bin/osysmond.bin
grid     14511     1  1 03:11 ?        00:00:01 /data01/app/11204/grid_11204/bin/ocssd.bin
root     14627     1  0 03:11 ?        00:00:00 /data01/app/11204/grid_11204/bin/octssd.bin reboot
grid     14647     1  0 03:11 ?        00:00:00 /data01/app/11204/grid_11204/bin/evmd.bin
root     14911     1  2 03:12 ?        00:00:00 /data01/app/11204/grid_11204/bin/crsd.bin reboot
root     15215 10009  0 03:13 pts/2    00:00:00 grep d.bin

[oracle@node1 ~]$ srvctl start instance -d racdb -i racdb1
[oracle@node1 ~]$ srvctl status database -d racdb -v
Instance racdb1 is running on node node1. Instance status: Open.
Instance racdb2 is running on node node2. Instance status: Open.
[oracle@node1 ~]$

[oracle@node1 ~]$ /data01/app/11204/grid_11204/bin/crsctl status res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS      
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.DATA_NEW.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.FRA.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.LISTENER.lsnr
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCRNEW.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCRVOTE.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCR_VOTE.dg
               ONLINE  OFFLINE      node1                                      
               ONLINE  OFFLINE      node2                                      
ora.asm
               ONLINE  ONLINE       node1                    Started            
               ONLINE  ONLINE       node2                    Started            
ora.gsd
               OFFLINE OFFLINE      node1                                      
               OFFLINE OFFLINE      node2                                      
ora.net1.network
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.ons
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.registry.acfs
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       node1                                      
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       node2                                      
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       node2                                      
ora.cvu
      1        ONLINE  ONLINE       node2                                      
ora.node1.vip
      1        ONLINE  ONLINE       node1                                      
ora.node2.vip
      1        ONLINE  ONLINE       node2                                      
ora.oc4j
      1        ONLINE  ONLINE       node2                                      
ora.racdb.db
      1        ONLINE  ONLINE       node1                    Open              
      2        ONLINE  ONLINE       node2                    Open              
ora.scan1.vip
      1        ONLINE  ONLINE       node1                                      
ora.scan2.vip
      1        ONLINE  ONLINE       node2                                      
ora.scan3.vip
      1        ONLINE  ONLINE       node2                                      
[oracle@node1 ~]$

==========================================================================
Follow same process on node2:

[oracle@node2 ~]$ srvctl stop instance -d racdb -i racdb2
[root@node2 ~]# /data01/app/11204/grid_11204/bin/crsctl stop crs

[root@node2 ~]# /data01/app/11204/grid_11204/bin/crsctl start crs
[oracle@node2 ~]$ srvctl start instance -d racdb -i racdb2

===========================================================================
[oracle@node1 ~]$ srvctl stop database -d racdb

[oracle@node1 ~]$ srvctl status database -d racdb -v
Instance racdb1 is not running on node node1
Instance racdb2 is not running on node node2
[oracle@node1 ~]$
[oracle@node1 ~]$ /data01/app/11204/grid_11204/bin/crsctl status res -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS      
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.DATA_NEW.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.FRA.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.LISTENER.lsnr
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCRNEW.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCRVOTE.dg
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.OCR_VOTE.dg
               ONLINE  OFFLINE      node1                                      
               ONLINE  OFFLINE      node2                                      
ora.asm
               ONLINE  ONLINE       node1                    Started            
               ONLINE  ONLINE       node2                    Started            
ora.gsd
               OFFLINE OFFLINE      node1                                      
               OFFLINE OFFLINE      node2                                      
ora.net1.network
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.ons
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
ora.registry.acfs
               ONLINE  ONLINE       node1                                      
               ONLINE  ONLINE       node2                                      
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       node1                                      
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       node2                                      
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       node2                                      
ora.cvu
      1        ONLINE  ONLINE       node2                                      
ora.node1.vip
      1        ONLINE  ONLINE       node1                                      
ora.node2.vip
      1        ONLINE  ONLINE       node2                                      
ora.oc4j
      1        ONLINE  ONLINE       node2                                      
ora.racdb.db
      1        OFFLINE OFFLINE                               Instance Shutdown  
      2        OFFLINE OFFLINE                               Instance Shutdown  
ora.scan1.vip
      1        ONLINE  ONLINE       node1                                      
ora.scan2.vip
      1        ONLINE  ONLINE       node2                                      
ora.scan3.vip
      1        ONLINE  ONLINE       node2                      

[root@node1 ~]# /data01/app/11204/grid_11204/bin/crsctl stop cluster -all
CRS-2673: Attempting to stop 'ora.crsd' on 'node1'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'node1'
CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'node1'
CRS-2673: Attempting to stop 'ora.FRA.dg' on 'node1'
CRS-2673: Attempting to stop 'ora.OCRNEW.dg' on 'node1'
CRS-2673: Attempting to stop 'ora.OCRVOTE.dg' on 'node1'
CRS-2673: Attempting to stop 'ora.registry.acfs' on 'node1'
CRS-2673: Attempting to stop 'ora.DATA_NEW.dg' on 'node1'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN1.lsnr' on 'node1'
CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.node1.vip' on 'node1'
CRS-2677: Stop of 'ora.LISTENER_SCAN1.lsnr' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.scan1.vip' on 'node1'
CRS-2677: Stop of 'ora.FRA.dg' on 'node1' succeeded
CRS-2677: Stop of 'ora.DATA_NEW.dg' on 'node1' succeeded
CRS-2677: Stop of 'ora.OCRNEW.dg' on 'node1' succeeded
CRS-2677: Stop of 'ora.scan1.vip' on 'node1' succeeded
CRS-2677: Stop of 'ora.node1.vip' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.crsd' on 'node2'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'node2'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN3.lsnr' on 'node2'
CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'node2'
CRS-2673: Attempting to stop 'ora.FRA.dg' on 'node2'
CRS-2673: Attempting to stop 'ora.OCRNEW.dg' on 'node2'
CRS-2673: Attempting to stop 'ora.OCRVOTE.dg' on 'node2'
CRS-2673: Attempting to stop 'ora.registry.acfs' on 'node2'
CRS-2673: Attempting to stop 'ora.DATA_NEW.dg' on 'node2'
CRS-2673: Attempting to stop 'ora.oc4j' on 'node2'
CRS-2673: Attempting to stop 'ora.cvu' on 'node2'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN2.lsnr' on 'node2'
CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.node2.vip' on 'node2'
CRS-2677: Stop of 'ora.LISTENER_SCAN3.lsnr' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.scan3.vip' on 'node2'
CRS-2677: Stop of 'ora.LISTENER_SCAN2.lsnr' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.scan2.vip' on 'node2'
CRS-2677: Stop of 'ora.cvu' on 'node2' succeeded
CRS-2677: Stop of 'ora.scan3.vip' on 'node2' succeeded
CRS-2677: Stop of 'ora.node2.vip' on 'node2' succeeded
CRS-2677: Stop of 'ora.OCRNEW.dg' on 'node2' succeeded
CRS-2677: Stop of 'ora.FRA.dg' on 'node2' succeeded
CRS-2677: Stop of 'ora.DATA_NEW.dg' on 'node2' succeeded
CRS-2677: Stop of 'ora.scan2.vip' on 'node2' succeeded
CRS-2677: Stop of 'ora.registry.acfs' on 'node1' succeeded
CRS-2677: Stop of 'ora.OCRVOTE.dg' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'node1'
CRS-2677: Stop of 'ora.asm' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.ons' on 'node1'
CRS-2677: Stop of 'ora.ons' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.net1.network' on 'node1'
CRS-2677: Stop of 'ora.net1.network' on 'node1' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'node1' has completed
CRS-2677: Stop of 'ora.crsd' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.ctssd' on 'node1'
CRS-2673: Attempting to stop 'ora.evmd' on 'node1'
CRS-2673: Attempting to stop 'ora.asm' on 'node1'
CRS-2677: Stop of 'ora.OCRVOTE.dg' on 'node2' succeeded
CRS-2677: Stop of 'ora.evmd' on 'node1' succeeded
CRS-2677: Stop of 'ora.registry.acfs' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'node2'
CRS-2677: Stop of 'ora.asm' on 'node2' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'node1' succeeded
CRS-2677: Stop of 'ora.asm' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'node1'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'node1'
CRS-2677: Stop of 'ora.cssd' on 'node1' succeeded
CRS-2677: Stop of 'ora.oc4j' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.ons' on 'node2'
CRS-2677: Stop of 'ora.ons' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.net1.network' on 'node2'
CRS-2677: Stop of 'ora.net1.network' on 'node2' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'node2' has completed
CRS-2677: Stop of 'ora.crsd' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.ctssd' on 'node2'
CRS-2673: Attempting to stop 'ora.evmd' on 'node2'
CRS-2673: Attempting to stop 'ora.asm' on 'node2'
CRS-2677: Stop of 'ora.evmd' on 'node2' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'node2' succeeded
CRS-2677: Stop of 'ora.asm' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'node2'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'node2'
CRS-2677: Stop of 'ora.cssd' on 'node2' succeeded

[root@node1 ~]# /data01/app/11204/grid_11204/bin/crsctl check has
CRS-4638: Oracle High Availability Services is online

[root@node1 ~]# ps -ef | grep d.bin
root     14259     1  0 03:11 ?        00:00:10 /data01/app/11204/grid_11204/bin/ohasd.bin reboot
grid     14430     1  0 03:11 ?        00:00:00 /data01/app/11204/grid_11204/bin/mdnsd.bin
grid     14441     1  0 03:11 ?        00:00:01 /data01/app/11204/grid_11204/bin/gpnpd.bin
grid     14452     1  0 03:11 ?        00:00:08 /data01/app/11204/grid_11204/bin/gipcd.bin
root     14466     1  1 03:11 ?        00:00:19 /data01/app/11204/grid_11204/bin/osysmond.bin

[root@node1 ~]# /data01/app/11204/grid_11204/bin/crsctl stop has
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'node1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'node1'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'node1'
CRS-2673: Attempting to stop 'ora.crf' on 'node1'
CRS-2677: Stop of 'ora.mdnsd' on 'node1' succeeded
CRS-2677: Stop of 'ora.crf' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'node1'
CRS-2677: Stop of 'ora.drivers.acfs' on 'node1' succeeded
CRS-2677: Stop of 'ora.gipcd' on 'node1' succeeded
CRS-2673: Attempting to stop 'ora.gpnpd' on 'node1'
CRS-2677: Stop of 'ora.gpnpd' on 'node1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'node1' has completed
CRS-4133: Oracle High Availability Services has been stopped.


[root@node2 ~]# /data01/app/11204/grid_11204/bin/crsctl check has
CRS-4638: Oracle High Availability Services is online
[root@node2 ~]# ps -ef | grep d.bin
root      7169     1  0 02:59 ?        00:00:14 /data01/app/11204/grid_11204/bin/ohasd.bin reboot
grid      7303     1  0 02:59 ?        00:00:00 /data01/app/11204/grid_11204/bin/mdnsd.bin
grid      7315     1  0 02:59 ?        00:00:01 /data01/app/11204/grid_11204/bin/gpnpd.bin
grid      7326     1  0 02:59 ?        00:00:10 /data01/app/11204/grid_11204/bin/gipcd.bin
root      7340     1  1 02:59 ?        00:00:38 /data01/app/11204/grid_11204/bin/osysmond.bin
root     14636  5007  0 03:41 pts/0    00:00:00 grep d.bin

[root@node2 ~]# /data01/app/11204/grid_11204/bin/crsctl stop has
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'node2'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'node2'
CRS-2673: Attempting to stop 'ora.crf' on 'node2'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'node2'
CRS-2677: Stop of 'ora.drivers.acfs' on 'node2' succeeded
CRS-2677: Stop of 'ora.crf' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'node2'
CRS-2677: Stop of 'ora.mdnsd' on 'node2' succeeded
CRS-2677: Stop of 'ora.gipcd' on 'node2' succeeded
CRS-2673: Attempting to stop 'ora.gpnpd' on 'node2'
CRS-2677: Stop of 'ora.gpnpd' on 'node2' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'node2' has completed
CRS-4133: Oracle High Availability Services has been stopped.
[root@node2 ~]#