troubleshooting early ssh access

[bootcd.git] / initscripts / pl_netinit

diff --git a/initscripts/pl_netinit b/initscripts/pl_netinit
index 5e288c2..b652674 100755 (executable)
--- a/initscripts/pl_netinit
+++ b/initscripts/pl_netinit
@@ -1,6 +1,8 @@
 #!/bin/sh
 #-*-shell-script-*-
 
+set -x
+
 # the name of the floppy based network configuration
 # files (checked first). the name planet.cnf is kept
 # for backward compatibility with old nodes, and only
@@ -58,7 +60,22 @@ DEFAULT_NET_CONF=0
 
 function net_init_failed() {
     echo
-    echo $(date "+%H:%M:%S") " pl_netinit: network initialization failed,"
+    echo $(date "+%H:%M:%S") " pl_netinit: network initialization failed with interface $ETH_DEVICE"
+    echo
+    echo For forensics
+    echo
+    echo ========== lspci beg
+    /sbin/lspci -n | /bin/grep "Class 0200"    
+    echo ========== lspci end
+    echo
+    echo ========== ifconfig beg
+    /sbin/ifconfig
+    echo ========== ifconfig end
+    echo ========== ip address show beg
+    ip address show
+    echo ========== ip address show end
+    echo 
+    echo $(date "+%H:%M:%S") " pl_netinit: network initialization failed with interface $ETH_DEVICE"
     echo $(date "+%H:%M:%S") " pl_netinit: shutting down machine in two hours"
     /bin/sleep 2h
     /sbin/shutdown -h now
@@ -250,7 +267,7 @@ ETH_DEVICE=
 if [[ -n "$NET_DEVICE" ]]; then
     # the user specified a mac address we should use. find the network
     # device for it.
-    NET_DEVICE=$(tr A-Z a-z <<<$NET_DEVICE)
+    NET_DEVICE=$(tr A-Z a-z <<< $NET_DEVICE)
 
     pushd /sys/class/net
     for device in *; do
@@ -283,20 +300,47 @@ if [[ -z "$ETH_DEVICE" ]]; then
     net_init_failed
 fi
 
+# within a systemd-driven startup, we often see this stage
+# triggered before the network interface is actually exposed
+# by udev/kernel
+# although of course we have network-online.target
+# as a requirement; go figure what systemd actually does..
+
+# in any case, let us try to work around that by allowing some delay
+# here
+
+# tmp: Thierry June 2015
+# on fedora 21 nodes we see this running in a context where eth0 is not known to the system
+# could be related to a dependency that we poorly describe to systemd
+# I am increasing this timeout to 2 minutes in order to check that conjecture
+ALLOW=60
+COUNTER=0
+while true; do
+    if /sbin/ifconfig $ETH_DEVICE >& /dev/null; then
+       echo $(date "+%H:%M:%S") " pl_netinit: device present $ETH_DEVICE, proceeding (${COUNTER}s/${ALLOW}s)"
+       break
+    fi
+    echo $(date "+%H:%M:%S") " pl_netinit: waiting for device $ETH_DEVICE - ${COUNTER}s/${ALLOW}s"
+    set -x
+    /sbin/ifconfig
+    journalctl -b | egrep 'eth|bnx|udev'
+    systemctl list-unit-files | grep -i network
+    set +x 
+    COUNTER=$(($COUNTER+1))
+    [ $COUNTER -ge $ALLOW ] && net_init_failed
+    sleep 1
+done
+
 
 # actually check to make sure ifconfig <device> succeeds
-/sbin/ifconfig $ETH_DEVICE up 2>&1 > /dev/null
-if [[ $? -ne 0 ]]; then
+
+/sbin/ifconfig $ETH_DEVICE up 2>&1 > /dev/null || {
     echo $(date "+%H:%M:%S") " pl_netinit: device $ETH_DEVICE does not exist, most likely"
     echo $(date "+%H:%M:%S") " pl_netinit: this CD does not have hardware support for your"
     echo $(date "+%H:%M:%S") " pl_netinit: network adapter. please send the following lines"
     echo $(date "+%H:%M:%S") " pl_netinit: to your PlanetLab support for further assistance"
-    echo
-    /sbin/lspci -n | /bin/grep "Class 0200"    
-    echo
-
     net_init_failed
-fi
+}
 
 echo $(date "+%H:%M:%S") " pl_netinit: attempting to start networking"
 /sbin/service network start
@@ -305,3 +349,24 @@ echo $(date "+%H:%M:%S") " pl_netinit: attempting to start networking"
 /sbin/ifconfig $ETH_DEVICE > $IFCONFIG_OUTPUT
 
 echo $(date "+%H:%M:%S") " pl_netinit: network online"
+
+echo "BEG post pl_netinit forensics"
+
+file=/root/.ssh/authorized_keys
+echo "$file"
+if ! [ -f "$file" ]; then
+    echo "!!!!!!!!!! missing $file"
+else
+    echo "---------- $file"
+    cat $file
+fi
+
+file="/etc/ssh/sshd_config"    
+if ! [ -f "$file" ]; then
+    echo "!!!!!!!!!! missing $file"
+else
+    echo '---------- egrep Pass|Auth|PAM' $file
+    egrep 'Pass|Auth|PAM' $file
+fi
+
+echo "END post pl_netinit forensics"