troubleshooting early ssh access

[bootcd.git] / initscripts / pl_netinit

diff --git a/initscripts/pl_netinit b/initscripts/pl_netinit
index 0fa960a..b652674 100755 (executable)
--- a/initscripts/pl_netinit
+++ b/initscripts/pl_netinit
@@ -1,6 +1,8 @@
 #!/bin/sh
 #-*-shell-script-*-
 
+set -x
+
 # the name of the floppy based network configuration
 # files (checked first). the name planet.cnf is kept
 # for backward compatibility with old nodes, and only
@@ -56,10 +58,24 @@ IFCONFIG_OUTPUT=/tmp/ifconfig
 DEFAULT_NET_CONF=0
 
 
-net_init_failed()
-{
+function net_init_failed() {
+    echo
+    echo $(date "+%H:%M:%S") " pl_netinit: network initialization failed with interface $ETH_DEVICE"
+    echo
+    echo For forensics
+    echo
+    echo ========== lspci beg
+    /sbin/lspci -n | /bin/grep "Class 0200"    
+    echo ========== lspci end
     echo
-    echo $(date "+%H:%M:%S") " pl_netinit: network initialization failed,"
+    echo ========== ifconfig beg
+    /sbin/ifconfig
+    echo ========== ifconfig end
+    echo ========== ip address show beg
+    ip address show
+    echo ========== ip address show end
+    echo 
+    echo $(date "+%H:%M:%S") " pl_netinit: network initialization failed with interface $ETH_DEVICE"
     echo $(date "+%H:%M:%S") " pl_netinit: shutting down machine in two hours"
     /bin/sleep 2h
     /sbin/shutdown -h now
@@ -67,8 +83,7 @@ net_init_failed()
 }
 
 # Function for checking the IP address to see if its sensible.
-check_ip()
-{
+function check_ip() {
     case "$*" in
         "" | *[!0-9.]* | *[!0-9]) return 1 ;;
     esac
@@ -83,8 +98,7 @@ check_ip()
 # return 1 if found and parsed. if this is the case, DEFAULT_NET_CONF will 
 # be set to 1. For any found configuration file, $USED_NET_CONF will
 # contain the validated contents
-find_node_config()
-{
+function find_node_config() {
     /bin/rm -f $TMP_OLD_FLOPPY_CONF_FILE 2>&1 > /dev/null
 
     echo $(date "+%H:%M:%S") " pl_netinit: looking for node configuration file on floppy"
@@ -253,7 +267,7 @@ ETH_DEVICE=
 if [[ -n "$NET_DEVICE" ]]; then
     # the user specified a mac address we should use. find the network
     # device for it.
-    NET_DEVICE=$(tr A-Z a-z <<<$NET_DEVICE)
+    NET_DEVICE=$(tr A-Z a-z <<< $NET_DEVICE)
 
     pushd /sys/class/net
     for device in *; do
@@ -286,20 +300,47 @@ if [[ -z "$ETH_DEVICE" ]]; then
     net_init_failed
 fi
 
+# within a systemd-driven startup, we often see this stage
+# triggered before the network interface is actually exposed
+# by udev/kernel
+# although of course we have network-online.target
+# as a requirement; go figure what systemd actually does..
+
+# in any case, let us try to work around that by allowing some delay
+# here
+
+# tmp: Thierry June 2015
+# on fedora 21 nodes we see this running in a context where eth0 is not known to the system
+# could be related to a dependency that we poorly describe to systemd
+# I am increasing this timeout to 2 minutes in order to check that conjecture
+ALLOW=60
+COUNTER=0
+while true; do
+    if /sbin/ifconfig $ETH_DEVICE >& /dev/null; then
+       echo $(date "+%H:%M:%S") " pl_netinit: device present $ETH_DEVICE, proceeding (${COUNTER}s/${ALLOW}s)"
+       break
+    fi
+    echo $(date "+%H:%M:%S") " pl_netinit: waiting for device $ETH_DEVICE - ${COUNTER}s/${ALLOW}s"
+    set -x
+    /sbin/ifconfig
+    journalctl -b | egrep 'eth|bnx|udev'
+    systemctl list-unit-files | grep -i network
+    set +x 
+    COUNTER=$(($COUNTER+1))
+    [ $COUNTER -ge $ALLOW ] && net_init_failed
+    sleep 1
+done
+
 
 # actually check to make sure ifconfig <device> succeeds
-/sbin/ifconfig $ETH_DEVICE up 2>&1 > /dev/null
-if [[ $? -ne 0 ]]; then
+
+/sbin/ifconfig $ETH_DEVICE up 2>&1 > /dev/null || {
     echo $(date "+%H:%M:%S") " pl_netinit: device $ETH_DEVICE does not exist, most likely"
     echo $(date "+%H:%M:%S") " pl_netinit: this CD does not have hardware support for your"
     echo $(date "+%H:%M:%S") " pl_netinit: network adapter. please send the following lines"
     echo $(date "+%H:%M:%S") " pl_netinit: to your PlanetLab support for further assistance"
-    echo
-    /sbin/lspci -n | /bin/grep "Class 0200"    
-    echo
-
     net_init_failed
-fi
+}
 
 echo $(date "+%H:%M:%S") " pl_netinit: attempting to start networking"
 /sbin/service network start
@@ -309,3 +350,23 @@ echo $(date "+%H:%M:%S") " pl_netinit: attempting to start networking"
 
 echo $(date "+%H:%M:%S") " pl_netinit: network online"
 
+echo "BEG post pl_netinit forensics"
+
+file=/root/.ssh/authorized_keys
+echo "$file"
+if ! [ -f "$file" ]; then
+    echo "!!!!!!!!!! missing $file"
+else
+    echo "---------- $file"
+    cat $file
+fi
+
+file="/etc/ssh/sshd_config"    
+if ! [ -f "$file" ]; then
+    echo "!!!!!!!!!! missing $file"
+else
+    echo '---------- egrep Pass|Auth|PAM' $file
+    egrep 'Pass|Auth|PAM' $file
+fi
+
+echo "END post pl_netinit forensics"