Never miss an old file

[myops.git] / web / collect / server / cron.hourly / load_all_couchdb.sh

diff --git a/web/collect/server/cron.hourly/load_all_couchdb.sh b/web/collect/server/cron.hourly/load_all_couchdb.sh
index 40d76dd..5b43fd4 100755 (executable)
--- a/web/collect/server/cron.hourly/load_all_couchdb.sh
+++ b/web/collect/server/cron.hourly/load_all_couchdb.sh
@@ -1,28 +1,25 @@
 #!/bin/bash
 
-PREFIX=/var/www/html/uploadlogs/raw
-EXEC_PREFIX=/usr/share/myops/web/collect/server
 
-lasthour=`date +%Y-%m-%dT%H -d "1 hour ago"`
-mkdir -p $PREFIX/old/$lasthour
-for file in `ls $PREFIX/$lasthour*--upload`; do 
-       $EXEC_PREFIX/load_couch.py $file && mv $file $PREFIX/old/$lasthour
-done
+PREFIX=/var/www/html/uploadlogs/
+EXEC_PREFIX=/usr/share/myops/web/collect/server
 
-#NOTE: temporary; catch any stragglers still being sent to monitor and copied here
-for file in `ls /root/$lasthour*--upload`; do 
-       $EXEC_PREFIX/load_couch.py $file && mv $file $PREFIX/old/$lasthour
-done
+cd $PREFIX
+NS_LIST=`ls`
+for ns in $NS_LIST ; do
+       # NOTE: look at all uploaded files, and find unique oldhours.
+       for oldhour in `ls | awk -F: '{print $1}' | sort | uniq | grep T`; do 
+               
+               mkdir -p $PREFIX/$ns/old/$oldhour
+               for file in `ls $PREFIX/$ns/$oldhour*--upload`; do 
+                       $EXEC_PREFIX/load_couch.py $file && mv $file $PREFIX/$ns/old/$oldhour
+               done
 
-$EXEC_PREFIX/summarize_logs.sh || :
-$EXEC_PREFIX/summarize_rpms.sh || :
-$EXEC_PREFIX/summarize_stats.sh || :
+               $EXEC_PREFIX/summarize_logs.sh $ns $oldhour || :
+               $EXEC_PREFIX/summarize_rpms.sh $ns $oldhour || :
+               $EXEC_PREFIX/summarize_stats.sh $ns $oldhour || :
 
-thishour=`date +%Y-%m-%dT%H`
-mkdir -p $PREFIX/old/$thishour
-# NOTE: do it twice for all the files that were uploaded while the above was running.
-for file in `ls $PREFIX/$thishour*--upload`; do 
-       $EXEC_PREFIX/load_couch.py $file && mv $file $PREFIX/old/$thishour
+       done
 done
 
 curl -s 'http://HOSTNAME:5984/myops/_design/myops/_list/nodelist/node-status?fields=hostname&skip_header' | wc -l &> /dev/null