帮忙看下nginx日志切割的问题

筷子eason

rm-rf 发表于 2016-12-04 21:39
把日志写在管道里，然后通过awk对这个管道文件按时间关键字匹配，每10分钟切成一个新文件如何？

我和这个差不多的 等下发出来大家看看哈

筷子eason

回复 10# 筷子eason

我发下我的代码大家看看
#!/bin/bash
filename="access.log"
LANG=en
#sj=`date -d '-10 minute' |awk '{print $3"/"$2"/"$NF":"substr($4,0,4)}'`
sj=`date -d '-10 minute' |awk '{if($3>=10){print $3"/"$2"/"$NF":"substr($4,0,4)}else{print "0"$3"/"$2"/"$NF":"substr(
$4,0,4)}}'`
file=`echo $sj|awk -F'[/:]' '{print $3"-"$2"-"$1"_"$4$NF"0"}'`

#count=`grep $sj $filename|wc -l`
#[ $count -ne 0 ] && grep $sj $filename > $file
grep $sj $filename > $file

筷子eason

回复 10# 筷子eason

我发下我的代码 大家看看。
#!/bin/bash
filename="access.log"
LANG=en
#sj=`date -d '-10 minute' |awk '{print $3"/"$2"/"$NF":"substr($4,0,4)}'`
sj=`date -d '-10 minute' |awk '{if($3>=10){print $3"/"$2"/"$NF":"substr($4,0,4)}else{print "0"$3"/"$2"/"$NF":"substr(
$4,0,4)}}'`
file=`echo $sj|awk -F'[/:]' '{print $3"-"$2"-"$1"_"$4$NF"0"}'`

#count=`grep $sj $filename|wc -l`
#[ $count -ne 0 ] && grep $sj $filename > $file
grep $sj $filename > $file

筷子eason

#!/bin/bash
filename="access.log"
LANG=en
#sj=`date -d '-10 minute' |awk '{print $3"/"$2"/"$NF":"substr($4,0,4)}'`
sj=`date -d '-10 minute' |awk '{if($3>=10){print $3"/"$2"/"$NF":"substr($4,0,4)}else{print "0"$3"/"$2"/"$NF":"substr(
$4,0,4)}}'`
file=`echo $sj|awk -F'[/:]' '{print $3"-"$2"-"$1"_"$4$NF"0"}'`

#count=`grep $sj $filename|wc -l`
#[ $count -ne 0 ] && grep $sj $filename > $file
grep $sj $filename > $file

moperyblue

回复 14# 筷子eason


用不用考虑"漏采"的问题 如,程序停止一个小时 是不是这个小时的数据就无法生成到$file文件?

xiaojao

请参考，原始日志会按天截断的，脚本取记录上次取的结果，然后用Sed来处理。

#!/bin/bash
# by jiaozz
#
# need root permission

## add the file to archive#####################################################################


### avoid this scripts repetitive execution

if [ -f /var/run/ngx2flume ];then
echo "`basename $0`  is running ? "
echo "if no,  delete  /var/run/ngx2flume file !"
exit 1
fi

touch /var/run/ngx2flume

## 1 init the archive via #################################

ngxlogdir="/usr/local/nginx/logs"
flumespooldir="/usr/local/flume/logs_accesslog"


declare -a logfile=( a.abc.cn-access.log b.abc.cn-access.log)
rq=`date +%Y%m%d`

runlog="/var/log/run.ngxlog2flume.log"
dolog="/var/log/do.ngxlog2flume.$rq.log"
tempstatus="/tmp/tempusengx2flume"

creatrunstatus () {
   for f in ${logfile[@]}
   do
     if [ -f "$ngxlogdir/$f" ]
     then
     finode=`stat -c %i $ngxlogdir/$f`
     echo "$f'sizeis:0 lastlineis:0 filelineNumis:0 inodeisfinode seq:0"
     fi
  done
}

### 2  create file for this script use #################
if [ ! -f $runlog ];then
   creatrunstatus >$runlog
  # chattr +i $runlog
  echo "init the status file $runlog"
  rm -f /var/run/ngx2flume
  exit 0
fi


## allow change protection  file
chattr -i $runlog
##

### 3 mv this scripts used temfile#######################

if [ -f $tempstatus ];then /bin/mv -f $tempstatus $tempstatus.bak ; fi

#### 4  do with the log ##################################
for nf in ${logfile[@]}
do  
if grep -q ^$nf "$runlog" ;then   

   ### get old status
   oldsize=`grep "^$nf" "$runlog"     |awk '{print $1}'|awk -F: '{print $2}'`
   oldlastline=`grep "^$nf" "$runlog" |awk '{print $2}'|awk -F: '{print $2}'`
   oldtotalline=`grep "^$nf" "$runlog"|awk '{print $3}'|awk -F: '{print $2}'`
   oldinode=`grep "^$nf" "$runlog"    |awk '{print $4}'|awk -F: '{print $2}'`
   oldseq=`grep "^$nf" "$runlog"      |awk '{print $5}'|awk -F: '{print $2}'`
   #### get new file status
   newsize=`stat -c %s $ngxlogdir/$nf`
   newline=`wc -l $ngxlogdir/$nf|awk '{print $1}'`
   echo newline is $newline
   newinode=`stat -c %i $ngxlogdir/$nf`
   let newseq=$oldseq+1
  
   ####  empty file
   
   if [ $newsize -eq 0 -o $oldinode -ne $newinode ];then
       echo "$nf'sizeisnewsize lastlineis:0 filelineNumis:0 inodeisnewinode seqoldseq" >> $tempstatus
       echo "WARNINGnewseq `date` $nf is empty or not same file,  we do it next time! "  >>$dolog
       continue
   fi
   ### no change file
   if [ $oldinode -eq $newinode -a  $newline -eq $oldtotalline ];then
       echo "$nf'sizeisoldsize lastlineisoldtotalline filelineNumisoldtotalline inodeisnewinode seqoldseq" >> $tempstatus
       echo "WARNING:$newseq `date` $nf no changed ,we do it next time! "  >>$dolog
       continue
   fi
  ###  newline is less than lastline
   if [ $oldinode -eq $newinode -a  $newline -lt $oldtotalline ];then
       echo "$nf'sizeis:$newsize lastlineis:0 filelineNumis:0 inodeis:$newinode seq:$oldseq" >> $tempstatus
       echo "WARNING:$newseq `date` $nf maybe truncate ,we do it next time! "  >>$dolog
       continue
   fi
  ### newline is greater than lastline
   if [ $oldinode -eq $newinode -a  $newline -gt $oldtotalline ];then
       let oldlastline=$oldlastline+1
       echo "$nf'sizeis:$newsize lastlineis:$newline filelineNumis:$newline inodeis:$newinode seq:$newseq" >> $tempstatus

  ### do with the log ###
       echo " sed -n '$oldlastline,$newline'p $ngxlogdir/$nf > $flumespooldir/$rq.$newseq.$nf.tmp " >/tmp/cmdcmdcmdsed
       sh /tmp/cmdcmdcmdsed
       result=$?
       if [ $result -eq 0 ];then
       echo "INFO:$newseq `date` do $nf ok,lastline is $oldlastline,this endline is $newline,now rename it! " >>$dolog
       mv $flumespooldir/$rq.$newseq.$nf.tmp $flumespooldir/$rq.$newseq.$nf
       else
       echo "ERROR:$newseq `date` do $nf failed,sed return is $result " >>$dolog
       fi
   else
       echo "ERROR:$newseq `date`  $nf  file status error,no do with it " >>$dolog

   fi

else
echo "WARNING:find new name log now init with it " >>$dolog
echo "$nf'sizeis:0 lastlineis:1 filelineNumis:1 inodeis:0 seq:0" >> $tempstatus
fi

done


### 5 overrite the old status file #######################  
### deny write protection file
   copy="/bin/cp"
  $copy $tempstatus $runlog
# rm -f $tempstatus
chattr +i $runlog

##########################################################
rm  -f /var/run/ngx2flume

exit 0

xiaojao

不好意思多发了

xiaojao

不好意思多发了

筷子eason

回复 18# xiaojao

能否加你QQ

xiaojao

我QQ 124422735