10/29/15

【shell】日志归档分类归档

#!/bin/sh
#遍历当前目录,将日志压缩文件按照域名、日志日期分类
#为了日志统一管理,完善日志系统分析、采集、备份、统计等需求
#date:2015-10-29
#add by chen-123

for file in $(find . -name "*gz" -type f);
do
        path=${file#*/}
        cur_parent_dir=${path%%/*}
        old_dir=${path%/*}
        old_sub_dir=${old_dir%/*}
        project_log_dir_name=$(echo ${file##*/}|awk -F '[-_]' '{print $1}')
        log_filename=$(echo ${file##*/})
        if [ "${cur_parent_dir}" != "${project_log_dir_name}" ];then
                echo $log_filename
                if [ "${old_dir}" = "${old_sub_dir}" ];then
                        test -d "./"$project_log_dir_name || mkdir -p "./"$project_log_dir_name
                        moveto_path="./"$project_log_dir_name"/"$log_filename
                        test -f "${moveto_path}" && moveto_path="./"$project_log_dir_name"/"${log_filename/[_-]/_${cur_parent_dir}_/}
                        test -f "$moveto_path" ||  mv $file $moveto_path
                        test -f "$moveto_path" && echo "move $file to $moveto_path success"
                else
                        new_sub_dir=${old_dir##*/}
                        moveto_path=$project_log_dir_name"/"$new_sub_dir"/"$log_filename
                        test -d $project_log_dir_name"/"$new_sub_dir || mkdir -p $project_log_dir_name"/"$new_sub_dir
                        test -f "$moveto_path" && moveto_path="./"$project_log_dir_name"/"${log_filename/[_-]/_${cur_parent_dir}_/}
                        test -f "$moveto_path" ||  mv $file $moveto_path
                        test -f "$moveto_path" && echo "move $file to $moveto_path success"
                fi
        fi
done

08/19/14

【shell】清除nginx缓存目录下指定文件

描述:项目上线,运维一般会将样式、图片等文件缓存。上线初期经常需要更新样式文件,该脚本可以交由开发人员执行,减少运维工作量。

#! /bin/sh
#清除指定目录下,指定文件
#主要为了解决清除nginx缓存目录下的缓存文件
#link: blog.phpdba.com
#date: 2014-08-19
#author: chen-123

#Define Path
CACHE_DIR=/tmp/zaojiao.phpdba.com/
FILE="$*"

if [  "$#" -eq "0" ];then
        echo "Please Input clean File, Example: $0 index.html index.js style.css"
        sleep 2 && exit
fi
echo "The file : $FILE to be clean Cache ,please waiting ....."

for i in `echo $FILE`
do
        find ${CACHE_DIR} -name  $i > /tmp/cache_list.txt
        for j in `cat /tmp/cache_list.txt`
        do
                rm  -rf $j
                echo "$i $j is Deleted Success !"
        done
done

脚本下载链接:https://github.com/chen-123/phper/edit/master/cache_nginx.sh

08/19/14

【shell】批量删除缓存图片

描述:现在项目中有好多项目缓存了图片、样式等文件,动不动就10个G的级别

#/bin/sh
# 脚本用于批量删除指定目录下小文件
# link:blog.phpdba.com
# date: 2014-08-18
# author: chen-123

Base_Folder="/ask_d"
Folder='askques'
Old_Folder='askques_old'
Tmp_Folder='tmp'
Check_Old="yes"



if [ "`pwd`" != "$Base_Folder" ]
then
        echo "请将脚本放在"$Base_Folder"目录下执行"
        exit
fi


if [ -d $Tmp_Folder -a "`ls -A $Tmp_Folder`" != "" ]
then
        echo "请检查目录"$Tmp_Folder"是否为空目录"
fi

if [ -d $Old_Folder ]
then
        echo "目录存在"
        if [ "`ls -A $Old_Folder`" == "" ];then
                echo "$Old_Folder 是空目录,请重新执行一次"
                `rm -rf $Old_Folder`
        else
                Old_Size=`du -s ${Old_Folder}|awk '{print $1}'`

                if [ $Old_Size -ge 102400 ]
                then
                        echo $Old_Folder"目录比较大,请检查"
                        if [ "$Check_Old" != "yes" ]
                        then
                                if [ -d $Tmp_Folder ]
                                then
                                        rsync --delete-before -a -H -v --progress --stats $Base_Folder"/"$Tmp_Folder"/" $Base_Folder"/"$Old_Folder"/"
                                else
                                        echo "请确认"$Tmp_Folder"目录是否存在"
                                fi
                        fi
                else
                        echo "正在删除目录"$Old_Folder" ,请重新执行一次"
                        `rm -rf $Old_Folder`
                fi
        fi
else
        echo "目录不存在,创建目录"$Old_Folder
        `mv $Folder $Old_Folder && mkdir $Folder && chown -R apache:apache $Folder`
        if [ -d $Tmp_Folder ]
        then
                rsync --delete-before -a -H -v --progress --stats $Base_Folder"/"$Tmp_Folder"/" $Base_Folder"/"$Old_Folder"/"
                `rm -rf $Old_Folder`
        else
                echo "请确认"$Tmp_Folder"目录是否存在"
        fi
fi

脚本下载链接:https://github.com/chen-123/phper/edit/master/batch_delete_file.sh

07/29/14

【shell】按照月份将apache日志文件压缩

描述:nagios监控磁盘空间warning,检查发现apache日志占用空间比较大

#!/bin/sh
#将当前目录下常规apache日志文件,按照月份归集
#link: blog.phpdba.com
#date:2014-07-29
#author: chen-123

export LANG="en_US.UTF-8"
project_list=`ls -l|grep 'access\.'|awk '{print $NF}'|awk -F'-access.' '{print $1}'|uniq`
month_list=`ls -l|grep 'access\.'|awk '{print $NF}'|awk -F'-access.' '{print $2}'|sort -n|uniq|cut -c 1-2|uniq`
cur_month=`date +%m`
for project in $project_list
do
for month in $month_list
do
echo $project
log_num=`ls |grep $project'-access.'$month|wc -l`
echo $log_num
if [ $log_num -gt 0 ];then
        if [ "$month" != "$cur_month" ];then
                echo $project" "$month" "$log_num
                tar_file=${project}"_access_"${month}".tar.gz"
                files=${project}"-access."${month}"[0-9][0-9]"
                files_total_size=`ls $files|xargs du -cb|grep total|awk '{print $1}'`
                files_avg_size=`echo "$files_total_size/$log_num"|bc`
                echo $files_avg_size
                if [ $files_avg_size -gt 1073741824 ];then
                        for file in $files
                        do
                                day=`echo $file|awk -F'-access.' '{print $2}'`
                                tar_filenew=${project}"_access_"${day}".tar.gz"
                                echo $tar_filenew
                                [ -f $tar_filenew ] || /bin/tar zcf $tar_filenew $file
                                [ -f $tar_filenew ] && rm -f $file
                        done
                else
                        echo $tar_file
                        [ -f $tar_file ] || /bin/tar zcf $tar_file $files
                        [ -f $tar_file ] && rm -f $files
                fi
                #echo $files_total_size
                #/bin/tar zcf $tar_file $files
        fi
fi
done
done

脚本下载链接:https://github.com/chen-123/phper/blob/master/log_archive.sh

12/18/13

shell ASCII text转码

1、现象:

cat 可以正常显示,vi却乱码

2、

[root@yf_113 ~]# file phpdba.txt
phpdba.txt: ASCII text, with escape sequences

3、转码shell脚本

#!/bin/bash
CSI=’\x1b\[‘            # the two-character csi escape sequence .Esc [., followed by i
Rp=’[0-9:;<=>?]‘        # an optional sequence of parameter bytes in the range x30.x3F,
Ri=’[- !\"#$%&'\''()*+,./]‘ # an optional sequence of intermediate bytes in the range x20.x2F,
Rf=’[]@A-Z[\\^_`a-z{|}~]‘   # and a final byte in the range x40.x7e.
# The set of standard control sequence functions are defined in Ecma-48 / ISO/IEC 6429.
cat phpdba.txt | sed -e ‘s/’”$CSI$Rp”‘*’”$Ri”‘*’”$Rf”‘//g’ > phpdbautf8.txt

utf8格式转换成功

12/12/13

shell 分析iosapi接口调用情况

一、代码展示
#!/bin/bash
INIT_PATH="/opt/phpdba/apache/logs"
FILEGREP='1208'
Date='08/Dec/2013'
hour=`cat hour.txt`

for a in $hour
do
i=($(echo $a|awk -F'/' '{print $1}'))
e=($(echo $a|awk -F'/' '{print $2}'))

cat $INIT_PATH/www.phpdba.com-access.$FILEGREP |grep $Date$i|grep 'ios/api'|awk -F'"'  '{print $2" "$3" "$1}'>>phpdba/${e}_iosapi.log
cat $INIT_PATH/www.phpdba.com-access.$FILEGREP |grep $Date$i|grep 'ios/api'|awk -F'"'  '{print $6}'>>phpdba/${e}_iosapi_refer2.log
cat phpdba/${e}_iosapi_refer2.log |sort -rn|uniq -c|sort -rn>>phpdba/${e}_iosapi_refer_sort.log
cat phpdba/${e}_iosapi.log |awk '{print $1" "$2" "$(NF-6)}'>>phpdba/${e}_iosapi_good.log
cat phpdba/${e}_iosapi_good.log |awk '{print $3" "$1" "$2}'>>phpdba/${e}_iosapi_good_new.log
cat phpdba/${e}_iosapi_good_new.log|awk -F'?' '{print $1}'>>phpdba/${e}_iosapi_good_ok.log
cat phpdba/${e}_iosapi_good_ok.log|sort -rn|uniq -c |sort -rn>>phpdba/${e}_iosapi_good_ok_sortip.log
cat phpdba/${e}_iosapi_good_ok.log|awk '{print $3}'|sort -rn|uniq -c |sort -rn>>phpdba/${e}_iosapi_good_ok_urlsort.log
cat phpdba/${e}_iosapi_good_ok.log|awk '{print $2" "$3}'|sort -rn|uniq -c |sort -rn>>phpdba/${e}_iosapi_good_ok_getpost.log
num=`cat phpdba/${e}_iosapi_good_ok.log|wc -l`
echo "$e => $num">>phpdba/time_total.txt
cat phpdba/${e}_iosapi_good_ok_getpost.log>>phpdba/iosapi_good_ok_getpost_total.log
cat phpdba/${e}_iosapi_good_ok_sortip.log>>phpdba/iosapi_good_ok_sortip_total.log
cat phpdba/${e}_iosapi_good_ok_urlsort.log>>phpdba/iosapi_good_ok_urlsort_total.log

done
cat phpdba/iosapi_good_ok_getpost_total.log |awk '{arr[$2" "$3]+=$1}END{for(s in arr) print arr[s],s}'|sort -rn>>phpdba/iosapi_good_ok_getpost_total_ok.txt
cat phpdba/iosapi_good_ok_sortip_total.log |awk '{arr[$2]+=$1}END{for(s in arr) print arr[s],s}'|sort -rn>>phpdba/iosapi_good_ok_sortip_total_ok.txt
cat phpdba/iosapi_good_ok_sortip_total.log |awk '{arr[$3" "$4]+=$1}END{for(s in arr) print arr[s],s}'|sort -rn>>phpdba/iosapi_good_ok_sortip_total_ok2.txt
cat phpdba/iosapi_good_ok_sortip_total.log |awk '{arr[$4]+=$1}END{for(s in arr) print arr[s],s}'|sort -rn>>phpdba/iosapi_good_ok_sortip_total_ok3.txt
cat phpdba/iosapi_good_ok_urlsort_total.log |awk '{arr[$2]+=$1}END{for(s in arr) print arr[s],s}'|sort -rn>>phpdba/iosapi_good_ok_urlsort_total_ok.txt 

二、代码下载
phpdba.tar
11/26/13

sphinx coreseek-3.2.14 关键词搜索统计

1、keyword.txt

[root@ts7 keyword]# head -n 6 keyword.txt
宝宝长湿疹
宝宝湿疹怎么办
宝宝湿疹的原因
宝宝湿疹吃什么好
宝宝腹泻怎么办
宝宝腹泻用药

2、keyword.sh

#!/bin/sh
for i in `ls /opt/phpdba/coreseek-3.2.14/var/log/*.log`
do
        cat keyword.txt|while read line
        do
                tmp=`echo $i|awk -F'/' '{print $NF}'`
                echo $tmp
                `cat $i|grep "$line">${line}_$tmp`
                echo $line
        done
done

[root@ts7 keyword]# ll *.log
-rw-r–r– 1 root root    250 11-20 16:21 乳蛋白部分水解配方_query_bbs.log
-rw-r–r– 1 root root    296 11-20 16:22 乳蛋白部分水解配方_query_keyword.log
-rw-r–r– 1 root root    204 11-20 16:20 初生婴儿奶粉_query_app_yunqi.log
-rw-r–r– 1 root root    304 11-20 16:20 初生婴儿奶粉_query_ask.log
-rw-r–r– 1 root root   4806 11-20 16:21 初生婴儿奶粉_query_bbs3320.log
-rw-r–r– 1 root root  70080 11-20 16:22 初生婴儿奶粉_query_keyword.log
-rw-r–r– 1 root root  28351 11-20 16:23 初生婴儿奶粉_query.log
-rw-r–r– 1 root root    829 11-20 16:19 宝宝不吃奶怎么办_query_app_common.log
-rw-r–r– 1 root root    780 11-20 16:20 宝宝不吃奶怎么办_query_app_yunqi.log

3、deal_file_day.sh

#!/bin/sh
find . -name "*.log" -type f -size 0c|xargs -n 1 rm -f

for i in `find . -type f -name "*.log"`
do
key=`echo "$i"|awk -F'_' '{print $1}'`
cat $i|awk '{S[$2" "$3]++} END {for (K in S) print K,S[K]}'>>${key}_day.txt
#echo $i
done

deal_file_month.sh

#!/bin/sh
find . -name "*.log" -type f -size 0c|xargs -n 1 rm -f

for i in `find . -type f -name "*.log"`
do
key=`echo "$i"|awk -F'_' '{print $1}'`
cat $i|awk '{S[$2]++} END {for (K in S) print K,S[K]}'>>${key}.txt
#echo $i
done

[root@ts7 keyword]# ll *.txt
-rw-r–r– 1 root root    36 11-20 16:32 乳蛋白部分水解配方_day.txt
-rw-r–r– 1 root root    24 11-20 16:30 乳蛋白部分水解配方.txt
-rw-r–r– 1 root root   434 11-20 16:32 初生婴儿奶粉_day.txt
-rw-r–r– 1 root root   103 11-20 16:30 初生婴儿奶粉.txt
-rw-r–r– 1 root root   801 11-20 16:32 宝宝不吃奶怎么办_day.txt
-rw-r–r– 1 root root   247 11-20 16:30 宝宝不吃奶怎么办.txt
-rw-r–r– 1 root root  4296 11-20 16:32 宝宝便秘怎么办_day.txt
-rw-r–r– 1 root root   388 11-20 16:30 宝宝便秘怎么办.txt
-rw-r–r– 1 root root   489 11-20 16:32 宝宝吃奶哭闹_day.txt
-rw-r–r– 1 root root   116 11-20 16:30 宝宝吃奶哭闹.txt
-rw-r–r– 1 root root  4972 11-20 16:32 宝宝呕吐_day.txt
-rw-r–r– 1 root root   400 11-20 16:30 宝宝呕吐.txt
-rw-r–r– 1 root root     9 11-20 16:32 宝宝呕吐原因_day.txt
-rw-r–r– 1 root root     6 11-20 16:30 宝宝呕吐原因.txt
-rw-r–r– 1 root root    79 11-20 16:32 宝宝呕吐吃什么_day.txt
-rw-r–r– 1 root root    37 11-20 16:30 宝宝呕吐吃什么.txt

4、keyword_month.sh

#!/bin/sh

cat keyword.txt|while read line
do
        echo $line
        if [ -f ${line}.txt ];then
        cat ${line}.txt|awk '{S[$1]+=$2} END {for (k in S) print k,S[k]}'|sort -k 2 -rn|head -n 12
        else
        echo " 0"
        fi
done

keyword_day.sh

#!/bin/sh

cat keyword.txt|while read line
do
        echo $line
        if [ -f ${line}.txt ];then        
        cat ${line}_day.txt|awk '{S[$1" "$2]+=$3} END {for (k in S) print k,S[k]}'|sort -k 3 -rn|head -n 30
        else
        echo " 0"
        fi
done

5、执行脚本生产统计文本

sh keyword_day.sh >keyword_list_day.txt

sh keyword_month.sh >keyword_list_month.txt

宝宝烦躁怎么办
Jul 3 20
Sep 23 1
Oct 21 1
Nov 20 1
Nov 19 1
Nov 12 1
Jun 18 1
宝宝烦躁原因
0
初生婴儿奶粉
Oct 19 900
Sep 26 7
Sep 17 7
Sep 16 6
Nov 20 6
May 18 6
Nov 15 5
Oct 8 3
Oct 18 2
Nov 19 2
Nov 12 2
Jun 21 2
Jun 20 2
Dec 9 2
Aug 26 2
Sep 29 1
Sep 24 1
Sep 22 1
Sep 2 1
Sep 15 1
Sep 13 1
Oct 6 1
Oct 31 1
Oct 26 1
Oct 25 1
Oct 21 1
Oct 17 1
Oct 13 1
Oct 12 1
Oct 11 1
乳蛋白部分水解配方
Oct 17 2
Nov 19 2
部分水解蛋白配方奶粉
0
部分水解蛋白配方
Nov 11 14
Jul 14 13

宝宝腹胀
Nov 136
Sep 111
Jun 100
Oct 97
Jul 97
Aug 79
May 14
Mar 13
Jan 13
Apr 12
Feb 11
Dec 5
宝宝腹胀症状
0
宝宝烦躁怎么办
Jul 20
Nov 3
Sep 1
Oct 1
Jun 1
宝宝烦躁原因
0
初生婴儿奶粉
Oct 915
Sep 26
Nov 17
May 6
Jun 6
Jul 6
Aug 3
Dec 2
Jan 1
乳蛋白部分水解配方
Oct 2
Nov 2
部分水解蛋白配方奶粉
0
部分水解蛋白配方
Nov 14
Jul 13

11/22/13

squid 重启脚本

#!/bin/sh
num=`awk -F'|' '{gsub(/\/|:/," ",$1);if(mktime($1)>systime()-900) print $0;}' /opt/phpdba/squid/var/logs/cache.log|grep "Failed to select source"|wc -l`
echo $num
echo `date`
if [ $num -gt 5 ]
  then
echo "restart"
/opt/phpdba/squid/sbin/squid -k reconfigure
fi
10/11/13

shell 监控mysql 锁表并报警

1、monitor_dbLocked

#!/bin/sh
mark=`date +%Y%m%d%H%M`
time_out=20
status=”ok”
check_status=”Locked”
send_mail=”off”
mysql_default_password=”xxxxxxxx”
host_name=`hostname`

declare -a mysqllist
declare -a mysql_exec_path
declare -a mysql_password
declare -a mysql_locked_port
declare -a mysql_show_port

function usage(){
cat << HELP

$0 — USAGE: sh $0 mysql_exec_path mysql_password mysql_socket

Example: sh $0 /opt/xxxx/mysql/bin/mysql xxxxxxx /tmp/socket
HELP
}
function script_arg_init(){
if [ "x$1" != "x0" ] ; then
mysql_exec_path=$1
else
mysql_exec_path=”/opt/xxxx/mysql/bin/mysql”
fi

if [ "x$2" != "x0" ];then
mysql_password=$2
else
mysql_password=”xxxxxx”
fi

if [ "x$3" != "x0" ];then
mysql_socket=$3
else
mysql_socket=”/tmy/mysql.sock”
fi

if [ "x$4" != "x" ];then
mysql_locked_port=$4
else
mysql_locked_port=3306
fi
}

for i in $mysqllist;do
for j in `echo $i|awk -F ‘:’ ‘{ k=1;while(k<=NF){print $k;k++}}’`;do

if [ -n "${j}" ] && ([ `echo $j|grep 'datadir='` ] || [ `echo $j|grep 'socket='` ] || [ `echo $j|grep 'port='` ]);then
#mysql_datastr=(`echo $j|awk -F “=” ‘{print $2}’`)
if [ `echo $j|grep 'datadir='` ];then
mysql_datadir=(`echo $j|awk -F “=” ‘{print $2}’`)
elif [ `echo $j|grep 'socket='` ];then
mysql_socket=(`echo $j|awk -F “=” ‘{print $2}’`)
elif [ `echo $j|grep 'port='` ];then
mysql_port=(`echo $j|awk -F “=” ‘{print $2}’`)
fi
fi
done
if [ "$mysql_port" == "3312" ];then
main 0 “xxxxxxxxx” $mysql_socket $mysql_port
else
main 0 $mysql_default_password $mysql_socket $mysql_port
fi
done

if [ "$send_mail" == "on" ] ;then
ip=`/bin/sh get_ip.sh|/usr/bin/head -n 1`
email_content=`cat log_mail/log_mail_$mark.txt`
sendEmail -s smtp.163.com -f xxxx@163.com -t xxxx@xxx.com -a log_mail/log_mail_$mark.txt -u “ALERT:$host_name[$ip] Locked Table $mysql_show_port ” -m “$email_content” -xu xxxx -xp xxxxxxx
echo “$mark send email”
fi

2、get_ip

#!/bin/sh
OS=`uname`
IO=””
case $OS in
Linux) IP=`/sbin/ifconfig  | grep ‘inet addr:’| grep -v -E ’127.0.0.1|192.168.’ | cut -d: -f2 | awk ‘{ print $1}’`;;
FreeBSD|OpenBSD) IP=`ifconfig  | grep -E ‘inet.[0-9]‘ | grep -v -E ’127.0.0.1|192.168.’ | awk ‘{ print $2}’` ;;
SunOS) IP=`ifconfig -a | grep inet | grep -v -E ’127.0.0.1|192.168.’ | awk ‘{ print $2} ‘` ;;
*) IP=”Unknown”;;
esac
echo “$IP”

3、sendEmail

sendEmail -f ADDRESS [options]

Required:
-f ADDRESS                from (sender) email address
* At least one recipient required via -t, -cc, or -bcc
* Message body required via -m, STDIN, or -o message-file=FILE

Common:
-t ADDRESS [ADDR ...]     to email address(es)
-u SUBJECT                message subject
-m MESSAGE                message body
-s SERVER[:PORT]          smtp mail relay, default is localhost:25

Optional:
-a   FILE [FILE ...]      file attachment(s)
-cc  ADDRESS [ADDR ...]   cc  email address(es)
-bcc ADDRESS [ADDR ...]   bcc email address(es)
-xu  USERNAME             username for SMTP authentication
-xp  PASSWORD             password for SMTP authentication
Paranormal:
-b BINDADDR[:PORT]        local host bind address
-l LOGFILE                log to the specified file
-v                        verbosity, use multiple times for greater effect
-q                        be quiet (i.e. no STDOUT output)
-o NAME=VALUE             advanced options, for details try: –help misc
-o message-content-type=<auto|text|html>
-o message-file=FILE         -o message-format=raw
-o message-header=HEADER     -o message-charset=CHARSET
-o reply-to=ADDRESS          -o timeout=SECONDS
-o username=USERNAME         -o password=PASSWORD
-o tls=<auto|yes|no>         -o fqdn=FQDN

Help:
–help                    the helpful overview you’re reading now
–help addressing         explain addressing and related options
–help message            explain message body input and related options
–help networking         explain -s, -b, etc
–help output             explain logging and other output options
–help misc               explain -o options, TLS, SMTP auth, and more

4、脚本下载

monitor_dblocked_v1.tar

09/24/13

shell脚本分享【监控链接访问情况】

#!/bin/sh
id=`curl -I –connect-timeout 20 http://blog.phpdba.com/post/110|awk ‘{ print $2;exit}’`
status=no
while [ true ]
do
sleep 10
if [ "${id}" != 200 ] && [ "$status" = ok ] ; then
echo ‘什么事都不做’
sleep 5
elif [ "${id}" != 200 ]; then
echo ‘发邮件通知’
echo ‘重启’
sleep 5
status=ok
elif [ "${id}" = 200 ] && [ "$status" = ok ]; then
echo ‘发邮件通知服务恢复’
sleep 5
status=no
elif [ "${id}" = 200 ]; then
echo ‘正常’
fi
id=`curl -I –connect-timeout 20 http://blog.phpdba.com/post/110|awk ‘{ print $2;exit}’`
done