Linux与Shell实战之Nginx日志分析

拉勾名企直推营 拉勾名企直推营-4期
50 
url_summary() { awk '{print $7}' nginx.log | sed -e 's#/topics/[0-9]*/replies/[0-9]*/edit#/topics/int/replies/int/edit#g' | sed -e 's#/_img/uploads/photo/2018/[0-9a-zA-Z]*-.*#/_img/uploads/photo/2018/id.png!large#g'| sed -e 's#/topics/[0-9]*#/topics/int#g'|sort -nr -t'/'|uniq -c|sort -rn |head -10; }

课后作业
课后作业1174×198 8.76 KB

51

课后作业：

url_summary(){ awk '{print $7}' nginx.log|  sed 's/[0-9]\+/int/g' |sed -r 's/[^/]*\.jpg/id.jpg/g' |sed -r 's/[^/]*\.jpeg/id.jpeg/g' |sed -r 's/[^/]*\.png/id.png/g'|sed -r 's/[^/]*\.gif/id.gif/g'|sort|uniq -c |sort -nr|head -10; }

image

52

课后作业：

url_summary(){ awk '{print $7}' nginx.log | sed -r 's/\/topics\/[0-9]+/\/topics\/int/g;s/\/replies\/[0-9]+/\/replies\/int/g;s/\/_img\/uploads\/photo\/[0-9]+\/[0-9a-zA-Z-]+\./\/_img\/uploads\/photo\/id\./g' | sort | uniq -c | sort -nr | head -10; }

image
image943×237 6.9 KB

53

课后作业：

url_summary(){  awk '{print$7}' nginx.log | sed -r  's#\/[0-9]+#/int#' | sed -r 's#\/[0-9]+#/int#' | sed -r 's#\/[A-Za-z0-9]+\.#/id.#' | sort| uniq -c | sort -nr | head -10; }

image

54 
url_summary(){ awk '{print $7}' nginx.log | sed -r 's|/topics/[0-9]+|/topics/int|g;s|/replies/[0-9]+|/replies/int|g;s|/[0-9a-zA-Z-]+\.|/id.|g' | sort | uniq -c | sort -rn | head -10 ;}
55 
url_summary()
{
    awk '{print $7}' nginx.log | sed -r 's/\/[a-z0-9-]+\./\/id\./;s/\/[0-9]+/\/int/;s/\/[0-9]+\//\/int\//' | sort | uniq -c | sort -n -r | head -10;
}

image
image822×327 11.1 KB

57

url_summary (){ awk ‘{print $7}’ nginx.log | sed -r ‘s/[A-Za-z0-9-]+.png/id.png/g; s/[A-Za-z0-9-]+.jpg/id.jpg/g;s/[0-9]+/int/g’ | sort | uniq -c | sort -nr | head -10 ; }
image

58

…
awk ‘{print $7}’ nginx.log | sed -e “s//[0-9]+///int//g” -e “s//[0-9]+$//int/g” -e “s/(/[^/]*)(.png|.jpg|.gif)//id\2/g” | sort | uniq -c | sort -nr | head -10
…
image

59 
find_error_log() {
awk '$9~/404|500/{print $0}' nginx.log
}
60

课后作业：
url_summary(){ awk ‘{print $7}’ nginx.log |sed ‘s/topics/[0-9]/topics/int/g;s/replies/[0-9]/replies/int/g;s/[0-9a-z-].png/id.png/g;s/[0-9a-z-].jpg/id.jpg/g;s/[0-9a-z-].gif/id.gif/g;s/[0-9a-z-].jpeg/id.jpeg/g;s//[0-9]///int//g;s//[0-9][0-9]$//int/g;s/page=[0-9][0-9]*/page=int/g’| sort |uniq -c |sort -nr | head -10; }
image

61

url聚类 url_summary

 url_summary()
 { 
    awk '{print $7}' nginx.log | sed -E 's/[A-Za-z0-9\-]+\.png/id\.png/g;s/[A-Za-z0-9\-]+\.jpg/id\.jpg/g;s/[A-Za-z0-9\-]+\.jif/id\.gif/g;s/\/[0-9]+/\/int/g' | sort | uniq -c | sort -rn | head -10

}

image

62

find_error_log ()
{
cat /tmp/nginx.log | grep --color=auto -E ‘\s(404|500)\s’ | head -n 3
}

63 

online_user(){ ps -ef|grep 'sshd.*priv'|grep -v grep|awk '{print $9}'|sort|uniq -c|sort -nr;}
64 
find_error_log() { awk '$9~/404|500/{print $0}' nginx.log;}

find_before_and_after(){ grep -E -C1 '\b500\b' nginx.log;}

find_top_3(){ awk '{print $1}' nginx.log |sort |uniq -c|sort -rn|head -3;}

url_avg_time(){ awk '$7=="/topics"{print $(NF-1)}' nginx.log | awk '{sum+=$1}END{print sum/NR}';}
65 
url_summary () 
{ 
    awk '{print $7}' nginx.log | sed -E 's#\/[0-9]+\/#\/int\/#g;s#\/[0-9]+#\/int#g;s#\/[0-9a-z\-]+\.#\/png\.#g;s#\?.*##g' | sort | uniq -c | sort -rn | less
}

Screenshot from 2020-10-23 18-06-58
Screenshot from 2020-10-23 18-06-581153×597 73.8 KB

66 
 find_top_3(){ awk '{print $1}'| sort | uniq -c | sort -nr | hesd -n 3; }
67 
url_avg_time(){
awk '$7=="/topics"{print $(NF-1)}' nginx.log | awk '{sum+=$1}END{print sum/NR}'
}
68

作业

url_summary () 
{ 
    less nginx.log | awk '{print $7}' | sed -e 's#/topics/[0-9]*#/topics/int#' | sed 's#/2018/[a-Z0-9-]*.png#/id.png#' | sed 's#/topics/[0-9]*/replies/[0-9]/edit#/topics/int/replies/int/edit#' | sort | uniq -c | sort -nr | head -10
}
# 运行结果
url_summary 

    584 /cable
    237 /topics/int
     94 /
     41 /topics/int/show_wechat
     37 /topics/int?locale=zh-CN
     33 /topics/int?locale=en
     28 /topics/int?locale=zh-TW
     18 /uploads/photo/id.png!large
     15 /_img/uploads/photo/id.png!large
     12 /topics/int?order_by=created_at&
69 
url_summary() { awk '{print $7}' nginx.log |sed -E 's/\/[0-9]+/\/int/g'|sed -E 's/[A-Za-z0-9\-]+\.jpg/int\.jpg/g'|sed -E 's/[0-9a-zA-Z\-]+\.png/int\.png/g'|sort|uniq -c | sort -nr|head -n 10; }
70

find_error_log(){

grep -E ‘\b404\b|\b500\b’ nginx.log

}

find_before_and_after(){

grep -r -1 ‘\b500\b’ nginx.log

}

find_top_3(){

awk ‘{print $1}’ nginx.log| sort | uniq -c | sort -n -r | head -n 3

}

url_avg_time(){

grep ‘/topics’ nginx.log |awk ‘{print $(NF-1)}’|awk ‘{sum+ = $1}END{print sum/NR}’

}

url_summary () {
awk -F ’ ’ ‘{print $7}’ nginx.log | sed -e ‘s#/topics/[0-9]/replies/[0-9]/edit#/topics/int/replies/int/edit#g’|sed -e’s#/_img/uploads/photo/2018/[0-9a-zA-Z]-.#/_img/uploads/photo/2018/id.png!large#g’|sed -e ‘s#/topics/[0-9]*#/topics/int#g’|sort -nr -t’/’|uniq -c|sort -rn |head -10
}