【直播】三剑客实战Nginx日志分析

1442678679 · 2020 年7 月 4 日 01:06

‘’’
编写一个函数 find_error_log()找出log中的404 500的报错
less nginx.log | grep -E ‘(" 404|" 500)’
less nginx.log | awk ‘$9~/404|500/’

找出访问量最高的ip，统计分析，取出top3的ip和数量，打印出来
less nginx.log | awk ‘{print $1}’ | sort | uniq -c | sort -nr | head -n 3

找出首页访问 / 的平均响应时间，首页访问示例，get后面的地址是/，每次访问的响应时间是倒数第二个字段
less nginx.log | awk ‘$7 == “/”’ | awk ‘{ sum += $(NF-1)} END {print sum/NR}’

课后作业：
找出访问量最高的页面地址借助于sed的统计分析
/topics/16689/replies/124751/edit 把数字替换为 /topics/int/replies/int/edit
可以使用（less nginx.log | awk ‘{print $7}’ | sed -E ‘s/([ 0-9]+)/int/g’）
/_img/uploads/photo/2018/c54755ee-6bfd-489a-8a39-81a1d7551cbd.png!large 变成 /_img
/uploads/photo/2018/id.png!large
这个不知道为啥这样匹配不行：
大致思路就是对于.png替换成.png，觉得没什么不妥

输出
url pattern对应的请求数量
取出top 10请求量的url pattern
less nginx.log | awk ‘{print $7}’ | sed -E ‘s/([0-9]+)/int/g’| sort | uniq -c | sort -nr | head -n 10
不知道这是三个题还是一个题
‘’’

summer · 2020 年7 月 4 日 01:36

课后作业

less nginx.log | awk '{print $7}' | sed 's#?.*##g' |sed 's#/topics/[0-9]*#/topics/int#g' | sed 's#/replies/[0-9]*#/replies/int#'|sed 's#/avatar/[0-9]*/#/avatar/int/#'| sed 's#/2017/[0-9a-z-]*.#/2017/id.#g' | sed 's#/2018/[0-9a-z-]*.#/2018/id.#g' |sed 's#tar/int/[0-9a-z-]*.#tar/int/id.#g'| sed 's#tar/[0-9]*.jpg#tar/id.jpg#g'|sed 's#/[0-9]*\s#/int#g'|sed 's#/2014/[0-9a-z-]*.#4/id.#g'|sed 's#/2016/[0-9a-z-]*.#/2016/id.#g'|sed 's#/2015/[0-9a-z-]*.#/2015/id.#' |sort |uniq -c |sort -rn|head -10

meekcat · 2020 年7 月 4 日 06:41

20200702课后作业

url_summary（） {
    less nginx.log | awk '{print $7}' | sed -e 's/[a-z0-9]\{8\}\(-[a-z0-9]\{4\}\)\{3\}-[a-z0-9]\{12\}/id/g' | sed -e 's/\(topics\/\)[0-9]*\(\/replies\/\)[0-9]*\(\/edit\)/\1int\2int\3/g' | sed -e 's/\(topics\/\)[0-9]*/\1int/g' | sort | uniq -c | sort -nr | head -10
}

Baozi · 2020 年7 月 4 日 08:54

url_summary(){
less nginx.log | awk '{print $7}' | sed "s#/topics/[0-9]*#/topics/int#g" | sed "s#/replies/[0-9]*#/replies/int#g" | sed "s#/[a-z0-9].*.png#/id.png#g" | sort | uniq -c | sort -nr | head -10
}

38364841 · 2020 年7 月 4 日 11:07

url_summary(){
less nginx.log | awk ‘{print$7}’ | sort | sed -e ‘s#/topics/[0-9]/replies/[0-9]/#/topics/int/replies/int/#g’ -e ‘s#/_img/uploads/photo/2018/.*.png!large#/_img/uploads/photo/2018/id.png!large#g’ | uniq -c | sort -nr | head -10
}

624354119 · 2020 年7 月 4 日 11:14

url_summary(){
less nginx.log | awk '{print $7}' | sed -e 's/^\(\/topics\/\)[0-9]*\(\/.*$\)/\1\int\2/; s/^\(\/topics\/\)[0-9]*$/\1int/; s/^\(\/topics\/\)[0-9]*(\?).*$/\1\int/p; s/\(\/replies\/\)[0-9]*\(\/.*$\)/\1int\2/; s#^\(\/_img.*\/photo\/20[1-2][0-9]\/\).*\(\.png.*$\)#\1id\2#' | sort | uniq -c | sort -nr | head -10;
}

Qili-n_c · 2020 年7 月 4 日 11:17

url_summary(){
less nginx.log | awk '{print $7}' | sed 's#s/[0-9]*#s/int#g' | sed 's#/[a-z0-9]*.png#/id.png#g'| sed 's#/[0-9a-z]*.jpg#/id.jpg#g'| sort | uniq -c | sort -nr | head -10
}

zeroky · 2020 年7 月 4 日 15:02

课后作业：

 url_summary(){
    less nginx.log | awk '{print $7}' | sed -E -e 's#/[0-9]+#/int#g' -e 's#/[a-z0-9-]*\.jpe?#/id.jp#g' -e 's#/[0-9a-z-]*\.png#/id.png#g' -e 's#/[0-9a-z-]*\.gif#/id.gif#g' -e 's#\!.*#!xxx#g' -e 's#=.*#=xxx#g'| sort | uniq -c | sort -nr | head -10
    }

591237964 · 2020 年7 月 4 日 16:20

url_summary(){
less nginx.log | awk '{print $7}'| sed -r 's/([0-9]+)/id/g'|sed -r 's/^\/_img\/.*$/\/_img/g'|sed -r 's/^\/uploads.*large$/\/uploads/g'|sed -r 's/^\/uploads.*/\/uploads/g'|sort -n | uniq -c |sort -nr |head -n 10
}

232039123 · 2020 年7 月 5 日 03:07

课后作业

less nginx.log | awk '{print $7}' | \
sed -r 's#topics\/[0-9]+#topics\/int#g' | \
sed -r 's#replies\/[0-9]+#replies\/int#g' | \
sed -r 's#\/_img\/uploads\/photo\/2018\/.*\.png#\/_img\/uploads\/photo\/2018\/id\id\.png#g' | \
sed -r 's#\/_img\/uploads\/photo\/2017\/.*\.png#\/_img\/uploads\/photo\/2017\/id\.png#g' | \
sort | uniq -c | sort -nr | awk 'NR<11'

zengxuejie · 2020 年7 月 5 日 04:11

课后作业：

url_summary(){
less nginx.log | awk '{print $7}' | \
sed 's#s/[0-9]*#s/int#g' | \
sed 's/\([0-9][0-9]*\/\).*\(.jpg\)/\1id\2/g' | \
sed 's/\([0-9][0-9]*\/\).*\(.png\)/\1id\2/g' | \
sed 's/\([0-9][0-9]*\/\).*\(.gif\)/\1id\2/g' | \
sed 's/\([0-9][0-9]*\/\).*\(.jpeg\)/\1id\2/g' | \
sort | uniq -c | sort -nr | head -10
}

1984297068 · 2020 年7 月 5 日 04:21

url_summary(){
less nginx.log | awk '{print $7}' | sed -r 's/topics\/([0-9])+/topics\/int/g' | sed -r 's/_img\/uploads\/photo\/2018\/.*(.png)/_img\/uploads\/photo\/2018\/id\.png/g' | sed -r 's/(topic\/)[0-9]*/(topic\/)int/g' | sort  | uniq -c | sort -nr | head -10
}

作业0702

Mr.Wang · 2020 年7 月 5 日 04:36

less nginx.log | awk '{print $7}' | sed -E 's#([0-9]+)#int#g' | sed -E 's#(photo/int/[0-9a-z].*)#photo/int/_img#g' | sed -E 's#(avatar/int/[0-9a-z].*)#avatar/int/_img#g' | sed -E 's#(avatar/[0-9a-z].*)#avatar/_img#g' | sort | uniq -c | sort -rn | head -10

leesom · 2020 年7 月 5 日 04:50

url_summary(){
less nginx.log | awk '{print $7}' | sed -r 's#topics\/[0-9]+#topics\/int#g' | sed -r 's#replies\/[0-9]+#replies\/int#g'|sed -r 's#_img\/uploads\/photo\/2017\/.*\.png+#_img\/uploads\/photo\/2017\/id\.png#g' |sed -r 's#_img\/uploads\/photo\/2018\/.*\.png+#_img\/uploads\/photo\/2018\/id\.png#g'| sed -r 's#uploads\/photo\/2017\/.*\.png+#uploads\/photo\/2017\/id\.png#g'| sed -r 's#photo\/2015\/.*\.png+#photo\/2015\/id.png#g'| sed -r 's#photo\/2014\/.*\.png+#photo\/2014\/id.png#g' |sort |uniq -c|sort -nr|head -10
}

995671805 · 2020 年7 月 5 日 05:12

课后作业 url_summary

seveniruby:

url_summary(){

awk '{print $7}' nginx.log |   sed 's#\?.*##g' |   sed 's#%21#\!#g' |   sed -r 's#/(topics|replies|avatar)/[0-9]+#/\1/int#g' |   sed -r 's#(.+)/[^/].*(png|jpg|gif|jpeg.*)#\1/id\.\2#g' |   sed -r 's#^/[^/]+/(topics|following|replies|columns|favorites|reward|followers)$#/username/\1#g' |   sed -r 's#\.(tar\.tgz|tar\.gz|tar|gz|zip|rar)$#\.compress#g' |   sort | uniq -c | sort -nr | head -10 |less

}

1547408844 · 2020 年7 月 5 日 05:24

url_summary(){
awk '{print $7}' nginx.log| sed -e 's/^\(\/topics\/\)[0-9]*\(\/.*$\)/\1\int\2/; s/^\(\/topics\/\)[0-9]*$/\1int/; s/^\(\/topics\/\)[0-9]*(\?).*$/\1\int/p; s/\(\/replies\/\)[0-9]*\(\/.*$\)/\1int\2/; s#^\(\/_img.*\/photo\/20[1-2][0-9]\/\).*\(\.png.*$\)#\1id\2#' | sort | uniq -c | sort -nr | head -10;
}

yangyan · 2020 年7 月 5 日 05:32

课后作业：
‘’‘
url_summary() {

less nginx.log | awk '{print $7}' | sed -E 's#([0-9]+)#int#g' | sed -E 's#(int/.*.png)#id.png#g' | sort | uniq -c | sort -gr | head -10

}
’‘’

773695355 · 2020 年7 月 5 日 05:35

url_symmary(){
cat nginx.log | sed -e ‘s/"/topics/16689/replies/124751/edit"/"/topics/int/replies/int/edit"/g’ |sed -e ‘s/"/_img/uploads/photo/2018/c54755ee-6bfd-489a-8a39-81a1d755cbd.png!large"/"/_img/uploads/photo/2018/id.png!large"/g’|sed -e ‘s"/topics/9497/"/"/topics/int"/g’
cat nginx.log | sed -e ‘s/"/topics/16689/replies/124751/edit"/"/topics/int/replies/int/edit"/g’ |sed -e ‘s/"/_img/uploads/photo/2018/c54755ee-6bfd-489a-8a39-81a1d755cbd.png!large"/"/_img/uploads/photo/2018/id.png!large"/g’|sed -e ‘s"/topics/9497/"/"/topics/int"/g’|awk ‘print{$7}’|sort -n|uniq -c|sort -rn|head -10
}
手机上传，请忽略格式，另外感觉题目不太清晰，第一次看题以为是直接替换，第二次看题看到有要求，不上传了，手机编辑太难了，awk里面替换前加上正则表达式

giorgiaWray · 2020 年7 月 5 日 05:56

less nginx.log | awk '{print $7}' | sed -r 's#(topics|replies)\/[0-9]+#\1/int#g' |sed -r 's#(_img\/uploads\/photo|uploads\/photo|photo|uploads\/user\/avatar)\/(2017|2018|2016|2015|2014)\/.*\.(png|jpg|gif)+#\1\/\2\/id\.\3#g' |sed -r 's#/(avatar)/[0-9]+#/\1/int#g'|sed -r 's#int\/.*\.(png|jpg|gif)+#int\/id\.\1#g'|sort |uniq -c|sort -nr|head -10

614887736 · 2020 年7 月 5 日 05:57

url_summary(){
less nginx.log |sed -e 's#/[0-9]+#/int#g' -e 's#/[a-z0-9-]*\.jpe?#/id.jp#g' -e 's#/[0-9a-z-]*\.png#/id.png#g' -e 's#/[0-9a-z-]*\.gif#/id.gif#g' -e 's#\!.*#!xxx#g' -e 's#=.*#=xxx#g' | awk ‘{print $1, $2}’ | uniq -c | sort -nr | head -n 10
}