awk脚本结合shell使用

需求：判断hadoop用户是否存在
**************************************************
#!/bin/bash
result=`awk -F ":" '{if($1=="hadoop") print $1}' /etc/passwd`
if [ -n result ]
then
echo "the $result is exists"
fi
运行：
./myawk.sh
**************************************************

如何判断NameNode进程是否启动
jps|awk '{if($2=="NameNode")print $2}'

**************************************************

用户名=>用户路径
vi myawk01.awk
BEGIN{
print "$$$$$$$$$$$$$"
FS=":"
OFS="=>"
}
{
print $1,$6

}
END{
print "@@@@@@@@@@@"
}

执行：
awk -f ./myawk01.awk /etc/passwd
**************************************************

#给输出信息加上表头( 表示空格   表示换行)
awk -F ":" 'BEGIN {print "name shell --------------------------------"}
{print $1" "$6}' /etc/passwd

----------------------------------------------------------
#给输出信息加上表头和末尾
awk -F : 'BEGIN {print "name shell --------------------------------"}
{print $1" "$6} END {print "end-of-report"}' /etc/passwd

awk -F ":" 'BEGIN {print"--BEGIN--"}$1=="root" { print $1} END{print"----END------"}' /etc/passwd

awk -F ":" 'BEGIN {print"--BEGIN--"}
{if( $1=="root") print $1}
END{print"----END------"}' /etc/passwd

-------------------------------------------------------------------

----------------------------------------------------------
正则表达式匹配格式
1)$n~正则表达式
2)if($n~正则表示式) print $0
----------------------------------------------------------

#打印以root开头的行
awk -F ":" '$0 ~ /^root/' /etc/passwd
#打印以ntp开头并以login结尾的行
awk -F : '$0 ~ /^ntp.*login$/ ' /etc/passwd
#打印包含nobody的行
awk -F : '{if($1~/nobody/) print $0}' /etc/passwd
#打印包含nobody或Nobody的行
awk -F : '{if($1~/[nN]obody/) print $0}' /etc/passwd
#打印n/N开头的行,
awk -F ":" 'BEGIN{print "userName"}{if($1~/^(n|N).*$/) print$0}' /etc/passwd
----------------------------------------------------------

#精确匹配|
#打印名字为root的用户在/etc/passwd文件中的记录
awk -F : '$1=="root" {print $0}' /etc/passwd
awk -F : '{ if($1=="root") print $0 }' /etc/passwd
----------------------------------------------------------
在awk中使用条件操作符
< 小于
>= 大于等于
<= 小于等于
== 等于
!= 不等于
~ 匹配正则表达式
!~ 不匹配正则表达式
----------------------------------------------------------
#或/与运算
awk -F : '$1~/(root|shawn)/ {print $0}' /etc/passwd
awk -F : '$1~/root/ || $1 ~/shawn/ {print $0}' /etc/passwd
awk -F : '{if($1~/root/ || $1 ~/shawn/) print $0}' /etc/passwd
awk -F : '$6~/home/ && $6~/shawn/ {print $0}' /etc/passwd
awk -F : '{if($1~/root/ || ($1~/shawn/)) print $0}' /etc/passwd
awk -F : '{if($1~/root/ || ($1~/shawn/)){print $0} else {print "other:"$0}}' /etc/passwd
awk -F : '{if($0!~/^ntp.*nologin$/) print $0}' /etc/passwd
----------------------------------------------------------
#内置变量
ARGC 命令行参数个数
ARGV 命令行参数排列
FS 设置输入域分隔符，与-F同 *****
NF 记录域的个数 *****
NR 已读的记录数 *****
OFS 输出域分隔符 *****
ORS 输出记录分隔符
RS 控制记录分隔符
----------------------------------------------------------
#FS的用法，注意FS要加双引号
awk 'BEGIN {FS=":"}{print $1}' /etc/passwd

#内置变量用法
BEGIN{
FS=":"
}
{
print $1,$6
}
END{
print "ddddd"
}
----------------
BEGIN {
FS=":"
OFS="="
}
{
print $1,$6
}
----------------
BEGIN {
FS=":"
OFS="="
}
{
print "Row " NR ":" $1,$6
}
----------------
将
BEGIN {
FS=":"
OFS="="
}
{
print "Row " NR ":" $1,$6,$NF
}
END{
print "当前共有" NF "列"
}
awk -f myawk.awk /etc/passwd

----------------------------------------------------------
#求和
awk -F " " 'BEGIN { SUM=0 } { SUM = SUM + $5 } END{ print "总额为:" SUM}' ./ employee.txt

封装到脚本方式：
BEGIN{
FS=" "
sum=0
}
{
sum=sum+$5
}
END{
print sum
}

将执行awk命令封装到shell脚本：
vi myawk03_sh.sh
#!/bin/bash
total= awk -f ./myawk03.awk ./employee.txt
echo "$total"

运行：
./myawk03_sh.sh

-------------------------------------------------------
#求平均值一
BEGIN {
FS=":"
SUM=0
}
{
SUM = SUM + $5
}
END{
print "平均值为:" SUM / NR
}
#求平均值二
BEGIN{
FS=" "
SUM=0
NUM=0
}
{
if($5 ~ /[0-9]+/){
SUM = SUM + $5
NUM = NUM + 1
}
}
END{
print "平均值为:" SUM / NUM
}

#求最大值
BEGIN {
FS=" "
MAX=0
}
{
if ($5 > MAX){
MAX = $5
}
}
END{
printf("最大值为：%d ", MAX)
}

awk -f test4.awk employee.txt
----------------------------------------------------------
#数组应用一
BEGIN{
FS = " "
}
{
line[NR] = $0
}
END{
i = 1
while (i <=NR) {
print line[i]
i = i + 1
}
}
-----------------------------------------------------------
#数组应用二
BEGIN{
FS = ":"
}
{
line[NR] = $2
}
END{
for(i=1;i<=NR;i=i+1){
print line[i]
}
}

===================================================
1、统计各科修课人数
vi countstu.txt
xiaoA math 90
xiaoB chiese 23
xiaoD English 45
xiaoC jichu 76
xiaoM math 23
xiaoY chiese 12
xiaoK math 56
xiaoY math 23
xiaoQ math 34
xiaoY jichu 12
xiaoP jichu 100

vi countstu.awk

{
Number[$2]++
}
END{
for(course in Number)
printf("%10s %d ", course, Number[course])
}

运行：
awk -f ./countstu.awk countstu.txt
-------------------------------------------------------
2、awk实现wordcount案例
vi words.txt
wo shi tiancai
wo bu shi tiancai que ding ma

运行方式一：
vi wordcount.awk
{
for (i = 1; i<=NF; i++)
freq[$i]++
}
END{
for (word in freq)
printf "%s%d ",word,freq[word]
}
运行:
awk -f wordcount.awk words.txt

运行方式二：
vi wordcount_awk.sh
#!/bin/sh
awk -F " " '{
for (i = 1; i<=NF; i++)
freq[$i]++
}
END{
for (word in freq)
printf "%s%d ",word,freq[word]
}' $1

chmod u+x wordcount_awk.sh
./wordcount_awk.sh words.txt

作业2：要求：将案例1和案例2使用shell脚本封装并且执行
注意：我已经删除一些字符！！！！

案例3：查看NameNode服务是否启动，如果没有启动就启动NameNode
process=`jps|grep -v Jps|awk -F " " '{if($2=="NameNode")print $2}'`
if [ -z $process ]
then
echo "the process is not running,start to running Namenode process..."
path=$(which hadoop-daemon.sh)
$path start namenode
echo "the NameNode process is running..."
else
echo "the NameNode process is already run"
fi

执行：
/bin/bash ./myawk05.sh

案例4：重写一个hdfs的重启动文件
#!/bin/bash
#get all process
all=`jps | grep -v Jps | awk -F " " '{print $1}'`
if [ -n "$all" ] ; then
echo 'found hadoop process is running...'
jps | grep -v Jps | awk -F ' ' '{print "shutdown proccess > " $2}'
kill -9 `jps | grep -v Jps | awk -F ' ' '{print $1}'`
echo 'restart process that you need...'
if [ -f "$HADOOP_HOME/sbin/hadoop-daemon.sh" ]; then
#/opt/bigdata/hadoop-2.7.1/sbin/start-dfs.sh
$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode
$HADOOP_HOME/sbin/hadoop-daemon.sh start datanode
$HADOOP_HOME/sbin/hadoop-daemon.sh start secondarynamenode
fi
else
echo 'not found hadoop process is running...'
echo 'start process that you need...'
if [ -f "$HADOOP_HOME/sbin/hadoop-daemon.sh" ]; then
#/opt/bigdata/hadoop-2.7.1/sbin/start-dfs.sh
$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode
$HADOOP_HOME/sbin/hadoop-daemon.sh start datanode
$HADOOP_HOME/sbin/hadoop-daemon.sh start secondarynamenode
fi
fi

----------------------------------------------------------

如何将awk脚本结合shell使用
需求：判断hadoop用户是否存在
**************************************************
#!/bin/bash
result=`awk -F ":" '{if($1=="hadoop") print $1}' /etc/passwd`
if [ -n result ]
then
echo "the $result is exists"
fi
运行：
./myawk.sh