sort -u 去除重复并排序
uniq 去除连续的重复行
例：
video.txt
[root@hadoop01 test-shell]# cat video.txt Allen:HK:119:1982 Allens:KL:445:5851 Boys:HK:123:1988 Allens:KL:445:5851 Allens:KL:445:5851 Allens:KL:445:5851
sort -u
[root@hadoop01 test-shell]# sort -u video.txt Allen:HK:119:1982 Allens:KL:445:5851 Boys:HK:123:1988
uniq
[root@hadoop01 test-shell]# uniq video.txt Allen:HK:119:1982 Allens:KL:445:5851 Boys:HK:123:1988 Allens:KL:445:5851