常用的脚本整合

发表于 更新于 分类于

有一部分命令在<**运维-系统常用的命令**>

监控Yarn任务运行情况

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#### vi exec.sh
#!/bin/sh

title="SparkStreaming任务监控"

# 获取正在运行的任务
name=(`sshpass -p "password" ssh root@ip "yarn application -list -appStates RUNNING|sed -n '3~1p'"|awk -F '\t' '{print $2}'|sed -e 's/[[:space:]]//g'`)

cd `dirname $0`

base=`pwd`

#获取name文件中的内容，去除空格
arr=(`cat ./name|sed -e 's/[[:space:]]//g'`)
dt=`date "+%Y-%m-%d_%H%M%S"`
log="./"$dt".stop"

for i in ${arr[@]}
do
     for j in ${name[@]}
     do
           [ "$i" = "$j" ] && flag="yes"
     done
     [ "$flag" != "yes" ] && echo $i >> $log
     flag="false"
done

#-e 判断文件是否存在
#cat $log|xargs echo 输出到一行
#'s/ /,/g' 以逗号分开

if [ -e $log ]
then
   z=`cat $log|xargs echo|sed -e 's/ /,/g'`
   cd $base && python $base/send_mail.py $title $z
   rm -rf $log
fi

#### vi name
jobname1
jobname2
jobname3

删除日志文件脚本

1
2
3
# 使用find
find /root/test/ -type f -newermt "Aug 19" -delete # 删除8月19日的文件
find /root/ -type f -mtime +1 -delete # 删除一天前的文件

增删后缀

1
2
3
4
5
6
7
# 增加后缀
for i in *
   do mv $i $i".bak"
done

# 删除后缀
rename 's/\.xml$//' *.xml

crontab定时脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
crontab -l # 列出定时任务
crontab -e # 编辑定时任务

# 定时任务格式
* * * * * sh /root/os.sh > /root/os.log
第一个*为分(0-59)
第二个*为时(0-23)
第三个*为日(1-31)
第四个*为月(1-12)
第五个*为周(1-7)或(0-6),周日=0或7

n-m表示到n到m之间
*/n表示每n段时间
n,m表示第n和第m

> /dev/null 2>&1
>代表重定向
/dev/null代表空设备文件
2>&1表示标准错误输出重定向等同于标准输出

发送邮件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
#### vi send_mail.py
# 使用python来进行发送邮件
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import sys 
import smtplib
from email.mime.text import MIMEText
from email.header import Header
from email.mime.multipart import MIMEMultipart
 
# 第三方 SMTP 服务
mail_host="smtp.sina.com"  #设置服务器
mail_user="username"    #用户名
mail_pass="password"   #口令 
 
sender = '发送人'
receivers = ['接收邮箱1','接收邮箱2']

def send_file(title,file):
    message = MIMEMultipart()
    message['From'] = Header("发送人")
    subject = title
    # 标题
    message['Subject'] = Header(subject, 'utf-8')

    # 信息
    message.attach(MIMEText(title, 'plain', 'utf-8')) 

    # 附件
    att1 = MIMEText(open(file, 'rb').read(), 'base64', 'utf-8')
    att1["Content-Type"] = 'application/octet-stream'
    att1["Content-Disposition"] = 'attachment; filename="att.log"'
    message.attach(att1)
    
    smtpObj = smtplib.SMTP() 
    try:
        smtpObj.connect(mail_host, 25)    # 25 为 SMTP 端口号
        smtpObj.login(mail_user,mail_pass)  
        smtpObj.sendmail(sender, receivers, message.as_string())
        print "邮件发送成功"
    except smtplib.SMTPException:
        print "Error: 无法发送邮件"

def send_html(title,msg):
    message = MIMEMultipart()
    message['From'] = Header("发送人")
    subject = title
    message['Subject'] = Header(subject, 'utf-8')

    # HTML信息
    message.attach(MIMEText("<font color='red'>"+msg+"</font>", 'html', 'utf-8')) 
    
    smtpObj = smtplib.SMTP() 
    try:
        smtpObj.connect(mail_host, 25)    # 25 为 SMTP 端口号
        smtpObj.login(mail_user,mail_pass)  
        smtpObj.sendmail(sender, receivers, message.as_string())
        print "邮件发送成功"
    except smtplib.SMTPException:
        print "Error: 无法发送邮件"

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print "usage: python send_mail.py title msg"
        sys.exit(1)
    
    title = sys.argv[1]
    msg = sys.argv[2]
    send_html(title,msg)

Flink任务重启脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
#### vi getConfigChk.sh

#!/bin/bash
# 获取配置文件中的ck目录
# fs.backend.dir=/job/flink/ck/job1
dir=`cat $1 | grep "fs.backend.dir" | awk -F "=" '{print $2}'`

# 获取HDFS上最新的CK文件
path=`hdfs dfs -ls -R ${dir} | grep _metadata | sort -k 6,7 -r | sed -n 1p | awk '{print $8}'`

if [[ ! "$path" =~ ^hdfs* ]]; then
  path="hdfs://$path"
fi

echo $path

#### vi exec.sh

#!/bin/bash

BASE="/root/xiashuai/real_job/data_sync"
config="conf.properties"

# flink 作业样例
chk=`getConfigChk $BASE/$config`
jarName='real_data_clean-full.jar'
className='com.example.SyncToHiveMain'
appName='DataSync'
parallelism=1
slot=1
noRecover=false
for arg in $@; do
  if [ "$arg" = "-init" ]; then
   noRecover=true
  fi
done
if [ "$chk" = "hdfs://" -o $noRecover = true ]; then
 echo "##########################################"
 echo "#              初始化运行                #"
 echo "##########################################"
 flink run -m yarn-cluster \
 -ynm $appName \
 -p $parallelism -ys $slot -ytm 2048 \
 -c $className \
 $BASE/../$jarName \
 $BASE/$config
else
 echo "##########################################"
 echo "# 加载 checkpoint => $chk                 "
 echo "##########################################"
 flink run -m yarn-cluster \
 -ynm $appName"-recover" \
 -p $parallelism -ys $slot -ytm 2048 \
 -c $className \
 -s $chk \
 $BASE/../$jarName \
 $BASE/$config
fi