Newer
Older
#!/usr/bin/env python3
import os
import sys
from dateutil import parser
import json, logging
from flask import Flask, request
from rocketchat_API.rocketchat import RocketChat
from requests import sessions
listenPort = 29120
listenIP = '::1'
botUser = '{{ prometheus_alertmanager.rocketchatbot.user }}'
botPass = '{{ prometheus_alertmanager.rocketchatbot.pass }}'
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
botChatURL = '{{ prometheus_alertmanager.rocketchatbot.url }}'
# ignore fields while printing labels
label_ignore_fields = ['ansible_group_ungrouped']
icons = {
'firing': ':warning:',
'resolved': ':white_check_mark:'
}
'''
Example attachment:
message.attachment = [{
"color": "#ff0000",
"text": "Yay for gruggy!",
"ts": "2016-12-09T16:53:06.761Z",
"thumb_url": "data:image/gif;base64,...", # add thumb image as base64 if needed
"message_link": "https://google.com",
"collapsed": false,
"author_name": "Bradley Hilton",
"author_link": "https://rocket.chat/",
"author_icon": "https://avatars.githubusercontent.com/u/850391?v=3",
"title": "Attachment Example",
"title_link": "https://youtube.com",
"title_link_download": true,
"image_url": "http://res.guggy.com/logo_128.png",
"audio_url": "http://www.w3schools.com/tags/horse.mp3",
"video_url": "http://www.w3schools.com/tags/movie.mp4",
"fields": [{
"short": true,
"title": "Test",
"value": "Testing out something or other"
},{
"short": true,
"title": "Another Test",
"value": "[Link](https://google.com/) something and this and that."
}]
}]
Example alert json:
[{
"receiver": "default",
"status": "resolved",
"alerts": [{
"status": "resolved",
"labels": {
"alertname": "ProbeFailed",
"ansible_group_ungrouped": "1",
"instance": "monitoring.cccv.de",
"job": "blackbox",
"module": "http_2xx",
"scraper": "monitoring.cccv.de",
"severity": "critical",
"target": "https://shells.darmstadt.ccc.de/~psy/alerttest.html"
},
"annotations": {
"description": "monitoring.cccv.de probe for https://shells.darmstadt.ccc.de/~psy/alerttest.html failed",
"title": "monitoring.cccv.de: target https://shells.darmstadt.ccc.de/~psy/alerttest.html failed"
},
"startsAt": "2021-01-04T15:36:30.85487796Z",
"endsAt": "2021-01-04T17:18:30.85487796Z",
"generatorURL": "http://monitoring:29090/graph?g0.expr=probe_success%7Binstance%3D%22monitoring.cccv.de%22%2Cjob%3D%22blackbox%22%7D+%3D%3D+0+or+absent%28probe_success%7Binstance%3D%22monitoring.cccv.de%22%2Cjob%3D%22blackbox%22%7D%29&g0.tab=1"
}],
"groupLabels": {
"alertname": "ProbeFailed"
},
"commonLabels": {
"alertname": "ProbeFailed",
"ansible_group_ungrouped": "1",
"instance": "monitoring.cccv.de",
"job": "blackbox",
"module": "http_2xx",
"scraper": "monitoring.cccv.de",
"severity": "critical",
"target": "https://shells.darmstadt.ccc.de/~psy/alerttest.html"
},
"commonAnnotations": {
"description": "monitoring.cccv.de probe for https://shells.darmstadt.ccc.de/~psy/alerttest.html failed",
"title": "monitoring.cccv.de: target https://shells.darmstadt.ccc.de/~psy/alerttest.html failed"
},
"externalURL": "http://monitoring:29093",
"version": "4",
"groupKey": "{}:{alertname=\"ProbeFailed\"}"
}]
'''
app = Flask(__name__)
app.secret_key = os.urandom(128)
api_session = sessions.Session()
api = RocketChat(botUser, botPass, server_url=botChatURL, session=api_session)
@app.route('/<chatName>/alert', methods=['POST'])
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
try:
content = json.loads(request.get_data())
except Exception as error:
api.chat_post_message("Error to read json: " + str(error), channel=chatName, alias='Alertmanager')
app.logger.info("\t%s", error)
return "Alert fail", 200
success = True
for alert in content['alerts']:
try:
send_alert_message(chatName, alert)
except Exception as error:
success = False
app.logger.info("\t%s", error)
api.chat_post_message("Error to post alert: " + str(error), channel=chatName, alias='Alertmanager')
if success:
return "Alert OK", 200
else:
return "Alert fail", 200
def send_alert_message(channel, alert):
message = "[**{state}**] [{instance}] {here}".format(state=alert['status'],
instance=alert['labels'].get('instance', 'unknown'),
here='@here' if alert['status'] == 'firing' else '')
attach = []
# attach details if alert is firing
if alert['status'] == 'firing':
attach.append({
"color": "#ff0000",
"text": "\n".join(["**{}**: {}".format(key, value) for key, value in alert['annotations'].items() if
key not in ['title']]),
"ts": parser.parse(alert['startsAt'] if alert['status'] == 'firing' else alert['endsAt']).isoformat(),
"message_link": alert.get('generatorURL', '#'), # link for timestamp, mandatory to display timestamp
"collapsed": True, # collapse details by default
"author_name": alert['labels'].get('instance', 'unknown'),
"title": alert['annotations']['title'],
"fields": [{"short": True, "title": key, "value": value} for key, value in alert['labels'].items() if
key not in label_ignore_fields]
})
res = api.chat_post_message(message, attachments=attach, channel=channel,
alias=alert['labels'].get('alertname', 'unknown alert'),
emoji=icons.get(alert['status']))
if not res.ok:
raise Exception(res.content)
if len(sys.argv) == 1:
logging.basicConfig(level=logging.INFO)
app.run(host=listenIP, port=listenPort)
else:
from gevent.pywsgi import WSGIServer
http_server = WSGIServer((listenIP, listenPort), app)
http_server.serve_forever()