Added Grafana alert silencing
This commit is contained in:
parent
186aaaf33e
commit
29da835843
2 changed files with 69 additions and 0 deletions
|
@ -21,3 +21,9 @@ enable = True
|
|||
threads = 4
|
||||
[reconstructPar]
|
||||
enable = True
|
||||
|
||||
[grafana]
|
||||
pause = True
|
||||
url = https://mon.edgarpierre.fr/
|
||||
alert_name = CPU Usage
|
||||
#apikey =
|
||||
|
|
|
@ -7,9 +7,15 @@ from pprint import pp
|
|||
|
||||
from time import time
|
||||
from datetime import timedelta
|
||||
from datetime import datetime
|
||||
|
||||
from pathlib import Path
|
||||
import subprocess as sp
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
import json
|
||||
|
||||
|
||||
_t0 = time()
|
||||
parser = argparse.ArgumentParser(
|
||||
|
@ -33,6 +39,41 @@ log = logging.getLogger('openfoam')
|
|||
|
||||
log.info('Starting program')
|
||||
|
||||
if config.getboolean('grafana', 'pause', fallback=False):
|
||||
log.info('Silencing Grafana alert')
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
urllib.request.Request(
|
||||
urllib.parse.urljoin(
|
||||
config.get('grafana', 'url'),
|
||||
'api/alertmanager/grafana/api/v2/silences',
|
||||
),
|
||||
headers={
|
||||
'Authorization':
|
||||
f'Bearer {config.get("grafana", "apikey")}',
|
||||
},
|
||||
),
|
||||
data=bytes(json.dumps({
|
||||
"comment": "Openfoam run",
|
||||
"createdBy": "Python OpenFoam run",
|
||||
"endsAt": (datetime.utcnow() + timedelta(days=1)).isoformat(),
|
||||
"matchers": [
|
||||
{
|
||||
"isEqual": True,
|
||||
"isRegex": False,
|
||||
"name": "alertname",
|
||||
"value": "CPU Usage"
|
||||
}
|
||||
],
|
||||
"startsAt": datetime.utcnow().isoformat()
|
||||
}), 'utf-8'),
|
||||
) as api_req:
|
||||
api_res = api_req.read()
|
||||
grafana_alert_id = json.loads(api_res)['id']
|
||||
except urllib.error.HTTPError as e:
|
||||
log.error(f'Grafana silence returned code {e.code}')
|
||||
sys.exit(1)
|
||||
|
||||
input_dir = Path(config.get('main', 'input_dir'), config.get('main', 'case'))\
|
||||
.expanduser()
|
||||
work_dir = Path(config.get('main', 'work_dir')).expanduser()
|
||||
|
@ -199,5 +240,27 @@ if config.getboolean('olaFlow', 'enable', fallback=False):
|
|||
log.info(f'Deleting {proc_dir}')
|
||||
shutil.rmtree(proc_dir)
|
||||
|
||||
if config.getboolean('grafana', 'pause', fallback=False):
|
||||
log.info('Unsilencing Grafana alert')
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
urllib.request.Request(
|
||||
urllib.parse.urljoin(
|
||||
config.get('grafana', 'url'),
|
||||
'api/alertmanager/grafana/api/v2/silence/'
|
||||
f'{grafana_alert_id}',
|
||||
),
|
||||
headers={
|
||||
'Authorization':
|
||||
f'Bearer {config.get("grafana", "apikey")}',
|
||||
},
|
||||
method='DELETE',
|
||||
),
|
||||
) as api_req:
|
||||
api_res = api_req.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
log.error(f'Grafana unsilence returned code {e.code}')
|
||||
sys.exit(1)
|
||||
|
||||
_t1 = time()
|
||||
log.info(f'Program ended successfully after {timedelta(seconds=_t1-_t0)}')
|
||||
|
|
Reference in a new issue