diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml index e8af624..d68ae8b 100644 --- a/ansible/group_vars/all/vault.yml +++ b/ansible/group_vars/all/vault.yml @@ -1,201 +1,254 @@ $ANSIBLE_VAULT;1.1;AES256 -39383462626535653332386339666338643666353535313064613633353230366534396665376235 -3061353662623030623038663063646661616537306531340a346430623235633264346163396661 -61333533646538373838383231353338313262656466613664393132356536396134306265633036 -3861643930326163610a303834613330326536616432646463323731666536383563306162616130 -61613665313665643265393938383664353538653937616230316136643765653632313932643235 -38346639646630396566656165376639383034323439386563376563666662363866303639343230 -37313938613261313962303666666431383864383963333833316531333065323630306165623562 -34303736646663373230303733356536656634656330316362303066303939376230393330376338 -30653666376330303234376132343731376464383061306139643237393463613565613561363335 -63336366613736616336643937326663383161376534626432646437633136646339313566626335 -32303638616466376535373331363564386534396164323538363664623965626635653438613436 -34313466363939343934636635363762626662636462356332646232633539323034646338643139 -34626462303937633032396334643232336363663732636132316136666232623865663434336433 -32363164626635356366323331366634613532666530643233643531613264393639333065396139 -39306132383462633138626231383463623335363130303565336662343933643235303331333335 -30373665303665393532613333383539323332363636366363616236343533373663333864653961 -63613465656465653763313635343263636430653964626662666434666634313664363930656165 -35313265643031653861653237643763663332383266646463633732616666346638333937613339 -61653265663765343764303439663230326234363961303835353535346332666561643266386632 -33633437336639663063373737356536326332336631303834363838346634646535366331616661 -38633235336361633132306330393131643530626661663733366132633939633637656231623438 -36646332363065643930343831316238333566323234643435346530323038626562666165373934 -35376136343566656436343438336363363364303965363533393330393430316566313033313737 -33303536323235346437316636643566306465393031353439666564313165663230643732323563 -38386265633938316462343636636666343638356338393061323134303837393231653435653833 -32616433343465623536653165623435383933656534366664343461363662313736623061616163 -65343664643736316337356134336236663434353635633032626664633333356661646433353566 -39643664366132623363336339313739346634616439396535346533396438646433323538653933 -36373639646566353931643630656432316166306531363562343666353633633063346263626532 -65343437656564633466306166366535623265363566323461626237333862623465653163643463 -38316338353233653131373336306537386638616462366463306464616566623932323633316631 -32386231313430343765306130616361343537393333616466663763633736303131356261626137 -38396634613734356465313865326361343432633066393861383363343262306662316662363330 -61393035666437366635396361313831626138396337616630346339643133366363616264393635 -63336339333862353838383437346165326238313632323862616161303330376464306265306661 -34623530656331623863376661373032333462616462646630303630626631623830343637386133 -61383236636534643932393262316330326161323435316162666566663235336639373431386463 -65363336313439323064333633353236313537613132396632343665616636396632363362366132 -62386538336162333062306136383737323336646163393161316164363561306461373731653334 -64313765313330653762336633396262626361353730353737646534383262663330363335633539 -33646534643231663464376638373934613432623639643061613262383635323133326564346263 -38303863333638353864636436623938326538323765623165633465353433663638363736316432 -66626264646335376432376561336261336633643039663533343562393736303164643833346566 -36656361356130303931666539363231633232393731653262666162396163376230643635636163 -35376163653036383364393638343663383937353239663030396663303431356632346339313931 -34326638333766353763623433393938663732373538323735323166613439663132653961366132 -63333465653035336238633665393134633436386263383862386166633436633834393034623231 -31613362346239313639346262633566373662623261343939313434616336663337306632656434 -37373331333163343035353731363462613436613833386561366430343336386261373835663237 -35643032323065313835353637666663323337626330353138386661383039383162336437343764 -31643037386465623138393236613166366335636633383762623762336535323936363134396436 -66323339383833613363653331643032616438383238643666653938613964393464613434623138 -32343835653332386135653762326238313833653264396266383830633536396131373839343332 -62316132326361646633653732663063366663646366366633353133663963356637663137663263 -62636261336463626439616334633464373566373864393965623166633733643730393664393131 -38636333326465386132636139356339613730366539323762653564616639656630623435656262 -39333331353933396534323562333665373233616135386336353334383436356630373931653237 -66656461353239346435643034643166363765623631373736656233373965373264636566313139 -63613839636565613663623831636465616335633636613437363066353961396631306332636537 -38393230616132316266613331643534363764666637663532353539333661636331623364623433 -31363433326536646439336131633961303737633364623961306466643836653133363235326439 -37616461346433623130616432353035653464366136353062633732313362643033323830626163 -64343866386162313133393637343365336363386563653436336438653836643831333064613461 -32646133636130303166333466363139386330653030326230626136643830666436653036323061 -61306530323666616332373635633132346635376237393237636638346433393131323533643761 -32613230663161643238623865613333323263303831366237386331353935636532343264653134 -61636136636538666461373537376539343136363466343766613961386231313336633438373965 -35666130373163373236396638303738306538653238343838386130663865366137613730616161 -31313330343262393632316635326134646662323535366338656434363162363461646362663761 -63653266653835333762353433633562346164623661653762666135363435376665663437333363 -63646663343030373566383665383864383463366363353630366138366433363362346634383633 -39396235303830626138363539333138363230313463303062383562633234616230333365333831 -66323632656336303231653138636164396532616265346261663839346235643439616466643438 -61653837336532316230313664623233353634643037653138313936653261363537656365323961 -66323563376363396336623164643866373833643138646361383132363537323862336262633766 -35643235613033333530346432376639353266346433386439323830363237666464306535386463 -36303032346234336335613764383165326536643439656563326465366136653964623066343036 -39333439343932346162373030343332373135343064346336353431303330666561623963366465 -38336162303337303235636131656564666536336139393537323831633739643531333838653432 -31633535386230306335616564653832656361643339386638623937326635643465643564623833 -34376633643830316635646131373039653932626231646536386165623435353861353639343734 -65393234646662363435636536346265336337616235376466613865383965316234383434303930 -31363932633765653335393262316363356265643533663762356235386437643238316334356131 -62333836666430616632613962373635633333333135656338323636346432636131613165663830 -34616338386232633066323131663464393963386538393632626265623133363637393537626535 -33303232643465353365626364306663363233333965643464393734626634353065663637363164 -66396261643531386463363931343531663733316336313537316538326436386662616338613765 -36363037323436353035353638613931373364393961336637636438306263376565346234366539 -62353530313335303734643731393636303532343362626361366162663362356463376636646661 -31633161633134626164363430306234366130396235613238646661393034656638313333343766 -39383266623234653338396637336435666134653164383463373734616364323731616462376634 -62643039623931393736383261393131333261623130333363636462343830626435623336666161 -38663936303531353932316430626261316437333536313230353431393262616463653430643136 -32626539316330323664643533653661323038313063623262346264393333343030656365653566 -38303835363837383539333265613865663930633535373230653763323437363535656265363633 -64653230646363373032653032336163353033336539303964663830323935393732393538373835 -34323938616333346139316361376463383531393134316262376332636637353834623937333264 -32633135386232656662383838623238323336633638636631386361306238333035636232376432 -39383634396635343032313164636132363331373830656632626461643464343631363838323233 -61653530616666313635383364633738346134356533336431313935353364376537666635633936 -64303764386538363438393430653236653862653165303631383434613037303639626461623066 -63383966343437646433656531333761393430636236326362656137663861663938323333313238 -34323433653363353565396636623030323633303536303366323030386566313661373063353435 -36346232353838663962353436316133643331383065346536663961373665343338373761653439 -63353463326130356565626164613836393931383236363638643565386230386363363236366536 -35653366393139616362633737616165663438653738636337313861313035353430393962373264 -64303138373339616436623664663638623666336637653662313131646630376134323437616166 -36366330303836396133393764653462316532313434633738316335626430653133633162623734 -31633465623766343566393431306535633033363937376638643865306239316565366162613366 -36323439663836373630643138303235313636663431663738653436646134636230393535623236 -30393036336137346630623035626239313661376539333237313261323438386632303939336566 -31346433623235326630613165323762363835343939623666613638616164626236646433373864 -39623866373238356138336266313135393063373733373332303338343965303538316431663462 -65613766623538633138616632366436316230646162323432313434393938356635663836633036 -62356631626138343436393238306539663532353431366162336130626334633230653638323134 -62313830343730653230346233626633386433313131663233303435373561343036393838633830 -61346135313130323464643531646139393633386461666630613637363833393466353737306530 -33653331346464613431643735396661346431653933346563643531626638653932316238356632 -30343864353431393231386639343237373130376639343236613337643739316132333133373962 -34393962393065363133353338613739613332653365316335306663643331383138323537613866 -35336339613138323031366436616139306339346363663662316265306161343935376539376166 -31316264353636343463636230646431333164626135663833316661363261663332366335326130 -36393438626531663538313931623132303332366538396666303939663061353865633936343162 -61303533383439626636646230623931383636633136336636643436646662396630303637333535 -63623661376130303134633761393835643266326133616236363464326238376466326266633763 -62656637663265363338636137346138616431616139653266386432373461316634326339333932 -31373562366132326635366164353637626334353464323764636463316665373936643030333539 -35646634643339653564303835333163383936386335353139636563363939323632343562633662 -65663033623232393430396139626135326236663463333639353338633736356430323036356537 -35326538616664386233333264313637626436613033643936343063383261653333353339306538 -38313631663263303230363830623235343732626638373535316537343938633632633534386365 -36623336343866383132373162626565313665393964663562653463396130623238643631363838 -38313363666635653134353663653461343135313138636434376362303639356362613639346238 -39343432613164643265333231366139326462313363633234313832386132323239323166613931 -37666135313036313236643261333739306362626236613435343766303162373464306531326333 -66363664306535666235636234346164373064373863646562353331346536613764646633333861 -65323361666430336462656238633735366661666662643632646562366535346432346233663330 -30336261636239613932303032653962376231333536633830646364663135626561336339393364 -30633532616163616363623835306335313163636337353263333836326261643334636666333836 -66653731316639376162386363303766323734653938373866306264376462643038386436323438 -62396136353961613633316661666136323037613863383338613633386536336330333363626237 -66376266623963353439613433633037623135333062636436656663633034343437346262333534 -65333331633037396239653838383462386136623764343436623737643564663261313834333133 -38343934636164323135653435653835333331623864383133313965373065643439623361613631 -62393036356164313665326537626363633839313935663636613434363934616135323036346131 -32363033323862613562353364373035653631366431303865646134316438366266643965623536 -37633835386439343932663861666230653463336332343864363035346135643535343535366135 -61653262363937373036616661333639383362383337316561303163643639306466353334353866 -38653862633732333962616135383234336233383434643264396461646264363134353531356537 -31643030386437303563313033373366366565633435616435643164626334643364613535363232 -64623835393434316131303237393038323930633861396431643833346636353534663362336264 -35326337303036303737336333636432353033366461653435626464646434646461303337643437 -34666635626335333133326663353234393637386330393964633836666364336164633030643534 -33346335643962323866306634383335613463633136636362396464383135666162333866333037 -34383461306332306132303063346634643838363835393564646537663236343432313766336265 -30336537313735386261376365623461303337303430613461373866366665623332343633643439 -62663366323165383463333864653132363632636134333932393333366466356536336666393330 -37346361393831363633623735363032646464303765353935643265373463323366353461316535 -33366265333863613531653663326162383431353932366231396332306234363963653566656130 -32313831653634353865313561363966653633313535363464326664396432623561653738313561 -62626239383562626139623865353039643365613565386337313666386364656537373165663833 -36353264613462393365396562636138663535613433653461303935646438653039653736616564 -66643737643437313366333830303839333965626164646637326331626265323035626464326462 -61343861316532666461336533656430633261393732323232313864623832323662613334386334 -65653837366532613538373138303633346566663330613965633963653864666330346532313038 -37616163626235303364303534383930376262643539633630336162383864386437633162323332 -37393236346334646139373130653438393663393335336331316336386236303535366237616330 -61346663323234626637343532666135633133343234393064323033623832613264623337356139 -39373530366564333437383266333636666363393461303034346566626562393065633732643564 -61393936316330313137386338323461353339323331383533376362323833666635383836333736 -65383738373537633164626237626364623866663031313864633032353137353331393765393339 -62653337623538623664326563316266613263376536653138656632303939626133653964616537 -39613763383938633163323234356139313261613332373861393462323930353066333134306532 -36303061363763316632616134353633323739633665313962623364356562366535326339646635 -62333838373833333133653338366638386566366337303232353537303961373030303464613964 -39626538353033383430333132306637363633353135333232613462656634626438646139653266 -61636365623634396438633731616665623333353735663036303664333461636536313733363232 -61656333613736363039613865383566343336346466366165323737336233303239333163326437 -66373231323333316630653039343731323332656131633131663635636433313732663935303232 -64346664353836383864333163353834313732376638303036323235303034373666653666383264 -35316537646237373535366262656532623331393566316235396631353030663961323238376236 -64303766636134626236646530616331323334613233633932653861333961346138646266363864 -36616563313131633931376230613861623331373839353733353730363234343339303037653663 -63646561346466386131626361326533363739383832383437393033376261323363643966396662 -63323234343931326663636662363837313665383962373965623335346330623735346665306364 -64333239303031666262316239613033303837386164333034616461316237376431643862366231 -63613935653533373530346131346265363565343166656563303164663132346430616234346235 -38343363633063326233306261643466383262346661363933316238303836373831623438653238 -39646438306639383561646666646137356561633131363563663066613263333234636466366636 -63323065346430353834636332383536643061356464333863356666333264356633306131303464 -34343230396339646466663963613333356235316638636463643333346262613637633936623839 -30666430353261363432653162353166326263396333653633663764376535643331336661626666 -64653433666638313838353766376663633564326633376461336234366534656531656638323164 -33636130333032393030633131646532323766383062646431646461353035636338363865366564 -32373664636362613539356665316461326638363631316664626435333238653135376134303035 -32396631666466363338626131393562316638393531633438613133623734323461326232383665 -65356538616538363065326264306236613330303439636232316339363333306238393766363361 -39396634366333636637643136653164623165613733373039366335323061636332363636336330 -61306237646637613961326564626334316334623835616365623132303039373930316137653433 -38306134383738346662 +63323431376238353966386463626539656230326233323861656165386335383832316631353236 +3731396264313366313166653861313736666435346537630a303363346131366433313264626135 +30633836613636393833333239666364623962383763343434353463343739633033383433306664 +6339616335366562310a393463663239333530633034373462313537376266393937373033346233 +62333331613063356535353134613538636663383166353731616633336366653864656333356330 +37323139616436353065363866633139323764336432656663326236636466356432333132656530 +39396566656161373365653738316162653134393234613130356561663464623564316161383566 +32356161646631623134313733343030353064666635346134653361366635316362316564323562 +38623236383137383665623934336631636537343361656539393538346665613234333638346466 +61653434653430636534636137613236326330356234616137303130666363323861623230626662 +37363839333334373561323863353563303861636137613338303736343064663232663064303466 +62396466363966303339643938653930313163323561633835396562636363633633646163363461 +61636331316161386630393766326431316239326563633033376538323330303863646162303332 +62306439666232626533306238343338343938626461363939326363333137306430643033373363 +65393164626166366663656637373330633939326361653261336339393135363934376164303537 +33326631396161393936656563336636643132666232343035633465383632613661633135343165 +33656162656334353636303130613231393835626166633666316561343165333138376439356539 +35356461353832643166613437343633346563393636323631353034366233323566613039343030 +65356438656664366434343638643963396563663434623961663432653334646639653435343262 +31366465366337633638393061643139643138316536396333653035613132623230646561373465 +65366332386565623161303563386237666538633433386438623535386564633937386434343264 +39653335343235386439383964356664313339356531323732363362643566363964653634393039 +37643338316264633733383531633934373132643034653433316438303962356139376364306466 +37613232636365623732313233303766373566623162303965353163373131363763346135313230 +39316466613136323039373765613336333835323465323737323535393736366433343664616539 +32303839376538393238613763366433346433643436613662383062306366643561626164363430 +38353738653734393237663765613733633230343965363732643336333337646438623562303561 +34373031623466343539663738613561346665316163623631643236633236633765616361623764 +35343465346435656533336464393633616663343239343162343664363763373736656431663433 +62306565643862613065386265613036326336353130633530373166343966343033346665346235 +37333761613437393236393935646239373930613239323639616564383336376664386532366236 +31313862363232656537613733666239373433343835633964333164633335656436373766353838 +36303234353333333233313531396464336262663864633638306236343633336432333737376539 +36623439336133356362626632663966336162646636393932353537356638336337346663303230 +66623964656133333534313231656337336463346363363635396135393062643736336133393134 +66376233376161373561386531633639376137643732613535333066646666616133623963323137 +63356565383430656535633639643363653232323737363566663832633931616230343132663563 +37633435626266393937373831366632643535623634386131343065353036653163326464666364 +39613732653739643436303665363132336235333339653630353335646432306432643235376139 +33643935366365383232646234313436383534353130633039376562363939363033643830303936 +64376131346631306263363137316435343661393562386638363636336261623831616232313934 +30373133633034626430313936373537323366626562323239623538306130623135333562333333 +33303761623232666165323438643364313530316563396466653331636531366166633964353033 +61633861363833613264353036383766343233646439393764636664353938656335623330323062 +64306565626630326632383963353266383464393530623639333836663739306132346336373936 +36636532623032666433303339643062383663646536636663383662646237323336386366386138 +31643930356330343138323132613238333837356137643061346364653134346165383661366663 +31643630353631656131633430323838383233323936623661613466663761313264383338373636 +63323062646334666665633662366364383434393561313863653862633533356166396336303133 +61383637326535656539656538353238326337353138616533313534643131346163356533396331 +30303031656133623735366163323664626362356365663730306438653132396461393539386138 +65343865633765306337663830343931376265666362653361616662666665386138313462633432 +32623436643239363437373161353162626663623332373163656133613465333139356564383130 +36653433653535326433313337636635336132663636383764653237356432313362393535306630 +32663738663835336131353737303262313966663264303264313864353663323733653334313263 +32333866613335666436303236393536633835653837363133666437303736303630633939616232 +65316337646665343062353864623839613263356335353938386136666166363565303062633331 +39633761316630656662636136393834626564616139396336663363373931666132383062626536 +61376462333131633634383130393765353037653536653837373032663636623031316530613961 +63633861613366373235653735616333303262333765636531353734633664383766623261636238 +32623132336137666366643035306562643833333537633666343637366230623366666230366638 +37333238336435636235333063636538666635666438383239336164613536666262343132646634 +38633132343966663564326138346561313731623966356435653937383431396261313138656261 +65326265323333366538656335306431396666643663313561663964626436356464356336323661 +66663763396166313831623534353131633566346634613738316634653935373038303730336662 +34656634343834623930623833386632306136323738646535396539643463393861343832333065 +35333464316163626130366233663439633435333462353534616530316464666537613436386637 +37363939333834626638396337636234343561313565613636643463393132626466333632636463 +31326261656537616130613634323736313132633361653162666631303965613036653434653236 +33313037313034356139353261666138386636663637353831666330616563323536303834623863 +30346539376536396337313561396561363237613865623533633063613936316162353138306164 +64646634396236646535633135613036376538343964363663616234666532386165643938396436 +36333832313136363965383334323430313630663336343562626365633461366133653931376139 +38613364383236393634386436643733333433336563306138356337636337623239646164306539 +61336433383339353861313131343166353265393132356631366230343438646239323730613865 +35333438326236353264386436373338303532646336333161636232393235653830303237323137 +36323464303634663839326230613539376238323931663137616363663434373662636535373266 +38363261653937633363316665613130386135626135333662356261323462303939383962363462 +37663333386336626261663464393561666135633537616365376665393664346466633932656635 +65343762656162613831643265633562373865616662313631313034363964626538383633656363 +61363639343663643638353935643635643663636139313963303739343830613336343661663733 +32323463626264663639396130343930383162373133386232613635633934393334626631303561 +64346636636330623538343534663037363635323138343566303664613337346638353637306336 +36386432326661316633303764323761303363646337303539633831666665373833666663333766 +62363461616462383663316434373561653063323062356131643930633838336364643065613838 +66386565373635396138623832646137636236646335386463326265326565353566346136306564 +36323666333331396636353762363866616239343362313431313765373334386637633366623936 +31333432663331623232373330306430626264333761373362303365656262303164656439623637 +36663065376137336435646232323262646439623266333534623766353035376663353535333137 +38373335313833633239313963613439316433643832653938386434323666333263373437663337 +32613661666664366636616562396232366237333237316430346565653066623561636263343265 +34313137643933366239353836383265633030373636393232663534393036343130633438643331 +36643236636232323532363036646563646436613630323638356534373831373737386139393664 +62616133323861333265626665616331333665356666643734666537356565393334373036663330 +66346361363361303538663936616664663864316530303338316634616636353235353161366135 +33363630626566336363336630656331343437393666353262396137386163633134343238623361 +61306163333831626164326461616630636266633666383765356539656536363133393537333263 +66646635343437643762373432333164626334333730373762336565636533333132363965636334 +64316232623437353263656131633335396166616133653234623432386234666531356431373333 +63316462643464626362646463323935303765643061343535393065333961303931663564353735 +33336565383332303637383538316237343430626432393533376166323565303263643435313136 +30626165623765346430373165323030386635376338666235306534643730656564653531633765 +36663864396638666165303837646236386434326166663365356164353036646464363132356261 +66373861343834643161343665616139373466346130353233623135656663373230333630306132 +66653233623632303463343261653764636230333038623936353138356565623061643432346265 +36326263613833386237333566333534636238336539303638643233376331616331626636666635 +66303535383838336135306439306239323531343331343832376636363931626663333337616464 +61656532623437363634343039313534356565383361303562336666383561333235303939303339 +65326336633631313133396263326139636535346166373333343934316435363435353236363331 +30396231383336326461326462393633353739346463653636393331313534613466363738376263 +62663264643833333032343839373165353262663637376635373430333631316532336335653038 +32303835333461363635306664653331613164316264613632326131633639666263633234306336 +30656562383439333639623534303164323231626337396362373235383530323731626139333335 +38386238396565643533303030393364373735376564373765373632386335613432313735336132 +63306231373038366131353934393735323932633233646439383666366333653535373634386333 +31643130376266626236353937633235623765323462396130383831376366313337643939363366 +62326439623135393539366133386337353964306637343238633730363639373831633662653565 +64333134336666643465633565333765613835373765663664653331353935666437633566386165 +61656239653264646530306165396234626365326238616566303831353935626362366338613339 +30303364356137323935626662636663383761383935343531666461656537643333346637666365 +37663836386466303433313339663531373432643732333461613739636233336566356139323934 +36663965626330333436373764613730393365616166653866306336633939393765633564626331 +31353131663038323235396564623234396138386237663030316530353337373934323232633433 +39363931656466313639346265363466646631363032353363383662306436346162363431353833 +36303934323364363465343236633064336239643565393034353934373239376139386535373061 +34643365393866636430356338626438386136646161376538363762336265653632643334336331 +38646131386463666135376162613864636534366264336630356135316637393135646630333733 +64343731633836306565623238313936656164653038393236356130306162346331396231393436 +38303435383963626434323230643832303838646239646263353737323166613965623734313933 +38313164313062373263326138656239366135383361616436333539356331383064363562376366 +31333838313664653432373937306239363631626234336136396364396166656234623365616237 +36363434393537396639633062626639353738386232393066333034343132303831366362623031 +36643862376637643739626662656239633361653933313130646661656332316535396230386162 +39373738366138373130643636663339613732626532383465316365363638386361623838656630 +66636262666536303739616361343763366135313835353938323330363635343135633138306361 +61346433336366643430646334336136346136646166363962613336366239653236373135376138 +61316464646264316532333839626634623165336334323836643130323137303632353232616631 +32653036313133333237363233653932366334623133646565343461373132306365313331373335 +33623561623163383562366137663733343833333136353738386535313439386164346164383565 +32393331373130633065323266623465613732353431623234653435383133373537363436346463 +63323666666136613034643237623463383462326334343334303731363961303438646330646562 +64336132663561353565376361306331363138306638353834613231323331356238343730616164 +35666432626436376539373633383466343732616430333661613865653530366439626131303233 +63353633613232636238316532633231633539633363623734316337633764333430366466626430 +34336534616537656531613538356231343061326462333663653662343762666131393765306462 +34353435626337656362663637613763623537623534666336643833643963626265303266623434 +32323836663432313438363530663839653637346562626566316163373137363063633564323731 +64336638616661366437656431396435303439373132383330646635306435646138633531336461 +39663366643961303832323838643530653632383230373366396531646639336138373434396431 +32383064313533393735336235356432646162353236343365313435313666363935333835336331 +39316330306336396637613634306531346531393036326536653961313438623664643362623937 +36363535383562663535343332363064326530636335343963366166333365613234643035343764 +62366334623332653931393836303934333139636635363638303734663230363537663338383461 +34366661313931316639303762373131636530363232386263313361666561313034343033626333 +63386630396136316338653134306632323664396466343361306636623164616562393430353732 +32646537653963383639616530646561653065386631386138616130343936333935393861643161 +36613166633561646138336539373731373939633234633861636634396563646137316134636463 +30363133633131303632316634376361623637373133393234393362386338626536653438666539 +31633837353935346438316133333136326237303430313533643265363966396537373161346236 +33656163636338363565653834636461333837653433633639393361636565613337653766353765 +36613834633631346136366232663539636566666339343939383732306537396465373932646434 +31663736323535353763623633366535333262636234316363366537393531666430633631373364 +36393766343863353135333864656536393739636563363631306362336665616636303832666330 +66623336663566316330333337343064626366323563613463646338613433623637363064656639 +63626465613963333932346131656639653239353034613862323565666436316338666563623836 +37356465633635363834316564313839366137656232346637663231373261393035623035643065 +30316663613732616237376265363561326164313631366466653139323534623531323537623164 +35383365336438356538343934356134646337626635326266336533386262323262323734663534 +35313239393232393435346531643138376362336437323963613933623739343463613831326539 +61643939393461386534656664366361363062643939393962376664353563643635373336333266 +39383132363964656438623031366336306362353736633634353033656539376233343131653030 +62653736663637653166373930313664333930346262396339386432353066646465373937393964 +35656538373064643936323731393436386665353838313563613832323834316539366163373638 +30623365333266393331396135613663393139303136663636383766613731636435386635666132 +30633664303830353137393438303666656235616332393132613364333238646566303363363365 +61636132663366666139363965663239623764366334333432633830316636303263373331313931 +30323337383931353363343231656632663534323435353338646664393635323733623166343830 +39633865336666363635346462306261613935383666396261653531396164383961383830333434 +39303063383365306435303430633834386566316332313864373664383766646463393734343961 +34656339343761613863363831393933396438636332333339393433636435316330393634353032 +66303139333933393164363735303534346332336564333366343032353631396633366337333831 +63616637313262316335386230343034323038613530663432353764656337343638383135316361 +32376339333235316335656638613434306633316631376230383434303234666532303662333630 +32626562656363633837316335643162363232623265396362653264313934336566366130376634 +63633132626539353138373263313135633265623761393063383136376130646132643664343431 +32616136306335336636343434366236376536663730643638636234623136383262643766613137 +34666431333265633063356566623139666266643138653365613961613532343337346161643337 +65316332393762386133633030366534353763323738303437636537326137306365626632383630 +34363566643436376535386134383638313237643934303931653839616637643036373134363830 +66656463303331626430663063356239373434386361323139343838333763343565396637356561 +63386436353934646336616537323233336339326261653466653135653735316463653666343231 +36343261616338303864343162646338323339633634663032306433313138376539643236393462 +35333734306662393836386231306466616133383762353464343965376132643634393237396164 +34373831316634626133353735376661633464373238383561316565616331666431356165623039 +34386130363761663365666238323534393933313866333030643132346563646463393063333764 +64646432616231343032633965643337363234393435346430303931633665623362306465383962 +39383437623761613339353733643862376462623166636264303833666437393231656437663764 +65313065323134623265333466373461306639326465326362646532643037333230373837653230 +62383033363561343735623639626565333666636563306639643139666134383762353261343864 +34386334306237363762313465333163643037626534613830663463653563663537396231356431 +36646139333534643765653732303330343532373933656537633465326632663539373638323865 +33386363333065373565336566613333313738313437613764663664633032393865313331633433 +65336664386437353034366638643533366365393864363033366363626630336132313038653430 +30396539313865646333666462643061666566313033613634326537363132663462623861666137 +66386663633634396163666164316631393635623938333137653361363963303235363732623930 +30376632376264343036643461613438396531373865613162643734303963373539363464343337 +39653066626635626164613435643738616534623064306338303734653830633737386531393435 +61303461666666363230643638626230353839653962663439353166346635376438623832663637 +35663738356136363165653133323666383935363566616135376161356165316239373661643039 +35353666326336623561356635343232393137383536356565353762616639383932313464383063 +35393734306636313561383133393961646132633639363136326332366338633535666139643634 +39303735653564656232326339313238383137333630383539623139356539323561353366323131 +37393130356639623133653134623131633031393633386330653034353166376466633739303638 +66613534393331383438326230313936653532336134643632306634323530343630363236646235 +38656463343932333637613866386232376162623939626466303063306466623132643731623338 +63656662376132653238343537343265373236306134376565323961323264383830363065323738 +33366137323535633732316563386237646230613339666337386162633166353533346565393337 +34316233323734643262663532373964346331393031356134373266616265373564386236616237 +38373730623261346365353363383833633964383132623361343333373637386339313262396633 +37666437613361646337323039343938353466346538356138376130313337633533666137626266 +66313838323638626230306138623135363266346366316339313164626233656237376266353762 +65356461346239613066613038353735626233383939666330393131363064363337356330363435 +39656633616166356430343564366433323864333236623934356335336661346338653834363362 +39343435653934643132363931363334366631646463306261666537303938363633336464666537 +38336632643561633461383965333264376462306131666232623735313265373832343762393434 +61386534393034353363613230626330393234326234363837393738376634633561613562323137 +66613930306330623434323539366333663364396165663466303365653331363431656365353266 +63623430353734616439393735623564626638313336613636383438363531306234663939383066 +37623165333233643465363334663433663034636438613433633966306334376462316233343338 +61333530376236333134306164323263633266376666663030303438646335393661316137646237 +32383865636433366635323134663938383339393933656438633662333334313264643338636563 +64343533643432386164333630366531383434333231393762613136616435376534653530346364 +64396531383363376437613366613066633534616163396133323835323431353034373563306536 +35616335636463623565636534346435643463376330333962353261663062613034653863373834 +37653566383164373263616265643536343037346464633930303935393337336333616338633730 +35316166656164356535613364386366373666306131373063376465663935303530666432383435 +65393934396639333765313933643263306337623635623930656430343361653861653039323861 +62326165313038313137323539343934366134383630363632653939633331626566653663613666 +30336161616136613034353133663738646464306164663931373365343664373337303564643565 +31623132396431336130396236656136656335336236306364656164353431633136343732663631 +37316366633837323961643338343538653933306664356236373165333464643032623864633538 +32613030313563343930653863346638393662303030396537343264326234643735323532376337 +61313936623663323961333364306664613331626233393430626632373765333832616136333065 +65323837373231633439333437616536306466656530646332386164373963386431653532666262 +62386234393432633430636439663331386235366630333630383336363664663333383331386362 +31376362643263396634623662373134343039323663343433383836663261376463306337656236 +32633536336238326337336666313263613433353739333530316630653735636133303635313436 +66383761613932653132663939353734623663333736666462363235333336333733323963623663 +33616265636664373638656363636363656639373634353732366664363565383737313863666139 +35303364343063663764393261333336373864373839666664356166626238363035343163653131 +63316436373331393461626164346362366530636335613966353335376334643433333963396563 +39373364356439663363333566656565616130643037613332363937313964363433613436666230 +6232 diff --git a/ansible/infra/400_host_monitoring.yml b/ansible/infra/400_host_monitoring.yml new file mode 100644 index 0000000..ac81289 --- /dev/null +++ b/ansible/infra/400_host_monitoring.yml @@ -0,0 +1,175 @@ +--- +# Host-level monitoring for the whole estate, reported to Gatus. +# +# Every check here PUSHES. Gatus never reaches out, which matters because nodito +# and its VMs sit behind NAT, and because four of the five checks are internal +# state with no pollable surface at all - disk usage, CPU temperature, ZFS pool +# health and UPS mains status cannot be observed from outside the machine. +# +# Liveness is a push too, and that is a choice rather than a limitation. A +# heartbeat proves the host is running AND can reach the internet; an ICMP probe +# from one vantage point only proves it answers pings from there. And because +# Gatus alerts when a heartbeat window expires, a check that stops running +# raises the alarm by itself - a dead timer looks exactly like a dead host, +# which is the correct reading. +# +# Each host has ONE bearer token, shared across its own checks: a token can only +# write results for that host's endpoints, so a compromised host can lie about +# itself, which it could do anyway. +# +# The push URL must use Gatus's own key format (config/key/key.go): +# key = sanitize(group) + "_" + sanitize(name) +# where sanitize lowercases and replaces / _ . , space # + & with "-". So +# knots_box_local becomes knots-box-local in the URL but stays readable in the +# name. host_key below is the Jinja equivalent; do not hand-write these. + +# ───────────────────────────────────────────────────────────────────────────── +# Register everything with Gatus. +# +# This play runs FIRST on purpose. Gatus reloads its config within 30s, and the +# host plays below take minutes, so every endpoint exists before its first push +# arrives. Registering afterwards would 404 every first report. +# +# Heartbeat windows are several times the check interval, so one missed run - a +# slow apt run, a reboot - does not raise an alarm, but a check that has +# genuinely stopped does. +# ───────────────────────────────────────────────────────────────────────────── +- name: Register the host checks with Gatus + hosts: observability + become: yes + vars: + monitored: "{{ groups['managed'] | sort }}" + + tasks: + - name: Build the liveness endpoint list + ansible.builtin.set_fact: + liveness_endpoints: "{{ liveness_endpoints | default([]) + [{ + 'name': item, + 'group': 'liveness', + 'token': gatus_push_tokens[item], + 'heartbeat': '16m'}] }}" + loop: "{{ monitored }}" + + - name: Build the disk endpoint list + ansible.builtin.set_fact: + disk_endpoints: "{{ disk_endpoints | default([]) + [{ + 'name': item, + 'group': 'disk', + 'token': gatus_push_tokens[item], + 'heartbeat': '30h'}] }}" + loop: "{{ monitored }}" + + - name: Register liveness endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: liveness + gatus_endpoint_external: "{{ liveness_endpoints }}" + + - name: Register disk endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: disk + gatus_endpoint_external: "{{ disk_endpoints }}" + + - name: Register the hypervisor endpoints + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: hypervisor + gatus_endpoint_external: + - name: cpu + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "16m" + - name: zfs + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "30h" + - name: ups + group: hypervisor + token: "{{ gatus_push_tokens['nodito'] }}" + heartbeat: "16m" + +- name: Deploy host liveness and disk checks + hosts: managed + become: yes + vars: + gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + host_key: "{{ inventory_hostname | lower | regex_replace('[/_.,# +&]', '-') }}" + host_token: "{{ gatus_push_tokens[inventory_hostname] }}" + + tasks: + - name: Is the host up? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: liveness + healthcheck_description: "Liveness heartbeat for {{ inventory_hostname }}" + healthcheck_check: liveness + healthcheck_interval: "5min" + healthcheck_boot_delay: "1min" + healthcheck_push_url: "{{ gatus_api }}/liveness_{{ host_key }}/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is the disk packed? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: disk-usage + healthcheck_description: "Disk usage for {{ inventory_hostname }}" + healthcheck_check: disk-usage + # Daily. RandomizedDelaySec spreads twelve hosts across the hour rather + # than having them all report in the same second. + healthcheck_on_calendar: "*-*-* 07:00:00" + healthcheck_randomized_delay: "3600" + healthcheck_boot_delay: "5min" + healthcheck_push_url: "{{ gatus_api }}/disk_{{ host_key }}/external" + healthcheck_push_token: "{{ host_token }}" + +- name: Deploy the hypervisor-only checks + hosts: hypervisor + become: yes + vars: + gatus_api: "https://{{ subdomains.gatus }}.{{ root_domain }}/api/v1/endpoints" + host_token: "{{ gatus_push_tokens[inventory_hostname] }}" + + tasks: + - name: Is the CPU hot? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: cpu-temp + healthcheck_description: "CPU temperature for {{ inventory_hostname }}" + healthcheck_check: cpu-temp + healthcheck_packages: [curl, lm-sensors] + healthcheck_interval: "5min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_cpu/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is ZFS broken? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: zfs-health + healthcheck_description: "ZFS pool health for {{ zfs_pool_name }}" + healthcheck_check: zfs-health + healthcheck_packages: [curl, jq] + healthcheck_zfs_pool: "{{ zfs_pool_name }}" + healthcheck_on_calendar: "*-*-* 07:20:00" + healthcheck_boot_delay: "10min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_zfs/external" + healthcheck_push_token: "{{ host_token }}" + + - name: Is the UPS online? + ansible.builtin.include_role: + name: healthcheck + vars: + healthcheck_name: ups-status + healthcheck_description: "UPS mains status for {{ ups_name }}" + healthcheck_check: ups-status + healthcheck_ups_name: "{{ ups_name }}" + healthcheck_interval: "5min" + healthcheck_push_url: "{{ gatus_api }}/hypervisor_ups/external" + healthcheck_push_token: "{{ host_token }}" diff --git a/ansible/roles/gatus/defaults/main.yml b/ansible/roles/gatus/defaults/main.yml index 95c41ae..9c7cb12 100644 --- a/ansible/roles/gatus/defaults/main.yml +++ b/ansible/roles/gatus/defaults/main.yml @@ -57,6 +57,21 @@ gatus_storage_type: sqlite gatus_storage_path: "/data/gatus.db" gatus_storage_caching: true +# Per-endpoint row caps. Gatus bounds the database by COUNT, not by time, and +# trims inline on insert (storage/store/sql/sql.go, InsertEndpointResult) - so +# there is no retention job to write and no way for this to fill a disk. +# +# History depth is therefore a function of check frequency, not of days: +# 900 results is ~3 days of a 5-minute liveness check, and ~2.5 years of a daily +# disk check. Upstream's default is 100, which would have been 8 hours of +# liveness - not enough to still see a weekend incident on Monday. +# +# Note the uptime table is separate and its 30-day retention is hard-coded +# upstream (uptimeRetention), so uptime percentages top out at 30 days whatever +# this is set to. +gatus_storage_max_results: 900 +gatus_storage_max_events: 50 + # ── Alerting ───────────────────────────────────────────────────────────────── # Pass-through: rendered verbatim under `alerting:`, so any provider Gatus # supports works without touching this role. Empty means "check and record, @@ -69,6 +84,10 @@ gatus_default_alerts: [] gatus_basic_auth: {} gatus_maintenance: {} + +# Keep serving the previous config if a contributed endpoint file is malformed, +# instead of panicking. See the note in config.yaml.j2. +gatus_skip_invalid_config_update: true gatus_log_level: INFO # ── Self-check ─────────────────────────────────────────────────────────────── diff --git a/ansible/roles/gatus/templates/config.yaml.j2 b/ansible/roles/gatus/templates/config.yaml.j2 index 67d6244..6a2a72e 100644 --- a/ansible/roles/gatus/templates/config.yaml.j2 +++ b/ansible/roles/gatus/templates/config.yaml.j2 @@ -23,12 +23,21 @@ web: address: 0.0.0.0 port: {{ gatus_port }} +# Gatus reloads when its config changes. If the NEW config fails to parse it +# calls panic() - unless this is set, in which case it logs the error and keeps +# running on the old config. Endpoint files are contributed by other playbooks, +# so one malformed file would otherwise take the monitor down, which is the +# worst possible time to lose it. +skip-invalid-config-update: {{ gatus_skip_invalid_config_update | bool | lower }} + storage: type: {{ gatus_storage_type }} {% if gatus_storage_type != 'memory' %} path: {{ gatus_storage_path }} {% endif %} caching: {{ gatus_storage_caching | bool | lower }} + maximum-number-of-results: {{ gatus_storage_max_results }} + maximum-number-of-events: {{ gatus_storage_max_events }} ui: title: {{ gatus_ui_title }} diff --git a/ansible/roles/gatus_endpoint/defaults/main.yml b/ansible/roles/gatus_endpoint/defaults/main.yml new file mode 100644 index 0000000..7d49d7c --- /dev/null +++ b/ansible/roles/gatus_endpoint/defaults/main.yml @@ -0,0 +1,28 @@ +--- +# One invocation writes ONE file into Gatus's endpoints directory. Gatus merges +# every *.yaml under GATUS_CONFIG_PATH and appends lists, so each caller owns +# its own file and they compose without coordinating - the same shape as +# caddy_site, where each service contributes its own vhost. + +# Filename stem: .yaml +gatus_endpoint_name: "" + +# PULLED endpoints - Gatus makes the request and evaluates conditions. +# - {name, group, url, interval, conditions: [...], alerts: [...]} +gatus_endpoint_pulled: [] + +# EXTERNAL endpoints - the host pushes its own result. Gatus never reaches out, +# which is what makes this work for machines behind NAT and for state that has +# no pollable surface at all (disk usage, ZFS health, UPS mains). +# +# - {name, group, token, heartbeat, alerts: [...]} +# +# `heartbeat` is the important one: if nothing reports within that window Gatus +# alerts. That is what makes a push check detect its own failure - a dead timer +# looks exactly like a dead host, which is the correct reading. +gatus_endpoint_external: [] + +# Where the files live. Matches roles/gatus. +gatus_config_dir: /opt/gatus/config +gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" +gatus_gid: 10001 diff --git a/ansible/roles/gatus_endpoint/tasks/main.yml b/ansible/roles/gatus_endpoint/tasks/main.yml new file mode 100644 index 0000000..40f3448 --- /dev/null +++ b/ansible/roles/gatus_endpoint/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: Assert gatus_endpoint parameters are sane + ansible.builtin.assert: + that: + - gatus_endpoint_name | length > 0 + - gatus_endpoint_pulled | length > 0 or gatus_endpoint_external | length > 0 + fail_msg: >- + gatus_endpoint needs a name and at least one of gatus_endpoint_pulled or + gatus_endpoint_external. An empty file would contribute nothing, and a + config with no endpoints at all makes Gatus refuse to start. + +- name: "Write Gatus endpoints '{{ gatus_endpoint_name }}'" + ansible.builtin.template: + src: endpoints.yaml.j2 + dest: "{{ gatus_endpoints_dir }}/{{ gatus_endpoint_name }}.yaml" + owner: root + group: "{{ gatus_gid }}" + mode: "0640" + # No handler. Gatus polls its own config every 30s + # (main.listenToConfigurationFileChanges) and reloads itself, so writing the + # file IS the deploy. A handler here would also fail whenever this role is + # used without roles/gatus loaded. diff --git a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 new file mode 100644 index 0000000..1c84996 --- /dev/null +++ b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 @@ -0,0 +1,40 @@ +# {{ gatus_endpoint_name }}.yaml — managed by Ansible (roles/gatus_endpoint) +# +# Contributed by a playbook, not hand-edited. Gatus appends the lists in every +# *.yaml under its config directory, so this file adds to whatever else is +# registered without knowing about it. +{% if gatus_endpoint_external %} + +external-endpoints: +{% for e in gatus_endpoint_external %} + - name: {{ e.name }} + group: {{ e.group }} + token: "{{ e.token }}" +{% if e.heartbeat is defined %} + heartbeat: + interval: {{ e.heartbeat }} +{% endif %} +{% if e.alerts | default([]) %} + alerts: +{{ e.alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} +{% endfor %} +{% endif %} +{% if gatus_endpoint_pulled %} + +endpoints: +{% for e in gatus_endpoint_pulled %} + - name: {{ e.name }} + group: {{ e.group }} + url: "{{ e.url }}" + interval: {{ e.interval | default('60s') }} + conditions: +{% for c in e.conditions %} + - "{{ c }}" +{% endfor %} +{% if e.alerts | default([]) %} + alerts: +{{ e.alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{% endif %} +{% endfor %} +{% endif %} diff --git a/ansible/roles/healthcheck/defaults/main.yml b/ansible/roles/healthcheck/defaults/main.yml new file mode 100644 index 0000000..d29cbd1 --- /dev/null +++ b/ansible/roles/healthcheck/defaults/main.yml @@ -0,0 +1,38 @@ +--- +# One health check: a script, a systemd service, a timer, and an optional push. +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed -healthcheck.service +# Reporting anywhere else is optional and generic. Point healthcheck_push_url at +# Gatus, or at whatever replaces it, or at nothing. + +healthcheck_name: "" # e.g. disk-usage -> disk-usage-healthcheck +healthcheck_description: "" + +# The check itself. Pick ONE: +# healthcheck_check: a template under templates/checks/ (without .sh.j2) +# healthcheck_command: a shell one-liner that exits 0 for healthy +healthcheck_check: "" +healthcheck_command: "" + +# systemd timer. OnUnitActiveSec unless healthcheck_on_calendar is set. +healthcheck_interval: "5min" +healthcheck_on_calendar: "" +healthcheck_boot_delay: "2min" + +# ── Reporting ──────────────────────────────────────────────────────────────── +# Gatus external endpoints: +# POST {url}?success=true|false&error=... +# Authorization: Bearer {token} +# Empty url = check and log only, which is a valid state and not an error. +healthcheck_push_url: "" +healthcheck_push_token: "" + +healthcheck_script_dir: /usr/local/bin +healthcheck_log_dir: /var/log/healthchecks + +# Per-check knobs, consumed by the templates under checks/ +healthcheck_disk_threshold: 85 # percent +healthcheck_cpu_temp_threshold: 80 # celsius +healthcheck_zfs_pool: "" +healthcheck_ups_name: "" diff --git a/ansible/roles/healthcheck/tasks/main.yml b/ansible/roles/healthcheck/tasks/main.yml new file mode 100644 index 0000000..fb0257b --- /dev/null +++ b/ansible/roles/healthcheck/tasks/main.yml @@ -0,0 +1,78 @@ +--- +- name: "Assert healthcheck '{{ healthcheck_name }}' is fully specified" + ansible.builtin.assert: + that: + - healthcheck_name | length > 0 + - healthcheck_description | length > 0 + - (healthcheck_check | length > 0) != (healthcheck_command | length > 0) + - not (healthcheck_push_url | length > 0) or (healthcheck_push_token | length > 0) + fail_msg: >- + healthcheck needs a name, a description, exactly one of healthcheck_check + or healthcheck_command, and a token whenever a push URL is set. A push URL + with no token would report to Gatus and be rejected 401 on every run. + +- name: Install healthcheck dependencies + ansible.builtin.package: + name: "{{ healthcheck_packages | default(['curl']) }}" + state: present + +- name: Create the healthcheck log directory + ansible.builtin.file: + path: "{{ healthcheck_log_dir }}" + state: directory + owner: root + group: root + mode: "0750" + +- name: "Install the {{ healthcheck_name }} check script" + ansible.builtin.template: + src: healthcheck.sh.j2 + dest: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" + owner: root + group: root + mode: "0755" + +# The token is in this unit file, so it must not be world-readable. +- name: "Install the {{ healthcheck_name }} systemd service" + ansible.builtin.template: + src: healthcheck.service.j2 + dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.service" + owner: root + group: root + mode: "0600" + +- name: "Install the {{ healthcheck_name }} systemd timer" + ansible.builtin.template: + src: healthcheck.timer.j2 + dest: "/etc/systemd/system/{{ healthcheck_name }}-healthcheck.timer" + owner: root + group: root + mode: "0644" + +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: yes + +# `restarted`, not `started`: started is a no-op on an already-active timer, so +# a changed interval or a stuck timer would never be picked up. +- name: "Enable and start the {{ healthcheck_name }} timer" + ansible.builtin.systemd: + name: "{{ healthcheck_name }}-healthcheck.timer" + enabled: yes + state: restarted + daemon_reload: yes + +- name: "Run the {{ healthcheck_name }} check once now" + ansible.builtin.command: "{{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh" + environment: + HEALTHCHECK_PUSH_URL: "{{ healthcheck_push_url }}" + HEALTHCHECK_PUSH_TOKEN: "{{ healthcheck_push_token }}" + register: healthcheck_first_run + changed_when: false + failed_when: false + +- name: "Report the first {{ healthcheck_name }} result" + ansible.builtin.debug: + msg: >- + {{ healthcheck_name }}: {{ 'HEALTHY' if healthcheck_first_run.rc == 0 + else 'UNHEALTHY (rc=' ~ healthcheck_first_run.rc ~ ')' }} diff --git a/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 b/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 new file mode 100644 index 0000000..cf57509 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/cpu-temp.sh.j2 @@ -0,0 +1,28 @@ + # Hottest core across every thermal zone and hwmon sensor lm-sensors knows + # about. Reading the hottest rather than an average is deliberate: one core + # throttling is a real problem that an average hides. + local threshold={{ healthcheck_cpu_temp_threshold }} + local hottest=0 label="" + + while read -r t; do + [ -z "$t" ] && continue + t=${t%.*} + if [ "$t" -gt "$hottest" ]; then hottest=$t; fi + done < <(sensors -u 2>/dev/null | awk '/_input:/ && /temp/ {print $2}') + + # Fall back to the kernel thermal zones if lm-sensors reports nothing. + if [ "$hottest" -eq 0 ]; then + for z in /sys/class/thermal/thermal_zone*/temp; do + [ -r "$z" ] || continue + local milli; milli=$(cat "$z" 2>/dev/null) || continue + local c=$((milli / 1000)) + if [ "$c" -gt "$hottest" ]; then hottest=$c; label=$(cat "${z%/temp}/type" 2>/dev/null); fi + done + fi + + if [ "$hottest" -eq 0 ]; then + MESSAGE="no temperature sensors readable" + return 1 + fi + MESSAGE="${hottest}C${label:+ (${label})}" + [ "$hottest" -lt "$threshold" ] diff --git a/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 b/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 new file mode 100644 index 0000000..b0e76ce --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/disk-usage.sh.j2 @@ -0,0 +1,19 @@ + # Every real filesystem must be under the threshold. tmpfs, devtmpfs, + # squashfs and overlay are excluded: they are either RAM, read-only, or + # container layers, and none of them fills up in a way an operator can act on. + local threshold={{ healthcheck_disk_threshold }} + local worst=0 worst_mount="" over="" + + while read -r pct mount; do + pct=${pct%\%} + [ -z "$pct" ] && continue + if [ "$pct" -gt "$worst" ]; then worst=$pct; worst_mount=$mount; fi + if [ "$pct" -ge "$threshold" ]; then over="${over}${over:+, }${mount} ${pct}%"; fi + done < <(df -P -x tmpfs -x devtmpfs -x squashfs -x overlay --output=pcent,target 2>/dev/null | tail -n +2) + + if [ -n "$over" ]; then + MESSAGE="over ${threshold}%: ${over}" + return 1 + fi + MESSAGE="max ${worst}% on ${worst_mount:-/}" + return 0 diff --git a/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 b/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 new file mode 100644 index 0000000..141e478 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/liveness.sh.j2 @@ -0,0 +1,6 @@ + # Liveness has no test to run: the fact that this script executed at all is + # the signal. What proves the host is alive is the PUSH arriving at Gatus, + # and what detects the host being dead is the heartbeat window expiring with + # no push. So this always succeeds - the reporting is the check. + MESSAGE="up since $(uptime -p 2>/dev/null || echo unknown)" + return 0 diff --git a/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 b/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 new file mode 100644 index 0000000..508c860 --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/ups-status.sh.j2 @@ -0,0 +1,18 @@ + # OL means on line power. Anything else - OB (on battery), LB (low battery), + # or no answer at all - is a failure worth waking up for, because the + # hypervisor has a finite number of minutes left. + local ups="{{ healthcheck_ups_name }}" + local status charge runtime load + + status=$(upsc "${ups}@localhost" ups.status 2>/dev/null) + if [ -z "$status" ]; then + MESSAGE="cannot reach UPS ${ups} via upsd" + return 1 + fi + + charge=$(upsc "${ups}@localhost" battery.charge 2>/dev/null) + runtime=$(upsc "${ups}@localhost" battery.runtime 2>/dev/null) + load=$(upsc "${ups}@localhost" ups.load 2>/dev/null) + + MESSAGE="status=${status} charge=${charge}% runtime=${runtime}s load=${load}%" + [[ "$status" == *"OL"* ]] diff --git a/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 b/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 new file mode 100644 index 0000000..b6c7dae --- /dev/null +++ b/ansible/roles/healthcheck/templates/checks/zfs-health.sh.j2 @@ -0,0 +1,52 @@ + # Five conditions, all of which have to hold. Ported from the check that + # infra/nodito/32_zfs_pool_setup_playbook.yml deployed, which was correct - + # only its reporting was tied to Uptime Kuma. + local pool="{{ healthcheck_zfs_pool }}" + local json issues="" + + json=$(zpool status -j "$pool" 2>&1) || { MESSAGE="zpool status failed: ${json}"; return 1; } + + # 1. pool state + local state + state=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].state') + [ "$state" = "ONLINE" ] || issues="${issues}${issues:+; }pool ${state}" + + # 2. every vdev and device ONLINE + local bad + bad=$(echo "$json" | jq -r --arg p "$pool" ' + .pools[$p].vdevs[] | .. | objects + | select(.state? and .state != "ONLINE") + | "\(.name // "unknown"):\(.state)"' 2>/dev/null | paste -sd, -) + [ -z "$bad" ] || issues="${issues}${issues:+; }devices ${bad}" + + # 3. resilver in progress + local fn st + fn=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.function // "NONE"') + st=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.state // "NONE"') + if [ "$fn" = "RESILVER" ] && [ "$st" = "SCANNING" ]; then + issues="${issues}${issues:+; }resilvering" + fi + + # 4. read/write/checksum errors. ZFS reports these as strings. + local errs + errs=$(echo "$json" | jq -r --arg p "$pool" ' + .pools[$p].vdevs[] | .. | objects + | select(.name? and ((.read_errors // "0" | tonumber) > 0 + or (.write_errors // "0" | tonumber) > 0 + or (.checksum_errors // "0" | tonumber) > 0)) + | "\(.name) r=\(.read_errors) w=\(.write_errors) c=\(.checksum_errors)"' 2>/dev/null | paste -sd, -) + [ -z "$errs" ] || issues="${issues}${issues:+; }errors ${errs}" + + # 5. errors from the last scrub + local scan_err + scan_err=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.errors // "0"') + if [ -n "$scan_err" ] && [ "$scan_err" != "0" ] && [ "$scan_err" != "null" ]; then + issues="${issues}${issues:+; }scan errors ${scan_err}" + fi + + if [ -n "$issues" ]; then MESSAGE="$issues"; return 1; fi + + local scrub + scrub=$(echo "$json" | jq -r --arg p "$pool" '.pools[$p].scan_stats.start_time // "never"') + MESSAGE="${pool} ONLINE, last scrub ${scrub}" + return 0 diff --git a/ansible/roles/healthcheck/templates/healthcheck.service.j2 b/ansible/roles/healthcheck/templates/healthcheck.service.j2 new file mode 100644 index 0000000..574e37c --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description={{ healthcheck_description }} +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +ExecStart={{ healthcheck_script_dir }}/{{ healthcheck_name }}-healthcheck.sh +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +Environment=HEALTHCHECK_PUSH_TOKEN={{ healthcheck_push_token }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/healthcheck/templates/healthcheck.sh.j2 b/ansible/roles/healthcheck/templates/healthcheck.sh.j2 new file mode 100644 index 0000000..92c9d27 --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.sh.j2 @@ -0,0 +1,68 @@ +#!/bin/bash +# {{ healthcheck_name }} — {{ healthcheck_description }} +# Managed by Ansible (roles/healthcheck). Do not edit on the host. +# +# The exit code is the real answer; systemd stores it: +# systemctl is-failed {{ healthcheck_name }}-healthcheck.service +# The push below is an optional extra, and having no URL is normal. + +set -uo pipefail + +LOG_FILE="{{ healthcheck_log_dir }}/{{ healthcheck_name }}.log" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +PUSH_TOKEN="${HEALTHCHECK_PUSH_TOKEN:-}" + +log() { echo "$(date '+%Y-%m-%d %H:%M:%S') - $*" >> "$LOG_FILE"; } + +# Report to Gatus as an external endpoint. Note this is a POST with a bearer +# token, not a GET with a query string - it is not the shape Uptime Kuma used. +report() { + local success="$1" message="$2" + + # No push URL is normal, not an error: the exit code below is a complete + # answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + local encoded + encoded=$(printf '%s' "$message" | sed 's/%/%25/g; s/ /%20/g; s/&/%26/g; s/+/%2B/g; s/#/%23/g') + + local code + code=$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + --max-time 15 --retry 2 --retry-delay 3 \ + -H "Authorization: Bearer ${PUSH_TOKEN}" \ + "${PUSH_URL}?success=${success}&error=${encoded}" 2>/dev/null) + + if [ "$code" = "200" ]; then + log "reported success=${success}" + else + log "ERROR: report failed (HTTP ${code})" + return 1 + fi +} + +# ── the check ──────────────────────────────────────────────────────────────── +# +# The blank line before the closing brace below is load-bearing. Jinja strips an +# included template's trailing newline, and trim_blocks (on by default in +# Ansible) then eats the newline after {% raw %}{% endif %}{% endraw %} - so without it the brace lands +# on the same line as the check body's last statement, producing `return 0}` and +# a script that dies with "syntax error: unexpected end of file". +check() { +{% if healthcheck_check %} +{% include 'checks/' ~ healthcheck_check ~ '.sh.j2' %} +{% else %} + {{ healthcheck_command }} +{% endif %} + +} + +MESSAGE="" +if check; then + log "OK${MESSAGE:+ - $MESSAGE}" + report "true" "${MESSAGE:-ok}" + exit 0 +else + log "FAILED${MESSAGE:+ - $MESSAGE}" + report "false" "${MESSAGE:-check failed}" + exit 1 +fi diff --git a/ansible/roles/healthcheck/templates/healthcheck.timer.j2 b/ansible/roles/healthcheck/templates/healthcheck.timer.j2 new file mode 100644 index 0000000..0271735 --- /dev/null +++ b/ansible/roles/healthcheck/templates/healthcheck.timer.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=Run {{ healthcheck_description }} +Requires={{ healthcheck_name }}-healthcheck.service + +[Timer] +OnBootSec={{ healthcheck_boot_delay }} +{% if healthcheck_on_calendar %} +OnCalendar={{ healthcheck_on_calendar }} +{% else %} +OnUnitActiveSec={{ healthcheck_interval }} +{% endif %} +# Run a missed occurrence on the next boot rather than silently skipping it. +Persistent=true +# Spread the pushes out so twelve hosts do not all report in the same second. +RandomizedDelaySec={{ healthcheck_randomized_delay | default('30') }} + +[Install] +WantedBy=timers.target