diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml index 068a490..76eb7bf 100644 --- a/ansible/group_vars/all/vault.yml +++ b/ansible/group_vars/all/vault.yml @@ -1,154 +1,164 @@ $ANSIBLE_VAULT;1.1;AES256 -38396132363836373161663662323635613764643231316132333265396363353833383839613233 -3564323734323561366536303932636435373339363233650a343964613465353137396663666363 -63646331366162336537333035386263623264383566646334613332376561613231643235313565 -3638326434613462640a653364646361626636376464633734343332393030393232616362633532 -61313161643939383930303362363966326161313237346565323532626135643361633936313135 -38396534616239383735353532613834373564336537323531303533323434336166326263376238 -35303032336237666235396562313738363130326633336338306632353161336366346332393465 -61396163366432323335393362613539313661333431656465336662323336393330373332383834 -39636230313566663630656531323530653130643362363562326465643665326335323036346237 -32303439356432316564313730653332323731343235336535373462326162656638383735396135 -36633264613136323866643162353161306334666463303565333936653861643635336162623831 -65386231313465363864623035623364303039323961363237393637303133386530626266663435 -37336662643632666164386264646330363135623565306433646437333535363839663330663533 -64373231386231643863303933326132646364333339326431646531643962356663326363643734 -66346239663438363666613636336430336465623866643861666564353739623238376135346431 -38346234333364646230616632646561643638656535306630326332626136396538333534653061 -33643562326436636664343339643834323466313131323462653234303834373635313831353538 -35333739346563646163333937643232343866383337653462653530666165643361303332363364 -36326333333234343761386234666534323430323734616266326231386136336463636231346237 -33356361333562656666393630323530333731613964643966383065663635613734383066333532 -31366533373533656666666336363666313839316133333062356432633038366363623935373566 -33356563663933653539386263663530363239613461383831646262353165343638326663383364 -37386330356539343131353633633233323738396339336230666430363965353536343031646163 -38306162643733326563373835396264343433666335366136353439356333646131373764336438 -61393836303038306132383363633838333665343665303136316131616335643337313362306661 -37373137653830636635366266323538393338333463333963343931343433323437663661313366 -36616164346265626637616665663461336163343666623162663637633037666564363533623833 -35386537306366636533633930326339343136303839306539393866306462303632363163313535 -64326530323866313539623661636266663933343165653662373331663334376333393834303938 -30373233323664663935343364363731636334343065643662663266343366353031336439383766 -39313862616230346661313632613662613335666234333432336262663033633437383565383336 -35316365643030616331643333356361373236643731643733386237666430366566303933326563 -34336662623431333539616337303739373035313165396535393435383235313633616632616261 -37383633306235633939326339303861346335313361316438373630383863643864376462373464 -64663631656165316664633336333634643939333464303266366438633039653038356439393565 -35343237303533363032396365346232616131306666353766386433393737626639656139373332 -65346336373936393866666236346262623666666435363065386563613431626537636364396238 -64373133653132366538626262313434353930366338383866653139376538666136383363333666 -39636538323437616633356337626130356164363663333637333232663435303537336233643135 -64613638643564653431386634376538643564333438343835376633626230303264316337376434 -66316231386533313364343131316162623334373065626533303239383439326134343830376438 -34333831333035383830386436643239383138633166303033363563313562613564353538373536 -33343962346439616439366463383236373437613132326162323437343231666231396332623038 -37643932323538633365393033386462663735333034326332663461636363613932346637623638 -37323534613439373865646139656462303266373636356630306431323065656533663838303765 -30306534343965366566663337313263643832633861336334313835643738656334653135626563 -64353062343561626665613162346236393731623533663431333866613932653331613361333039 -65336337666264386334373039343965323065626334353030336336396136633363323339313031 -62633932613934623065306638656135393438383361376465653431316261633633353031323066 -32323466323861316338343137306333326137376434363735393664353761303265653165316634 -66623866373730646138333162663830363063326231653636643633623936313164363065396661 -64336263333234356263636630666264393734343732613231333331616230333662356164393539 -37616533356566323936386331623133666335613765303732313439376461363837623136386638 -38313666633436316336396337643666623735396562326538616564363066356136306632313138 -36643333346433366233636463626538313465666334373561666433396137323532366662393363 -62313434346636386566656437313130663836306265326463383264646564643538333738313436 -65343531356138353632346434616134316261306162636439666666326439373836333761333332 -30323836633438383166633364366365363561333731306435363433623333316665646334623061 -34396131666438613938343364663237393538313632356166346432346530386131623435643566 -30613233313739323562373332343537323665623330353135373735663035376138383931333938 -30363066303932643465633534663863336339356431393335343261333235396130616165636538 -32643335373137373638636537373635616133646333653230376633656561323533376136376564 -61306563653736656364383935326635363661636663626530313938646130356132373037373135 -64373034656232623530386435633838363262376464376361323161343163373561393839336165 -34323137363837616466326364376336646636396364393464626565623934636231643161343032 -61616663626130313764393735633866393566633533343737633035623138613662313335656136 -61303132333935613236336164396330383136303164633033356638643835313065393836326236 -36356235623838656534363639343261623939633864663631343363663262396231343562396530 -30653165663332626434636231303637356538666337643361653338303330633932643763323034 -66366538303939323232383631383562613732313065323933343761656535366134323164396432 -65663339366631623738323135633965623966613433376132363930383061613434386537373332 -39623136646638393835646163656364316665646464356535343839393966363631383734633536 -62393832346632323330356365393133346134653837663165353834313931356236373664656234 -39363431646239323665386639646165303836303731383063663266373236343436633163636535 -31393537653962303236383930613339613232643864356230616664313038396266333635396436 -36373930646435333863626333333439626663663930663730323236666233386231303730653132 -36613763303739376135653531326637313437623338343232303464303639633662373138326134 -33616632373031303139313439383465336565323837656632336532373737396230613837376564 -63663334363636666534616265623062366132376234326132393732336563313165386334626264 -34373934323930663832623338393964353964316132376262353662643534303436643537646361 -31663931613164646435656462613433356366633862636461633938343039353531633364373232 -31623131616534653533656566613038346133343266373637346663373739656161343765666636 -61656466636563643464616565643837623238636466316538636565626261313165346238306261 -62346264313739666338376439666633363033336435323932613664373234623931323962393035 -33313639626438656134363565396361613965376662386139366163383238346630623064343634 -38633962323232666636303830663538333737376437393465626338643632396238353863646263 -61363764346330663866396337626237653630633335363436613863613231356262383039613831 -62356465626564386537373134663536396231326136376234393966303065333264376630316633 -66303262383237386632656231663835383534326333633862383033356263306436623966383161 -65663530303836653235616466333736636463323065626335356333373334376337653637666133 -30363232383533646331323030303434633162643733623339346132666564396663343131303233 -62373431633635356235396130393330643463663239616365663930366138393930343334343432 -31616431666361353636316163313430633730366462616332333830353937616466343265343165 -36366265626363373164373463353561323437303936393337336238393137633931323537653666 -66346231363636326636333736383637376535393961303731386535386138633462323030346330 -64353936383162356433306164616264366164623731373733626261653238306362623264666633 -64316366393765356632323966343634653235636162353636306337383864326264343434613961 -65656433323164363738313535636637383365636435343933333265653333326536363864303866 -38396263626166316537326534363637393132363334616134653036663830343764303737383663 -31616639373330363365336231333938303138623165666531336336353133653263313263343231 -39666639383931653231373535396365386637643738346438333961393138633961376134303664 -39336331646236396634356337363336316537643133653561363937373963636238323837303833 -33396164656434666239393163363638373364383839633364393137313765613531626337316532 -33386465613034663630323638653839666264373235663364643461333238376138303765643364 -63623764363830356264306363613266393930376564616438636438373063323030633032326333 -66316235643036376165323035386164633536396365326333346638346133336235616132663532 -38396636376539656332613963646264316637616235316131316232623131386662333164333534 -31366534303166383062316263393833326165346333623830653266646637663364373365626335 -61623837323934343131353336393163303831613832303530356639393465633461326566363138 -39363730353862383365386339353536366537363933663566306132626135336339646533323763 -65616664393166666461643038336232303934306338323331366233643061373532623163663931 -37343839346637376337613438616131633239623234613539323138353233343238653661626462 -34353666643230633562323631303732373332616335326131396131653038643531646562646431 -33626130343834396439323138393162363831393534383839663638663838633061666534336138 -35313234653565316437336439376563666138653661613062643262656233316262343736363235 -32623466626366343965373638323239393933386434633531303363313061313832663166653634 -39333835303066303231363663666461323434363237646133323661386663633939313965363864 -38303861383565303766306533633132376632373234646338306264343065633631303935313266 -30333836616164363735363934386165663839393731633039383538623833353632373636656436 -62343934363834613464626364303434393262373365643130373437393730393964316130613565 -35373938306436666364646136306166663132393736383862623635316265386361663333356431 -35336363623535336631366531316661333130393034383566613138653834383333623437356139 -36353434626531616361333138386338636132646533326565336534643632316363313732336233 -38656134383565303132643161393032333334316265343862666532303438653161626665326632 -31343435393136313865356166363165613730373633623963303633343165613230393430633831 -31376361656361666434343866353037333834353735396566643935393439343334376461323133 -61663835656137356665623265376664336332313432636664313735643739626333383566336331 -35306539386433633636356636383734636633666137653334333162386564643464306538366461 -33323130656331313834323839303736366436633634356261363335343231303331353339313031 -61613563383937623162616133313633653833666634643566643766336533303566383561633237 -61316562353933333666323037376334326430623035643630323565386532313530383835343265 -61616165396664616332313262666534396537356564656236313864326533306338373362613034 -38623162336138366262626538366333373830623634616261353764666563663138663862656339 -65346137646263393064393536613963666434373663636336373531316464336264313061316439 -39623832666536356334383261653431343837663163623035366162396465646239613130643837 -64393735333438376262323834313330363434366163653833376334666666653535633330636535 -66346665643332393235313062313164333030353265386632326431333036333861356231666531 -63323966623361343363623561386633363734323864623961393832653037353766326637663735 -63396565326632303366396539396435626235643833323864663731396636313237373832646566 -66333839643734303734376338353037393266353232316264353736346338646262386636616335 -38313838363064643739633065343535643762626165343862646362613462386532356166383437 -38646130313462393863376339336436336630643061363130663831313532663466363936343934 -31656230323564303638653832626364633631336336613663356562636436643336323238393436 -63343462366436653365383766366262373332313339626431326538333562346438303730343937 -33326365636631633365336332636638353663303633623137643764303930656336323963643835 -34376136353532613938326432663835363530653264383561353533663863376661636137336437 -61353365333661623038363535643035393335613632656330333938623335343436663763646362 -63633130373530386235663661613335323531363234343465313861656639646461613936336134 -30613937376666623761626438396635313563306564633433613330633664356636636565613166 -32633430646139383362623133346435633063633466656635333034343937643639346536393134 -39383664383234643265323563646465373636303261646331366439666466343636333330633131 -37363963373361613331663263623737373464663831326630663432316336336163343866663834 -6630 +34383033613438623432656438313763613639313139626432303637383363643339613166626665 +6537663336353661373437303636613364643736663031300a623461353135623537343763383333 +37323731646262613436646231656263396532616132316237353762376363383062646132303435 +3435316666316235660a323163613932616661303935336633656332353766393536393736356539 +61396164386266303464373262373862363361363365363235383363613335663933313062663565 +39613466353161356462656539613335613536363532333431393935303430373435373737363537 +62383266323339373062313134316464303263313830646161333530383736336637333865373764 +31313165396366653763343466643561383236643836616662383439666165343139326232326364 +32633133373863616163343365656231353939366435346534383462396134653064663566343566 +66343464373865383864313264646635316233336133346365396634626261613561373965643339 +30353831393037316334313632376537393361336366326561343832373531303139323937613531 +34386164303430316162363536386137316330336638303365393462386465633862663932376563 +34323862636435306563333065383134343362333733623066656439613339376666353331373665 +32353938663364313036373237323062393430306661633732356434333232376338313835663330 +32306536393666323464663238306562613936376236613935616664653865373330626365626330 +64613066653632646638306338393331376430383165316363323437373362613866313962663432 +62356663636630356239613831353066303530363966396135376134633936313530643262353965 +38383330613962313935366633353635363638303937316362303866343433353437363334396431 +32393134626133353564643233616632376633303934653065613262353436333533376137303862 +35303861383833326537376433336634633730373833313036643365373431666438653033653930 +33346230396661636333613031353133353631306331666362386366633062666534303539316238 +39366634656333646634643032393738333330636230663366623265666533356534623465316665 +35336535326237343231653661613736626239316262386363666264613636326332366232333964 +38353635616662663133376534306366643037613732343233373336653166316330643638353539 +35353532643534613864333839393365313439343663333337373639633832383630316164306132 +63636162373034363839656638353534613733663737626432636436373164303239316436313566 +66623466396663353136323565643262383865323830616232373466373431373261306537646463 +61353966623062356438626635663461396165343366393132393965383734633632666335313030 +37333631373239303231396264393031373162373936303462363736313538343562343737356332 +63663762633032356130303964386664366137373432353533386566613037613163383963623732 +32343765376535343630656633333362393765666131653632613361363163383038323138333537 +37373333316466316562613439356365316163336366623435666166653835616531653563343664 +30303464663838646333373837363961356465613234636138633162366636633030383337643131 +62636236663066316630396132333934353139383465363034363033656161383663656462313939 +62353738396132343734343363636164323163616237613861643862303433633934656165343263 +35313737373333386335663731643664393630653839396464376639306231653934386336353630 +61393735336662663435613431386361373561363531643232303831363163636139653538393931 +66313965646562616237383439343637623835303065333730613865666638383131616261346463 +31396464306537613135653830653138323539393731626264326335336432666333333735646534 +30616439393464326631633635363466303336613135346231626232313361303666323661616638 +64363937343231363536353034363966326132333734386638613737636130646232363666643633 +61353563636466356630346232613761636432303430336461663434333636623962643764336639 +35303565613861323431393036303534303061666437366538306236373930313439353330313632 +61363363303562316664303065613863613339626632643931386438326330373938613762303334 +33316430363931393262373661623137633835656136313235613666613932313236343966306331 +36343031303632323432366337633637336335343638393564313738386162386164613339343662 +62383939663265373265613932633265626263623939303638383838646531343433393864363235 +37313761396436623833376435393537376136373162393465343764326533666533393061643965 +61306365346562333737313962633764313232323161623861336566343735353737633539656162 +34323939306661373662663331653333346264323930663636633134666532623438323537383165 +66343033363962633766303331623866306462623235373838616565653066646132633034363535 +31613462356361613535383963643662373162363363653334393937313039666266366363653537 +64356166396139393262313565303731346534646462333638316661383139346362333364613466 +38303561316263636136303431393366343936653161336238613439366266346136636465303337 +61373634376464333037303862616335623031656133636165636264386265643261373735663266 +39396265366138363430343035356337303438346165366361316230656239326633653537626339 +39613334316563376133343336343563363564653237633764393532336437303334363830396635 +31333963666462313830333163646465356337643263636462356363613266323630393866626535 +35336436663063333364353663616564626535396133386461326433323936323161613633353539 +66323935626433353162666335613839346561343264323763303034613366663233613037343538 +66376236373462383430313030373235373337653338333033306431616562353435656433356365 +37653063363435366532306334373438306632306435653334396630343863623938666335656164 +37386634366430383762316435396237313362313965353634613266616532633465646464313662 +35303433363236393465396161346431306231666432666531626136623966663864376663373562 +37653239633732346536323265643434616136326561666330633435383264333937353366653736 +61353430633363323263323163646639663330663632343038623964396433626337353232373334 +32343938356539303938666137323831646562393233323733336464316165623232326132316362 +61313631626234336164656533616231393663363364623130313439623466396231383537656562 +35393163323131383838356130643863633736613635306635306135666261623731613239303962 +61373432336638353339646665653561653738353731373032366361313562306466373365653333 +35306236336331333961613864346165613763383266613236343066623432303766333434333331 +66376263363132616132366165393235626336323665373135653462653533376138346632393363 +35396165383966326635333134663138366136336430393532373935373264616530613762386430 +37643862316232613261343430616438333831623835663239656531356666613032653837373531 +30386232643537306530323936386561646265616339376265353763663833623539653831656634 +63626564633964343663656435636635656562666236386639353332356566316631643431353330 +32643238623932643732353332633535643738663066383830376432346463393464333236386163 +64376165333266323166613564313061643832656239333165393035343836376434626162643061 +31373861393534636236383765313834356638613332626666653762343537363339383939666263 +61366230623930643534353565316165656132396664393138326139613536343738396330653735 +37623463343834626133313231393130646134363331393238663065643930383237326439343831 +32393536613361616637373761393361303936363863333838666633343063386235316237323531 +62633362316436623761616366383336613765373362393161303765613464643632666462343565 +63616430396164346263666436336463623638313130656231643131336663336565616135373433 +39326533643733333564373434333738343634383033353036646534646632383730613161393130 +62653836376435663435353739373536366466646332653030653332353761626466353631663163 +38613734383634663662376464653733306564313761336566336336656366653536643834653631 +35333963306562393234623039613338643266333762333236366331333431343830393864623830 +34323439386133613035363861623332663933313734313637623739613163663730316134303434 +35363239363836666463366334356435666632396132306163303037663430383535393934323537 +30353266373864346662333035393761646632313032383738346439343232663666343833333565 +35363564623038376666363436663830356165323234653230636264303066666230643632356237 +66633734346633366533623966393462313438333965623033343337323135353730613561363236 +30636333613034626162393137323762306432663761393766303163393536656338366664643432 +34636630323965396166356538623165663335333739323034333237306531393734306138343436 +36613937373932633234303336373530663533646635396664343935326334333239646534343336 +35396666353938326337646431353965653836323731313861613031356561376262363965623934 +65336164623134366365303062393161353965363937646363396564313138323938656330333964 +64663862363766613630333531623463616331363533343962653164376665303463336134346662 +66376238663632663932656230323638323135653666363736643065336330636236323064386361 +37646631303138616565383665663634333338613336346438366333636165366635313337373635 +62656236653961393061373262326534393339363236353336613339656462336661663739396261 +35363636653237316436656662633462306333313336313865333037636661383731336466666536 +36646537653431666261353232653233653930346538383438633466346636656432383963333765 +62346363633265306665366630383831303434336139343837616361666539656262633735363064 +30376138386531336430613563366666333964663231363465326230316233323532366632663930 +63653164373931353861653833326130643365373530663039363166366331303134366336613438 +63633336363539636265396331653630633162316366316130326531363463616464376635376538 +32376638353337343639633434613663666437313564333264383766623561393933333561663235 +37666163626538353736383435373530663438363636643665626266316338336532623235353337 +61653762383630616266353438386137653137616532633066623165383663646437343565366237 +33333463613539346136613135386466656135616136663036313661323738363066633038656139 +38353230313861666463383737303038613731316334333031643438343463666164633536313031 +39613663376436643165333362636462303230656336333234363336633262336363643736336231 +66316630613762383832613836333263303038306235663662323162633932653733626130356665 +65316630346534356261373961353130393334396365376237353439326261336339393336303030 +63323439666439313831626639323735653730356565643938376564316337613333616664643865 +30633530383865663731383635366566613565393632336337306334663937396135303539363536 +65656331386532306438383031313164626237393762303636663831666239336637666465633630 +63383961336333663537666561376532376631343132363638633364616531306363623432396332 +31343864626335613131653762363338393738343531626330663663376531336230656365363661 +64363530333965386439376161313534333435393966656231613161633034303964653564363436 +63396131623562303962383137666439373132633330363237353031306561306362666561376435 +66383535333233663736363762323662363264393365643135323834366565653932313863353437 +38363536323739336165356561613335653464363637636234313637383066633865353239393961 +31613532373361383062393238303430353662383930373764373237383035633364323661623661 +62653661623234366666616532663563316234663137616338393036333937633963323635366165 +61613032363531666338353132346365323661626265626461396362646338373536306362303338 +31646564343434633433663039366533656665643235336136333134366231376336343465336539 +37363139346239666232313366613432646265626564313838346364306133306434313337626562 +32346430316532623037633838303363346130623636386134313566383333613565316135373264 +38393261313435323433326130303032393538333963646430646366653363613830613534353832 +32643434613964613239363966376138376661636335373130356430336631333461623734383535 +35356461326136636234663738343738303064386536336462303632303461303733333666383036 +38383563393739343134363038623638393766316533373163336439336238613734656132623163 +32626564666430663339316163393934323133306238353562323866633738363737643937376435 +66306430623334643766623564363239346361393666663766306637313265313833396435626234 +35366239616132623863343365663833363934316362636638303536656631633364646235366566 +39386637613735373761643339396132323031323438316633363464336636316534643435323861 +62663732366430646538626663313035616235643537643234356434323635373962343633336266 +30323031306438656537626430356231393466313334633934623632663661306361313330633339 +33396234386536336331616161363132343765306562303932313963623037366633383765663134 +62636466636433666530363930633765626531613539363832313361666565323635326334326533 +36336439363836663033643262336139613437623633616138656564393032393263393132303031 +31626336663039633362366663306662333432643261613464303939326562653562346239383261 +37373966383635626433363831653936333964626262363839383936356634633233336365623765 +62313031353538636464303234383865613932323164386336316362323731303263346637346534 +32353734343130343364646161643431353230336364366330363261326334613936633234646264 +39623163346663613431323630303034383761663835373565663166366239303130386139396236 +62393565353761636636366462353038366664663430616332326465373364396264323064666536 +61653035633562316634336263636332363733663666626131383161656535383133613939623034 +66363632613733306336323165336534346337386163656631343332323636353539353737316131 +34616336343865656565646338653037333838663736333330376330663834373138363739633064 +61646439636365323838376131663266636235333062616532613936616339633661303634323531 +63356335386534353639323566376134373565353137333134363761666532366431633634333430 +39326662633765626230326130386333393463323433366162363432613234336634663439313339 +33653737333830336264343839396563316462613032376335356634383436353962613636333866 +32333032623635323430356636623138366635386533646133626164316438393937626462313239 +31636635363064363763396566306234363965346438653738333961623435303233396634643763 +65343763386464366636383734336335646464306262623363303934393734333666636635356365 +65353931303862343765343665373139376530656263356365366333323135383838383030613166 +30303963383833643937636539656164336537383337346436303534303035313935323633663765 +333464336235373230316337333632326138 diff --git a/ansible/infra/400_host_monitoring.yml b/ansible/infra/400_host_monitoring.yml index ac81289..3f6b119 100644 --- a/ansible/infra/400_host_monitoring.yml +++ b/ansible/infra/400_host_monitoring.yml @@ -34,6 +34,23 @@ # slow apt run, a reboot - does not raise an alarm, but a check that has # genuinely stopped does. # ───────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── - name: Register the host checks with Gatus hosts: observability become: yes @@ -47,7 +64,7 @@ 'name': item, 'group': 'liveness', 'token': gatus_push_tokens[item], - 'heartbeat': '16m'}] }}" + 'heartbeat': '11m'}] }}" loop: "{{ monitored }}" - name: Build the disk endpoint list @@ -56,13 +73,20 @@ 'name': item, 'group': 'disk', 'token': gatus_push_tokens[item], - 'heartbeat': '30h'}] }}" + 'heartbeat': '7h'}] }}" loop: "{{ monitored }}" - name: Register liveness endpoints ansible.builtin.include_role: name: gatus_endpoint vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h gatus_endpoint_name: liveness gatus_endpoint_external: "{{ liveness_endpoints }}" @@ -70,6 +94,13 @@ ansible.builtin.include_role: name: gatus_endpoint vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h gatus_endpoint_name: disk gatus_endpoint_external: "{{ disk_endpoints }}" @@ -77,20 +108,27 @@ ansible.builtin.include_role: name: gatus_endpoint vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h gatus_endpoint_name: hypervisor gatus_endpoint_external: - name: cpu group: hypervisor token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "16m" + heartbeat: "11m" - name: zfs group: hypervisor token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "30h" + heartbeat: "7h" - name: ups group: hypervisor token: "{{ gatus_push_tokens['nodito'] }}" - heartbeat: "16m" + heartbeat: "11m" - name: Deploy host liveness and disk checks hosts: managed @@ -120,10 +158,13 @@ healthcheck_name: disk-usage healthcheck_description: "Disk usage for {{ inventory_hostname }}" healthcheck_check: disk-usage - # Daily. RandomizedDelaySec spreads twelve hosts across the hour rather - # than having them all report in the same second. - healthcheck_on_calendar: "*-*-* 07:00:00" - healthcheck_randomized_delay: "3600" + # Every 6h rather than daily. Disk usage itself moves slowly, but the + # heartbeat can only be as tight as the push frequency - a daily push + # forces a >24h window, and a stuck check then hides for a day and a + # half. Six-hourly buys a 7h window. RandomizedDelaySec spreads the + # hosts so twelve boxes do not all report in the same second. + healthcheck_on_calendar: "*-*-* 00/6:00:00" + healthcheck_randomized_delay: "900" healthcheck_boot_delay: "5min" healthcheck_push_url: "{{ gatus_api }}/disk_{{ host_key }}/external" healthcheck_push_token: "{{ host_token }}" @@ -157,7 +198,7 @@ healthcheck_check: zfs-health healthcheck_packages: [curl, jq] healthcheck_zfs_pool: "{{ zfs_pool_name }}" - healthcheck_on_calendar: "*-*-* 07:20:00" + healthcheck_on_calendar: "*-*-* 00/6:20:00" healthcheck_boot_delay: "10min" healthcheck_push_url: "{{ gatus_api }}/hypervisor_zfs/external" healthcheck_push_token: "{{ host_token }}" diff --git a/ansible/infra/401_service_monitoring.yml b/ansible/infra/401_service_monitoring.yml index f07a906..ab564a1 100644 --- a/ansible/infra/401_service_monitoring.yml +++ b/ansible/infra/401_service_monitoring.yml @@ -1,7 +1,7 @@ --- # Is each systemd-deployed service actually running? # -# Every 5 minutes, with a 16-minute Gatus heartbeat - three missed runs before +# Every 5 minutes, with an 11-minute Gatus heartbeat - one missed run before # it alarms, so a reboot or a slow check does not page anyone, but a host that # stops reporting does. # @@ -28,6 +28,23 @@ # Register one endpoint per unit. Runs first: Gatus reloads within 30s, and the # host play above takes minutes, so every endpoint exists before its first push. # ───────────────────────────────────────────────────────────────────────────── +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── - name: Register the service checks with Gatus hosts: observability become: yes @@ -47,13 +64,20 @@ 'name': (item.0.host | lower | regex_replace('[/_.,# +&]', '-')) ~ '/' ~ item.1, 'group': 'services', 'token': gatus_push_tokens[item.0.host], - 'heartbeat': '16m'}] }}" + 'heartbeat': '11m'}] }}" loop: "{{ host_units | subelements('units') }}" - name: Register the service endpoints ansible.builtin.include_role: name: gatus_endpoint vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h gatus_endpoint_name: services gatus_endpoint_external: "{{ service_endpoints }}" diff --git a/ansible/infra/402_public_monitoring.yml b/ansible/infra/402_public_monitoring.yml index b66d10f..d924468 100644 --- a/ansible/infra/402_public_monitoring.yml +++ b/ansible/infra/402_public_monitoring.yml @@ -75,23 +75,36 @@ 'group': 'domain', 'url': 'https://' ~ item, 'interval': '24h', - 'conditions': ['[DOMAIN_EXPIRATION] > 336h']}] }}" + 'conditions': ['[DOMAIN_EXPIRATION] > 336h'], + 'alerts': [{'type': 'signal', 'failure-threshold': 1, + 'success-threshold': 1, 'send-on-resolved': true, + 'minimum-reminder-interval': '168h'}]}] }}" loop: "{{ monitored_domains }}" # ── DNS ────────────────────────────────────────────────────────────────── + # 6h, not daily: a DNS query is cheap and a wrong record is an outage. The + # domain check stays at 24h because it does a WHOIS/RDAP lookup against a + # free service. Alert on the first failure - at a 6h interval, waiting for + # three would be nearly a day. - name: Build the DNS endpoints ansible.builtin.set_fact: dns_endpoints: "{{ dns_endpoints | default([]) + [{ 'name': item.sub ~ '.' ~ root_domain, 'group': 'dns', 'url': dns_resolver, - 'interval': '24h', + 'interval': '6h', 'dns': {'query-type': 'A', 'query-name': item.sub ~ '.' ~ root_domain}, 'conditions': ['[DNS_RCODE] == NOERROR', - '[BODY] == ' ~ hostvars[item.host].ansible_host]}] }}" + '[BODY] == ' ~ hostvars[item.host].ansible_host], + 'alerts': [{'type': 'signal', 'failure-threshold': 1, + 'success-threshold': 1, 'send-on-resolved': true, + 'minimum-reminder-interval': '24h'}]}] }}" loop: "{{ dns_records }}" # ── Public HTTP ────────────────────────────────────────────────────────── + # failure-threshold 3 at a 5m interval = 15 minutes. A pulled endpoint has + # no heartbeat window, so unlike the push checks the tolerance has to live + # in the threshold - and one failed poll of a public site is usually a blip. - name: Build the public HTTP endpoints ansible.builtin.set_fact: http_endpoints: "{{ http_endpoints | default([]) + [{ @@ -100,7 +113,10 @@ 'url': 'https://' ~ item.sub ~ '.' ~ root_domain ~ item.path, 'interval': '5m', 'conditions': ['[STATUS] == ' ~ item.status, - '[CERTIFICATE_EXPIRATION] > 168h']}] }}" + '[CERTIFICATE_EXPIRATION] > 168h'], + 'alerts': [{'type': 'signal', 'failure-threshold': 3, + 'success-threshold': 2, 'send-on-resolved': true, + 'minimum-reminder-interval': '6h'}]}] }}" loop: "{{ public_sites }}" # ── Public TCP ─────────────────────────────────────────────────────────── @@ -111,7 +127,10 @@ 'group': 'public', 'url': 'tcp://' ~ hostvars[item.host].ansible_host ~ ':' ~ item.port, 'interval': '5m', - 'conditions': ['[CONNECTED] == true']}] }}" + 'conditions': ['[CONNECTED] == true'], + 'alerts': [{'type': 'signal', 'failure-threshold': 3, + 'success-threshold': 2, 'send-on-resolved': true, + 'minimum-reminder-interval': '6h'}]}] }}" loop: "{{ public_tcp }}" - name: Register the public-facing endpoints diff --git a/ansible/infra/403_service_probe_registration.yml b/ansible/infra/403_service_probe_registration.yml index a8c8b35..c92b767 100644 --- a/ansible/infra/403_service_probe_registration.yml +++ b/ansible/infra/403_service_probe_registration.yml @@ -14,6 +14,23 @@ # They used to push to Uptime Kuma. The scripts now POST with a bearer token # instead of GETting ?status=up, and each host uses its own token. +# ───────────────────────────────────────────────────────────────────────────── +# Alerting thresholds, and why they differ by check type. +# +# `failure-threshold` counts CONSECUTIVE failures, but "consecutive" means a +# different amount of wall-clock time per check: +# +# push/heartbeat endpoints a failure is produced once per heartbeat window +# pulled endpoints a failure is produced once per interval +# +# So the default of 3 would mean 33 minutes on an 11m heartbeat and over a day +# on a 7h one - and the heartbeat window ALREADY encodes the tolerance. An 11m +# window on a 5-minute push is precisely "one missed push forgiven"; stacking a +# threshold of 3 on top triples a tolerance that was already chosen. +# +# Hence: push endpoints alert on the FIRST heartbeat failure. Pulled endpoints +# have no built-in tolerance, so the threshold is where it belongs for them. +# ───────────────────────────────────────────────────────────────────────────── - name: Register the per-service probes with Gatus hosts: observability become: yes @@ -36,12 +53,19 @@ 'name': item.name, 'group': 'probe', 'token': gatus_push_tokens[item.host], - 'heartbeat': '16m'}] }}" + 'heartbeat': '11m'}] }}" loop: "{{ probes }}" - name: Register the probe endpoints ansible.builtin.include_role: name: gatus_endpoint vars: + gatus_endpoint_default_alerts: + - type: signal + # 1, not 3: the heartbeat window is the tolerance. See the note above. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 6h gatus_endpoint_name: probes gatus_endpoint_external: "{{ probe_endpoints }}" diff --git a/ansible/playbooks/backups.yml b/ansible/playbooks/backups.yml index 61c005d..0be3807 100644 --- a/ansible/playbooks/backups.yml +++ b/ansible/playbooks/backups.yml @@ -65,13 +65,17 @@ store_sources: [arbret, headscale, memos, vaultwarden, lnbits, forgejo] tasks: + # 26h, not 7h: the DUMP is genuinely daily, so the window cannot be tighter + # than a day plus slack. The store-side check catches the same fault within + # 6h by reading the artefact's dump timestamp out of the filename, so this is + # the slow backstop rather than the primary signal. - name: Build the dump endpoint list ansible.builtin.set_fact: dump_endpoints: "{{ dump_endpoints | default([]) + [{ 'name': item.name, 'group': 'backup-dump', 'token': gatus_push_tokens[item.host], - 'heartbeat': '30h'}] }}" + 'heartbeat': '26h'}] }}" loop: "{{ dump_sources }}" - name: Build the store endpoint list @@ -80,16 +84,24 @@ 'name': item, 'group': 'backup-store', 'token': gatus_push_tokens['small_backups_local'], - 'heartbeat': '30h'}] }}" + 'heartbeat': '7h'}] }}" loop: "{{ store_sources }}" - name: Register the backup endpoints ansible.builtin.include_role: name: gatus_endpoint vars: + # Push endpoints: the heartbeat window is the tolerance, so alert on + # the first failure rather than waiting for three 7h windows to pass. + gatus_endpoint_default_alerts: + - type: signal + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + minimum-reminder-interval: 12h gatus_endpoint_name: backups gatus_endpoint_external: "{{ dump_endpoints + store_endpoints + [{ 'name': 'pull job', 'group': 'backup-store', 'token': gatus_push_tokens['small_backups_local'], - 'heartbeat': '30h'}] }}" + 'heartbeat': '7h'}] }}" diff --git a/ansible/roles/backup_store/defaults/main.yml b/ansible/roles/backup_store/defaults/main.yml index 3a0754c..e6d17a3 100644 --- a/ansible/roles/backup_store/defaults/main.yml +++ b/ansible/roles/backup_store/defaults/main.yml @@ -17,9 +17,12 @@ backup_store_sources: [] backup_store_check_push_base: "" backup_store_check_push_token: "" -# Runs after the 04:00 pull. Late enough that a slow pull has finished, early -# enough that a failure is visible before the working day. -backup_store_check_on_calendar: "*-*-* 05:30:00" +# Every six hours, offset past the 04:00 pull so the first run of the day sees a +# finished pull. The BACKUPS are daily, but this check is not - it reads the +# source's dump timestamp out of the artefact filename, so running it more often +# catches "the source stopped dumping" within hours rather than a day, and lets +# the Gatus heartbeat be 7h instead of 30h. +backup_store_check_on_calendar: "*-*-* 05:30:00,11:30:00,17:30:00,23:30:00" # An artefact older than this is stale. Sources dump daily at 02:00-02:30 and the # pull is at 04:00, so 26h tolerates exactly one missed night before alarming. diff --git a/ansible/roles/gatus/defaults/main.yml b/ansible/roles/gatus/defaults/main.yml index 9c7cb12..e8bf22f 100644 --- a/ansible/roles/gatus/defaults/main.yml +++ b/ansible/roles/gatus/defaults/main.yml @@ -100,3 +100,9 @@ gatus_self_check: true # container would get free only if it ran as root; it does not. Set false if # you never use icmp:// checks and want the capability dropped entirely. gatus_allow_icmp: true + +# ── Shared network ─────────────────────────────────────────────────────────── +# Gatus runs in a container, so the HOST's loopback is not reachable from it. +# Anything Gatus must talk to locally - the Signal API that sends its alerts - +# has to be on a shared docker network and addressed by service name. +gatus_network: monitoring diff --git a/ansible/roles/gatus/tasks/configure.yml b/ansible/roles/gatus/tasks/configure.yml index 94f71f0..ef5d7e7 100644 --- a/ansible/roles/gatus/tasks/configure.yml +++ b/ansible/roles/gatus/tasks/configure.yml @@ -5,6 +5,16 @@ changed_when: false failed_when: gatus_docker_check.rc != 0 +# Created explicitly rather than by either compose file, so neither the gatus +# stack nor the signal-api stack has to be deployed before the other. +- name: Ensure the shared monitoring network exists + ansible.builtin.command: "docker network create {{ gatus_network }}" + register: gatus_net + changed_when: "'already exists' not in gatus_net.stderr" + failed_when: + - gatus_net.rc != 0 + - "'already exists' not in gatus_net.stderr" + - name: Create the gatus directories ansible.builtin.file: path: "{{ item.path }}" diff --git a/ansible/roles/gatus/templates/docker-compose.yml.j2 b/ansible/roles/gatus/templates/docker-compose.yml.j2 index 67d7573..512c9c5 100644 --- a/ansible/roles/gatus/templates/docker-compose.yml.j2 +++ b/ansible/roles/gatus/templates/docker-compose.yml.j2 @@ -37,8 +37,17 @@ services: - NET_RAW {% endif %} + networks: + # Shared with signal-api, so alerts can be delivered by service name. + # 127.0.0.1 inside this container is the container, not the host. + - {{ gatus_network }} + logging: driver: json-file options: max-size: "10m" max-file: "3" + +networks: + {{ gatus_network }}: + external: true diff --git a/ansible/roles/gatus_endpoint/defaults/main.yml b/ansible/roles/gatus_endpoint/defaults/main.yml index 89b669d..5c330cc 100644 --- a/ansible/roles/gatus_endpoint/defaults/main.yml +++ b/ansible/roles/gatus_endpoint/defaults/main.yml @@ -31,3 +31,14 @@ gatus_endpoint_external: [] gatus_config_dir: /opt/gatus/config gatus_endpoints_dir: "{{ gatus_config_dir }}/endpoints" gatus_gid: 10001 + +# Alerts attached to every endpoint in this file that does not specify its own. +# +# Gatus's provider-level `default-alert` only supplies DEFAULTS - an endpoint +# still has to opt in with `alerts: - type: signal` or it alerts on nothing at +# all. With ~90 endpoints that cannot be written by hand, so it is applied here. +# +# failure-threshold is set by the CALLER, because the right value depends on the +# check's cadence and there is no single correct default. See the note in +# infra/400_host_monitoring.yml. +gatus_endpoint_default_alerts: [] diff --git a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 index fa13d1f..54340cd 100644 --- a/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 +++ b/ansible/roles/gatus_endpoint/templates/endpoints.yaml.j2 @@ -14,9 +14,10 @@ external-endpoints: heartbeat: interval: {{ e.heartbeat }} {% endif %} -{% if e.alerts | default([]) %} +{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} +{% if _alerts %} alerts: -{{ e.alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} {% endif %} {% endfor %} {% endif %} @@ -44,9 +45,10 @@ endpoints: {% for c in e.conditions %} - "{{ c }}" {% endfor %} -{% if e.alerts | default([]) %} +{% set _alerts = e.alerts | default(gatus_endpoint_default_alerts) %} +{% if _alerts %} alerts: -{{ e.alerts | to_nice_yaml(indent=2) | indent(6, true) }} +{{ _alerts | to_nice_yaml(indent=2) | indent(6, true) }} {% endif %} {% endfor %} {% endif %} diff --git a/ansible/roles/signal_api/README.md b/ansible/roles/signal_api/README.md new file mode 100644 index 0000000..eadcfb7 --- /dev/null +++ b/ansible/roles/signal_api/README.md @@ -0,0 +1,133 @@ +# signal_api + +Runs [signal-cli-rest-api](https://github.com/bbernhard/signal-cli-rest-api) on +the `observability` host. Gatus uses it to deliver alerts over Signal. + +Gatus does not speak Signal — it POSTs JSON to this service, which holds the +Signal identity and does the protocol work. + +## It is never published, and that is not optional + +**This API has no authentication of any kind.** No key, no token, no basic auth. +Anything that can reach the port can send messages as your identity and read +your Signal. So the compose file publishes **no ports at all** and there is no +Caddy vhost. + +Gatus reaches it over a shared docker network (`monitoring`) by service name: +`http://signal-api:8080`. That is also *why* a shared network is needed rather +than a published port — Gatus runs in a container, so `127.0.0.1` for Gatus is +the Gatus container, not the host. + +The network is created by an explicit Ansible task in both this role and +`gatus`, so neither stack has to be deployed before the other. + +## MODE, and why `native` + +Upstream offers `normal`, `native`, `json-rpc` and `json-rpc-native`. The +json-rpc modes keep a resident JVM daemon and upstream describes them as +"increased memory". + +**This VPS has 464 MB of RAM**, already running Gatus and Caddy. A resident JVM +is not affordable. `native` runs a precompiled GraalVM binary per request — no +daemon, no resident cost — and alerts are rare enough that paying startup cost +per alert is the right trade. + +## Linking the device — a one-time manual step + +Ansible cannot scan a QR code, so this is manual. **Do not use +`/v1/qrcodelink`** — it is broken in `native` mode. + +### The trap + +`GET /v1/qrcodelink?device_name=...` returns: + +```json +{"error":"Couldn't create QR code: no data to encode"} +``` + +The linking itself is fine: running the binary directly inside the container +emits a perfectly good provisioning URI. + +``` +$ docker exec signal-api signal-cli-native link -n gatus +sgnl://linkdevice?uuid=...&pub_key=... +``` + +It is the REST wrapper that fails to capture that output in `native` mode. + +**Do not "fix" this by switching MODE to `normal` or `json-rpc`.** That puts a +JVM in the path of *every alert* on a 464 MB host, permanently degrading the +running system to work around a step performed once. Generate the QR yourself +instead. + +### The procedure + +**`docker exec` runs as root, but the service runs as uid 1000.** Without +`--config`, signal-cli writes the linked account to `/root/.local/share/signal-cli` +— the container's ephemeral layer, NOT the mounted volume. It looks like it +worked (`Associated with: +34…`), `/v1/accounts` keeps returning `[]`, and the +account is destroyed on the next `docker compose up`. Always pass `--config`. + +1. Start the link and capture the URI. It must keep running while you scan: + + docker exec signal-api sh -c "rm -f /tmp/link.uri; \ + nohup signal-cli-native --config /home/.local/share/signal-cli \ + link -n gatus > /tmp/link.uri 2>/tmp/link.log & echo started" + sleep 10 + docker exec signal-api cat /tmp/link.uri + + Do **not** add `setsid`, and do **not** background `docker exec` itself from + the host — the first stops the URI appearing, the second is killed when the + Ansible task returns. The output is block-buffered because stdout is a file, + so the URI appears only after several seconds; `stdbuf` does not help, as the + buffering is GraalVM's, not libc's. + +2. Render the QR on your own machine and scan it: + + qrencode -o /tmp/qr.png -s 12 -m 4 "sgnl://linkdevice?uuid=...&pub_key=..." + +3. Phone: Signal → Settings → Linked devices → **+** → scan. Provisioning links + expire in a couple of minutes, so generate and scan in one sitting. + +4. Confirm — this must list the number, not `[]`: + + docker exec signal-api curl -s http://localhost:8080/v1/accounts + +5. Send a test message: + + docker exec signal-api curl -s -X POST -H "Content-Type: application/json" \ + -d '{"message":"test","number":"+34…","recipients":["+34…"]}' \ + http://localhost:8080/v2/send + +Alerts are sent **from your own number**, so sending to yourself lands in Note +to Self. If the device is ever unlinked from the phone, alerts stop silently — +which is why this service is itself monitored. + +### If the phone says "network error" + +The phone is not the problem. `chat.signal.org` resolves to AWS Global +Accelerator **dualstack** addresses with the AAAA records first, this container +has no IPv6 address at all, and this host's IPv6 path is broken — the same edge +that returned a bogus 404 for the Go tarball. signal-cli reaches for an +unreachable IPv6 address and dies with `Link request error: Connection closed!`, +while the phone can only report a failed handshake. + +That is what `gai.conf` (mounted at `/etc/gai.conf`) fixes. If linking starts +failing again, check it is still mounted and that `getent ahosts chat.signal.org` +returns an IPv4 address first. + +## Backups + +Deliberately **not** backed up. The data directory holds Signal private keys, +and the recovery path is to link again from the phone — which takes a minute and +does not depend on any stored artefact. Backing it up would copy a credential +off the host to buy nothing. + +## Verifying + +```bash +docker ps --filter name=signal-api +docker exec signal-api curl -fsS http://localhost:8080/v1/health +docker exec signal-api curl -fsS http://localhost:8080/v1/accounts +docker logs signal-api --tail 50 +``` diff --git a/ansible/roles/signal_api/defaults/main.yml b/ansible/roles/signal_api/defaults/main.yml new file mode 100644 index 0000000..6bb7d4f --- /dev/null +++ b/ansible/roles/signal_api/defaults/main.yml @@ -0,0 +1,39 @@ +--- +# signal-cli-rest-api: the transport Gatus uses to send Signal messages. +# +# Gatus does not speak Signal. It POSTs JSON to this service, which holds the +# actual Signal identity and does the protocol work. + +# Pinned by digest for the same reason as Gatus: a tag is mutable. +# Upstream publishes no versioned tags worth pinning to, so this pins the +# DIGEST that `latest` resolved to when this was reviewed. `latest` is a moving +# target; a digest is a content address, and `docker compose pull` either +# fetches exactly this image or fails. +signal_api_image_digest: "sha256:2399d449123cdad56c4d859277e3b9127e1a00c4d2ab4601c239882609286cf8" +signal_api_image: "bbernhard/signal-cli-rest-api@{{ signal_api_image_digest }}" + +signal_api_dir: /opt/signal-api +signal_api_data_dir: "{{ signal_api_dir }}/data" + +# MODE matters on this host. Upstream offers normal / native / json-rpc / +# json-rpc-native. json-rpc keeps a resident JVM daemon and upstream describes it +# as "increased memory" - this VPS has 464MB total and already runs Gatus and +# Caddy, so a resident JVM is not affordable. `native` runs a precompiled +# GraalVM binary per request: no daemon, no resident cost, and alerts are rare +# enough that paying startup per alert is the right trade. +signal_api_mode: native + +# Port INSIDE the shared docker network. Never published to the host: this API +# has NO AUTHENTICATION of any kind. Anyone who can reach it can send messages +# as you and read your Signal. +signal_api_port: 8080 + +# Both this and Gatus join this network so Gatus can reach the API by service +# name. Gatus runs in a container, so the host's loopback is NOT reachable from +# it - this is why a shared network is required rather than a published port. +signal_api_network: monitoring +signal_api_service_name: signal-api + +# The uid the upstream image drops to (`setpriv --reuid=1000`). The data +# directory must be owned by it or signal-cli cannot write the account. +signal_api_uid: 1000 diff --git a/ansible/roles/signal_api/tasks/main.yml b/ansible/roles/signal_api/tasks/main.yml new file mode 100644 index 0000000..1aa92b4 --- /dev/null +++ b/ansible/roles/signal_api/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: Assert Docker is available + ansible.builtin.command: docker --version + register: signal_docker_check + changed_when: false + +# Created explicitly rather than by either compose file, so neither stack has to +# be deployed before the other and neither owns it. +- name: Ensure the shared monitoring network exists + ansible.builtin.command: "docker network create {{ signal_api_network }}" + register: signal_net + changed_when: "'already exists' not in signal_net.stderr" + failed_when: + - signal_net.rc != 0 + - "'already exists' not in signal_net.stderr" + +- name: Create the signal-api directory + ansible.builtin.file: + path: "{{ signal_api_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +# Owned by the container's uid, NOT root. +# +# The image drops to uid 1000 (`setpriv --reuid=1000`), and a root-owned 0700 +# directory cannot be traversed by uid 1000 - signal-cli then fails to write the +# account and linking silently never completes, leaving a 39-byte accounts.json +# with no accounts and the API returning "Failed to read local accounts list". +# +# 0700 on uid 1000 is still private: only that uid and root can read the Signal +# private keys, which is the property actually wanted. +- name: Create the signal-api data directory owned by the container user + ansible.builtin.file: + path: "{{ signal_api_data_dir }}" + state: directory + owner: "{{ signal_api_uid }}" + group: "{{ signal_api_uid }}" + mode: "0700" + +- name: Write the IPv4-preference resolver config + ansible.builtin.template: + src: gai.conf.j2 + dest: "{{ signal_api_dir }}/gai.conf" + owner: root + group: root + mode: "0644" + +- name: Write the docker compose file + ansible.builtin.template: + src: docker-compose.yml.j2 + dest: "{{ signal_api_dir }}/docker-compose.yml" + owner: root + group: root + mode: "0644" + +- name: Pull the pinned signal-api image + ansible.builtin.command: + cmd: docker compose pull + chdir: "{{ signal_api_dir }}" + register: signal_pull + changed_when: "'Downloaded newer image' in signal_pull.stderr or 'Pull complete' in signal_pull.stderr" + +- name: Start signal-api + ansible.builtin.command: + cmd: docker compose up -d --remove-orphans + chdir: "{{ signal_api_dir }}" + register: signal_up + changed_when: "'Started' in signal_up.stderr or 'Created' in signal_up.stderr or 'Recreated' in signal_up.stderr" + +- name: Wait for the API to answer + ansible.builtin.command: + cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/health" + register: signal_health + until: signal_health.rc == 0 + retries: 12 + delay: 5 + changed_when: false + +- name: Report whether an account is linked yet + ansible.builtin.command: + cmd: "docker exec {{ signal_api_service_name }} curl -fsS http://localhost:{{ signal_api_port }}/v1/accounts" + register: signal_accounts + changed_when: false + failed_when: false + +- name: Show the linking status + ansible.builtin.debug: + msg: >- + {{ 'Linked account(s): ' ~ signal_accounts.stdout + if (signal_accounts.stdout | default('[]') | trim) not in ['[]', '', 'null'] + else 'NO ACCOUNT LINKED YET - this is a one-time manual step, see the role README.' }} diff --git a/ansible/roles/signal_api/templates/docker-compose.yml.j2 b/ansible/roles/signal_api/templates/docker-compose.yml.j2 new file mode 100644 index 0000000..9825299 --- /dev/null +++ b/ansible/roles/signal_api/templates/docker-compose.yml.j2 @@ -0,0 +1,47 @@ +# Managed by Ansible (roles/signal_api) +services: + {{ signal_api_service_name }}: + image: {{ signal_api_image }} + container_name: {{ signal_api_service_name }} + restart: unless-stopped + + environment: + MODE: "{{ signal_api_mode }}" + + volumes: + # Prefer IPv4. See gai.conf.j2 - without this, signal-cli reaches for + # chat.signal.org's IPv6 address, which is unreachable from here, and + # linking fails with an opaque "network error" on the phone. + - {{ signal_api_dir }}/gai.conf:/etc/gai.conf:ro + # Holds the Signal identity: the linked-device keys and registration + # state. Lose this and the device must be linked again by scanning a new + # QR code from the phone. It is also the most sensitive thing on this + # host - anyone with these keys can send and read Signal as you. + - {{ signal_api_data_dir }}:/home/.local/share/signal-cli + + networks: + - {{ signal_api_network }} + + # NO PORTS. Deliberately. + # + # This API has no authentication whatsoever - no key, no token, nothing. + # Publishing it, even on 127.0.0.1, would expose "send a Signal message as + # this identity" to anything that can reach the host. Gatus talks to it over + # the shared docker network by service name instead, which is why no port is + # published and why there is no Caddy vhost. + + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8080/v1/health"] + interval: 60s + timeout: 5s + retries: 3 + + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + +networks: + {{ signal_api_network }}: + external: true diff --git a/ansible/roles/signal_api/templates/gai.conf.j2 b/ansible/roles/signal_api/templates/gai.conf.j2 new file mode 100644 index 0000000..9b2cca4 --- /dev/null +++ b/ansible/roles/signal_api/templates/gai.conf.j2 @@ -0,0 +1,17 @@ +# Managed by Ansible (roles/signal_api) +# +# Prefer IPv4 over IPv6 in getaddrinfo. +# +# chat.signal.org resolves to AWS Global Accelerator dualstack addresses, and +# DNS returns the AAAA records first. This container has NO IPv6 address at all, +# and this host's IPv6 path is unreliable anyway - the same edge that made +# Google's IPv6 endpoint return a confident 404 for the Go tarball. +# +# signal-cli would connect to the AAAA address, fail, and report +# Link request error: Connection closed! +# while the phone showed a bare "network error" - a failure with no obvious +# cause on either end. +# +# This line flips the precedence so IPv4-mapped addresses sort first, which is +# the standard glibc fix. It does NOT disable IPv6; it only changes the order. +precedence ::ffff:0:0/96 100 diff --git a/ansible/services/gatus/deploy_gatus_playbook.yml b/ansible/services/gatus/deploy_gatus_playbook.yml index 704ada9..d6a5ac1 100644 --- a/ansible/services/gatus/deploy_gatus_playbook.yml +++ b/ansible/services/gatus/deploy_gatus_playbook.yml @@ -7,6 +7,29 @@ - name: Deploy Gatus on the observability host hosts: observability become: yes + vars: + gatus_alerting: + signal: + # NOTE: the key is `api-url`, not `url` as upstream's own README table + # says - see alerting/provider/signal/signal.go. Gatus appends /v2/send + # itself if the suffix is missing. + # + # Reached by service name over the shared docker network. Gatus runs in + # a container, so 127.0.0.1 here would be the Gatus container, not the + # host - and the Signal API deliberately publishes no ports because it + # has no authentication. + api-url: "http://signal-api:8080" + number: "{{ signal_number }}" + recipients: "{{ signal_recipients }}" + default-alert: + # Overridden per group by the registration playbooks; these are the + # values that apply if a caller sets nothing. + failure-threshold: 1 + success-threshold: 2 + send-on-resolved: true + # An ongoing outage should not become an ongoing phone buzz. + minimum-reminder-interval: 6h + roles: - gatus diff --git a/ansible/services/signal-api/deploy_signal_api_playbook.yml b/ansible/services/signal-api/deploy_signal_api_playbook.yml new file mode 100644 index 0000000..d4690f9 --- /dev/null +++ b/ansible/services/signal-api/deploy_signal_api_playbook.yml @@ -0,0 +1,44 @@ +--- +# The Signal transport for Gatus alerts. +# +# Deliberately NOT published and NOT fronted by Caddy: the API has no +# authentication of any kind, so it is reachable only from the shared docker +# network that Gatus is on. See roles/signal_api/README.md, including the +# one-time manual step to link the device. +- name: Deploy the Signal API on the observability host + hosts: observability + become: yes + roles: + - signal_api + + # post_tasks, not a second play: the registration below needs the role's + # defaults (service name, port) in scope, and a separate play would not have + # them. + post_tasks: + # Monitored, because a dead alert transport is the worst kind of dead: every + # check could be failing and nothing would tell you. Gatus polls it over the + # shared network - the same path the alerts take - so this proves the actual + # delivery route rather than merely that a container is running. + # + # Deliberately NOT backed up: the data directory holds Signal private keys, + # and the recovery path is to link the device again from the phone. Backing + # it up would copy a credential off the host to buy nothing. + - name: Register the signal-api health endpoint with Gatus + ansible.builtin.include_role: + name: gatus_endpoint + vars: + gatus_endpoint_name: signal-api + gatus_endpoint_pulled: + - name: signal-api + group: infrastructure + url: "http://{{ signal_api_service_name }}:{{ signal_api_port }}/v1/health" + interval: 5m + # Deliberately NOT alerted via Signal: if this endpoint is down, + # Signal is exactly what cannot deliver the alert. It is visible on + # the dashboard, and its failure shows up indirectly as every other + # alert going missing. + conditions: + # /v1/health answers 204 No Content, not 200 - checked live. Any + # 2xx is asserted rather than the exact code, so an upstream + # change from 204 to 200 does not read as an outage. + - "[STATUS] < 300" diff --git a/ansible/site.yml b/ansible/site.yml index 10ee4a7..35f5867 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -26,6 +26,10 @@ # Gatus first: the three plays below register endpoints with it, and registering # against a host that is not serving yet would simply fail. - import_playbook: services/gatus/deploy_gatus_playbook.yml +# The Signal transport for Gatus alerts. Shares a docker network with Gatus and +# publishes no ports - the API has no authentication. Needs a one-time manual +# device link; see roles/signal_api/README.md. +- import_playbook: services/signal-api/deploy_signal_api_playbook.yml - import_playbook: infra/400_host_monitoring.yml - import_playbook: infra/401_service_monitoring.yml - import_playbook: infra/402_public_monitoring.yml